diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a579afef6e..f26082d114 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -4,12 +4,14 @@ on: push: branches: - master + - Domathon + - SecretGoodStarPilot pull_request: workflow_call: inputs: run_number: default: '1' - required: true + required: false type: string concurrency: group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} diff --git a/.github/workflows/safety.yaml b/.github/workflows/safety.yaml new file mode 100644 index 0000000000..2faeb8b962 --- /dev/null +++ b/.github/workflows/safety.yaml @@ -0,0 +1,32 @@ +name: safety qualification + +on: + push: + branches: [master, Domathon, SecretGoodStarPilot] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + host_gates: + name: ${{ matrix.gate }} + runs-on: ubuntu-24.04 + timeout-minutes: 100 + strategy: + fail-fast: false + matrix: + gate: [coverage, opendbc-misra, panda-misra, panda-host] + steps: + - uses: actions/checkout@v7 + - run: ./tools/op.sh setup + - name: Run isolated source-pinned gate + run: .venv/bin/python tools/ci/run_safety_qualification.py --gate "${{ matrix.gate }}" --output "ci-results/safety-${{ matrix.gate }}" + - name: Save full gate evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: safety-${{ matrix.gate }} + path: ci-results/safety-${{ matrix.gate }}/ + if-no-files-found: error diff --git a/.github/workflows/simulator-driving.yaml b/.github/workflows/simulator-driving.yaml new file mode 100644 index 0000000000..d43aa13b4e --- /dev/null +++ b/.github/workflows/simulator-driving.yaml @@ -0,0 +1,28 @@ +name: simulator driving (manual) + +on: + workflow_dispatch: + +permissions: + contents: read + +env: + CI: 1 + PYTHONPATH: ${{ github.workspace }} + +jobs: + simulator_driving: + name: simulator driving + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - run: ./tools/op.sh setup + - name: Build openpilot + run: scons + - name: Driving test + timeout-minutes: 2 + env: + # MetaDrive renders offscreen through panda3d's EGL pipe on llvmpipe + EGL_PLATFORM: surfaceless + run: python openpilot/tools/sim/tests/test_metadrive_bridge.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d7386e9d3c..24b6ae8332 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -5,19 +5,23 @@ on: branches: - master - Domathon + - SecretGoodStarPilot pull_request: workflow_dispatch: workflow_call: inputs: run_number: default: '1' - required: true + required: false type: string concurrency: group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} cancel-in-progress: true +permissions: + contents: read + env: CI: 1 PYTHONPATH: ${{ github.workspace }} @@ -32,12 +36,25 @@ jobs: - uses: actions/checkout@v7 - name: Validate tracked dependencies run: python3 tools/vendor/check.py --revision HEAD - - name: Test dependency and packaging tools - run: | - python3 -m unittest discover -s tools/vendor/tests -v - python3 -m unittest discover -s tools/release/tests -v - python3 -m unittest discover -s openpilot/system/updated/tests -v + - run: ./tools/op.sh setup + id: setup + - name: Build native dependencies for packaging tests + id: build + run: scons -j2 openpilot/common/libparams_c.so msgq_repo/msgq/ipc_pyx.so + - name: Test dependency tools + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: python3 -m unittest discover -s tools/vendor/tests -v + - name: Test release packaging + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: python3 -m pytest -q tools/release/tests + - name: Test CI tools + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: python3 -m unittest discover -s tools/ci/tests -v + - name: Test updater + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: python3 -m unittest discover -s openpilot/system/updated/tests -v - name: Check setup and release shell syntax + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: | for script in tools/op.sh tools/release/*.sh openpilot/selfdrive/test/setup_device_ci.sh; do bash -n "$script" @@ -45,17 +62,24 @@ jobs: build_release: name: build release - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16"]') - || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04-arm env: STRIPPED_DIR: /tmp/releasepilot PYTHONPATH: /tmp/releasepilot + STARPILOT_CI_BUILD_MANIFEST: /tmp/starpilot-release-build-${{ github.run_id }}-${{ github.run_attempt }}.json steps: - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: '1.25.1' + cache-dependency-path: mapd_repo/go.sum + - name: Build the pinned release map provider + env: + GOTOOLCHAIN: local + run: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends pkg-config zlib1g-dev + (cd mapd_repo && go mod download) + PYTHONPATH="$GITHUB_WORKSPACE" python3 -m openpilot.starpilot.maps.package_shadow - name: Build devel timeout-minutes: 1 run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh @@ -72,6 +96,38 @@ jobs: timeout-minutes: 3 run: python3 tools/vendor/check.py --revision HEAD + developer_commands: + name: developer commands (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest] + runs-on: ${{ matrix.os }} + permissions: + contents: read + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - name: Install developer prerequisites + run: tools/setup_dependencies.sh + - name: Check launchers without starting tools + run: | + ./build --help + ./dev help + ./test --help + ./tool help + ./tools/host help + ./c3 --help + ./c4 --help + ./onroad --help + - name: Test commands through the isolated host environment + run: | + ./dev pytest -q tools/ci/tests/test_dev_runtime.py tools/ci/tests/test_dev_runtime_review.py tools/ci/tests/test_build_entry.py openpilot/starpilot/ui/tests/test_host_launch.py openpilot/tools/replay/tests/test_onroad_host.py openpilot/starpilot/ui/tests/test_host_window_failure.py + - name: Build both desktop UI entrypoints without a display + run: | + SP_C3_COMPILE_ONLY=1 ./c3 + SP_C4_COMPILE_ONLY=1 ./c4 + build_mac: name: build macOS runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }} @@ -110,14 +166,196 @@ jobs: || fromJSON('["ubuntu-24.04"]') }} steps: - uses: actions/checkout@v7 + - name: Provide JavaScript runtime for Galaxy contract tests + uses: actions/setup-node@v7 + with: + node-version: '24' + package-manager-cache: false + - name: Provide pinned map provider toolchain + uses: actions/setup-go@v7 + with: + go-version: '1.25.1' + cache-dependency-path: mapd_repo/go.sum + - name: Fetch pinned map provider dependencies + working-directory: mapd_repo + env: + GOTOOLCHAIN: local + run: go mod download - run: ./tools/op.sh setup + id: setup - name: Build openpilot + id: build run: scons - name: Run unit tests + if: ${{ !cancelled() && steps.build.outcome == 'success' }} timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }} env: RAYLIB_BACKEND: headless - run: tools/op.sh test + run: tools/op.sh test --json-output unit-test-results.json + - name: Test custom module-level pytest cases + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + timeout-minutes: 10 + env: + RAYLIB_BACKEND: headless + run: python3 tools/ci/run_host_tests.py --pytest + - name: Test GM vehicle configurations and CAN behavior + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + timeout-minutes: 5 + run: tools/op.sh test opendbc_repo/opendbc/car/gm/tests --json-output gm-test-results.json + - name: Test native independent-axis protocol + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + openpilot/selfdrive/pandad/tests/test_panda_usb + openpilot/selfdrive/pandad/tests/test_pandad_canprotocol + openpilot/selfdrive/pandad/tests/test_aol_protocol + openpilot/selfdrive/pandad/tests/test_aol_wire + openpilot/selfdrive/pandad/tests/test_producer_timing + openpilot/selfdrive/pandad/tests/test_producer_sink + - name: Test map provider to host messaging + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + working-directory: mapd_repo + timeout-minutes: 5 + env: + GOTOOLCHAIN: local + STARPILOT_SHADOW_IPC_REQUIRED: '1' + STARPILOT_SHADOW_EXEC_REQUIRED: '1' + STARPILOT_PYTHON: ${{ github.workspace }}/.venv/bin/python + run: go test -mod=readonly -race -run '^Test(MsgqNativeLayout|Shadow(GoPythonIPC|ExecutableHostGpsIPC))$' -count=1 -v + - name: Save complete unit test results + if: ${{ always() && (hashFiles('unit-test-results.json') != '' || hashFiles('gm-test-results.json') != '') }} + uses: actions/upload-artifact@v4 + with: + name: unit-test-results + path: | + unit-test-results.json + gm-test-results.json + if-no-files-found: error + + map_provider: + name: map provider + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: '1.25.1' + cache-dependency-path: mapd_repo/go.sum + - name: Install native build dependencies + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends pkg-config zlib1g-dev + - name: Test and build pinned map provider + run: python3 tools/ci/run_map_tests.py --output ci-results/map-provider + - name: Save test and build evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: map-provider + path: ci-results/map-provider/ + if-no-files-found: error + + vehicle_contracts: + name: vehicle contracts (${{ matrix.suite }}) + runs-on: ubuntu-24.04 + timeout-minutes: 35 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + suite: [interfaces, safety-debug, safety-release] + env: + FUZZ_SEED: '0' + steps: + - uses: actions/checkout@v7 + - run: ./tools/op.sh setup + - name: Build native host dependencies + run: scons + - name: Run explicit vehicle suite and record every result + timeout-minutes: 20 + run: python3 tools/ci/run_vehicle_tests.py --suite "${{ matrix.suite }}" --output "ci-results/${{ matrix.suite }}" + - name: Require current complete interface regression coverage + if: ${{ always() && matrix.suite == 'interfaces' }} + run: | + python3 tools/ci/fleet_coverage.py --require interfaces \ + --interface-coverage ci-results/interfaces/coverage.json \ + --interface-results ci-results/interfaces/results.json \ + --output ci-results/interfaces/fleet-coverage.json + - name: Save collection, outcomes, and native build evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: vehicle-contracts-${{ matrix.suite }} + path: ci-results/${{ matrix.suite }}/ + if-no-files-found: error + + recorded_vehicles: + name: recorded vehicles (${{ matrix.variant }}) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + variant: [debug, release] + env: + FUZZ_SEED: '0' + steps: + - uses: actions/checkout@v7 + - name: Install host prerequisites + run: tools/setup_dependencies.sh + - name: Replay pinned public CAN recordings + timeout-minutes: 20 + run: | + ./dev python tools/ci/run_recorded_vehicle_tests.py \ + --variant "${{ matrix.variant }}" \ + --cache "${{ runner.temp }}/recorded-fixtures" \ + --output "$GITHUB_WORKSPACE/ci-results/recorded-${{ matrix.variant }}" \ + --download + - name: Save recorded results and native build evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: recorded-vehicles-${{ matrix.variant }} + path: ci-results/recorded-${{ matrix.variant }}/ + if-no-files-found: error + + panda_build_variants: + name: Panda compile (${{ matrix.variant }}) + runs-on: ubuntu-24.04 + timeout-minutes: 35 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + variant: [debug, release] + steps: + - uses: actions/checkout@v7 + - run: ./tools/op.sh setup + - name: Compile firmware with the checked-in development certificate + timeout-minutes: 15 + shell: bash + run: | + set -euo pipefail + mkdir -p ci-results/firmware + if [[ "${{ matrix.variant }}" == release ]]; then + export RELEASE=1 + export CERT="$PWD/panda/board/crypto/certs/debug" + else + unset RELEASE + fi + scons --verbose panda/board/obj/version panda/board/obj/panda.bin.signed panda/board/obj/bootstub.panda.bin panda/board/obj/panda_h7.bin.signed panda/board/obj/bootstub.panda_h7.bin 2>&1 | tee ci-results/firmware/build.log + python3 tools/ci/record_firmware.py --variant "${{ matrix.variant }}" --build-log ci-results/firmware/build.log --output ci-results/firmware/provenance.json + - name: Save compile evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: panda-compile-${{ matrix.variant }} + path: ci-results/firmware/ + if-no-files-found: error process_replay: name: process replay @@ -137,10 +375,17 @@ jobs: run: openpilot/selfdrive/test/process_replay/test_processes.py -j$(nproc) - name: Print diff id: print-diff - if: always() + if: ${{ always() && hashFiles('openpilot/selfdrive/test/process_replay/diff.txt') != '' }} run: cat openpilot/selfdrive/test/process_replay/diff.txt - - name: Print diff report + - name: Save process replay input coverage if: always() + uses: actions/upload-artifact@v4 + with: + name: process-replay-coverage + path: openpilot/selfdrive/test/process_replay/coverage.json + if-no-files-found: error + - name: Print diff report + if: ${{ always() && hashFiles('openpilot/selfdrive/test/process_replay/diff_report.txt') != '' }} run: cat openpilot/selfdrive/test/process_replay/diff_report.txt - uses: actions/upload-artifact@v7 if: always() @@ -182,27 +427,6 @@ jobs: max_attempts: 3 command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force - simulator_driving: - name: simulator driving - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16"]') - || fromJSON('["ubuntu-24.04"]') }} - if: false # FIXME: Started to timeout recently - steps: - - uses: actions/checkout@v7 - - run: ./tools/op.sh setup - - name: Build openpilot - run: scons - - name: Driving test - timeout-minutes: 2 - env: - # MetaDrive renders offscreen through panda3d's EGL pipe on llvmpipe - EGL_PLATFORM: surfaceless - run: python openpilot/tools/sim/tests/test_metadrive_bridge.py - create_ui_report: name: Create UI Report runs-on: ${{ diff --git a/openpilot/common/SConscript b/openpilot/common/SConscript index 4e06ae88a1..847d16446a 100644 --- a/openpilot/common/SConscript +++ b/openpilot/common/SConscript @@ -17,3 +17,6 @@ Export('common_python') if GetOption('extras'): env.Program('tests/test_swaglog', 'tests/test_swaglog.cc', LIBS=[_common, 'json11', 'zmq', 'pthread']) + + env.Program('tests/bin/test_params_fork_lock', 'tests/test_params_fork_lock.cc', + LIBS=[_common, 'json11', 'zmq', 'capnp', 'kj', 'pthread']) diff --git a/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript b/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript index fa249765bc..ee68cc29cf 100644 --- a/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript +++ b/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript @@ -28,6 +28,8 @@ casadi_cost_0 = [ casadi_constraints = [ f'{gen}/long_constraints/long_constr_h_fun.c', f'{gen}/long_constraints/long_constr_h_fun_jac_uxt_zt.c', + f'{gen}/long_constraints/long_constr_h_e_fun.c', + f'{gen}/long_constraints/long_constr_h_e_fun_jac_uxt_zt.c', ] build_files = [f'{gen}/acados_solver_long.c'] + casadi_model + casadi_cost_y + casadi_cost_e + \ diff --git a/openpilot/starpilot/validation/README.md b/openpilot/starpilot/validation/README.md new file mode 100644 index 0000000000..2bc350eb45 --- /dev/null +++ b/openpilot/starpilot/validation/README.md @@ -0,0 +1,70 @@ +# Control-mode evidence contract + +This module validates recorded evidence. It does not select a driving mode, drive +actuators, change a safety hook, activate a vehicle policy, or qualify a release. +The initial tests contain synthetic validator fixtures only. + +`control-trace.schema.json` describes the JSON format. Run one trace with: + +```sh +python -m openpilot.starpilot.validation.control_modes trace.json --output report.json +``` + +`validate_trace(document)` checks one declared scenario. +`validate_matrix(documents, configurations)` requires every named scenario for +each exact configuration. Its configuration identity includes source, schema, +CarParams, settings, firmware, safety binary, policy, adapter, device and harness. +The caller must enumerate actual configurations; a platform name alone is not a +configuration inventory. Extra metadata is retained in the configuration identity. + +| Selected mode | Host lateral may be active | Host longitudinal may be active | +|---|---|---| +| off | no | no | +| lateral_only | yes | no | +| longitudinal_only | no | yes, only with system ownership | +| combined | yes | yes, only with system ownership | + +Selection is distinct from instantaneous request, safety permission, effective +host state, decoded active control commands and displayed state. Permission does +not by itself mean an axis is active. Command activity means an active actuator +request decoded using the identified vehicle adapter; receipt of a neutral CAN +packet or a safety allowlist decision alone does not establish it. Missing control +CAN must be recorded as unknown, not inactive. Stock ACC activity is a separate +observation and cannot be substituted for host longitudinal commands. + +Each axis records a policy-interpreted blocking fault and input freshness. The +validator does not infer fault scope from alert text, decide which pedal cancels +which axis, or invent a common recovery rule. Each frame supplies explicit expected +effective, command and displayed states from the named vehicle policy. A selected +indicator may intentionally remain visible while authorization is suppressed. +Vehicle policies must preserve deliberate activation-button mappings, pedal and +standstill behavior, steering overrides, cancellation and recovery semantics. + +The matrix requires all four steady modes plus brake, gas, standstill, gear change, +steering override, each axis fault, stale input, permission revocation, cancel, +policy recovery, restart and stock ACC handoff cases. Unsupported requested modes +must demonstrate denial. Steady supported modes require observed active samples; +an all-inactive trace cannot qualify an active mode. Event cases require their +named event and explicit expected states. Event annotations and hashes are supplied +by the adapter; this module does not independently authenticate a recording or +prove that an annotated physical transition occurred. The adapter, policy and +event extraction therefore require separate review and replay tests. An inapplicable +scenario needs an explicit vehicle-policy denial case; it is not silently skipped. + +Results preserve `errors`, `failures` and `uncovered` findings together: + +- `error`: malformed input prevents reliable evaluation. +- `failed`: an observed invariant or explicit policy expectation is violated. +- `uncovered`: observations, provenance, intervals, events or active samples are missing. +- `pass`: this declared trace contract is satisfied within its reported scope. + +Missing axes, unknown booleans, stale feedback and absent CAN never become zeroes. +A synthetic trace can pass its unit contract but cannot satisfy the recorded matrix +gate. A failure is retained even when another run of the same case passes. Every +report states `vehicle_qualification: not_established`; a passing matrix is only +coverage of the supplied configurations and scenarios, not a fleet safety claim. + +No production trace adapter or reviewed vehicle policies are installed yet. No +vehicle matrix is currently qualified by these fixtures. Fresh-install AOL defaults +and preservation of existing saved preferences belong to the separate state +migration contract, not to this observation validator. diff --git a/openpilot/starpilot/validation/control-trace.schema.json b/openpilot/starpilot/validation/control-trace.schema.json new file mode 100644 index 0000000000..3775e312df --- /dev/null +++ b/openpilot/starpilot/validation/control-trace.schema.json @@ -0,0 +1,88 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Recorded control-axis evidence, version 1", + "description": "Unknown observations are null or absent. A valid document or passing trace does not qualify a vehicle.", + "type": "object", + "required": ["schema_version", "scenario", "evidence_kind", "requirements", "frames"], + "properties": { + "schema_version": {"const": 1}, + "case_id": {"type": ["string", "null"]}, + "scenario": {"enum": ["mode_off", "mode_lateral_only", "mode_longitudinal_only", "mode_combined", "brake", "gas", "standstill", "gear_change", "steering_override", "lateral_fault", "longitudinal_fault", "input_stale", "permission_revoked", "cancel", "policy_recovery", "restart", "stock_acc_handoff"]}, + "evidence_kind": {"enum": ["synthetic", "replay", "bench", "vehicle_recording"]}, + "recording_sha256": {"$ref": "#/$defs/hash"}, + "configuration": {"$ref": "#/$defs/configuration"}, + "requirements": { + "type": "object", + "required": ["minimum_frames", "max_frame_gap_ns", "required_events"], + "properties": { + "minimum_frames": {"type": "integer", "minimum": 2}, + "max_frame_gap_ns": {"type": "integer", "minimum": 1}, + "required_events": {"type": "array", "items": {"type": "string", "minLength": 1}} + } + }, + "frames": {"type": "array", "items": {"$ref": "#/$defs/frame"}} + }, + "$defs": { + "hash": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$"}, + "boolean": {"type": ["boolean", "null"]}, + "axes": { + "type": ["object", "null"], + "properties": {"lateral": {"$ref": "#/$defs/boolean"}, "longitudinal": {"$ref": "#/$defs/boolean"}} + }, + "mode": {"enum": ["off", "lateral_only", "longitudinal_only", "combined"]}, + "configuration": { + "type": ["object", "null"], + "properties": { + "platform": {"type": ["string", "null"]}, + "device": {"type": ["string", "null"]}, + "harness": {"type": ["string", "null"]}, + "policy_id": {"type": ["string", "null"]}, + "source_commit": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}$"}, + "car_params_sha256": {"$ref": "#/$defs/hash"}, + "schema_sha256": {"$ref": "#/$defs/hash"}, + "settings_sha256": {"$ref": "#/$defs/hash"}, + "safety_binary_sha256": {"$ref": "#/$defs/hash"}, + "firmware_manifest_sha256": {"$ref": "#/$defs/hash"}, + "policy_sha256": {"$ref": "#/$defs/hash"}, + "adapter_sha256": {"$ref": "#/$defs/hash"}, + "longitudinal_owner": {"enum": ["system", "stock", "none", null]}, + "supported_modes": {"type": ["array", "null"], "uniqueItems": true, "items": {"$ref": "#/$defs/mode"}} + } + }, + "observed": { + "type": ["object", "null"], + "properties": { + "request": {"$ref": "#/$defs/axes"}, + "permission": {"$ref": "#/$defs/axes"}, + "effective": {"$ref": "#/$defs/axes"}, + "command_active": {"$ref": "#/$defs/axes"}, + "blocking_fault": {"$ref": "#/$defs/axes"}, + "input_fresh": {"$ref": "#/$defs/axes"}, + "displayed_active": {"$ref": "#/$defs/axes"}, + "control_can_observed": {"$ref": "#/$defs/boolean"}, + "safety_feedback_fresh": {"$ref": "#/$defs/boolean"}, + "stock_acc_active": {"$ref": "#/$defs/boolean"} + } + }, + "expected": { + "type": ["object", "null"], + "properties": { + "effective": {"$ref": "#/$defs/axes"}, + "command_active": {"$ref": "#/$defs/axes"}, + "displayed_active": {"$ref": "#/$defs/axes"}, + "stock_acc_active": {"$ref": "#/$defs/boolean"} + } + }, + "frame": { + "type": "object", + "required": ["time_ns", "mode"], + "properties": { + "time_ns": {"type": "integer", "minimum": 0}, + "mode": {"$ref": "#/$defs/mode"}, + "event": {"type": ["string", "null"], "minLength": 1}, + "observed": {"$ref": "#/$defs/observed"}, + "expected": {"$ref": "#/$defs/expected"} + } + } + } +} diff --git a/openpilot/starpilot/validation/control_modes.py b/openpilot/starpilot/validation/control_modes.py new file mode 100644 index 0000000000..fef7f05853 --- /dev/null +++ b/openpilot/starpilot/validation/control_modes.py @@ -0,0 +1,250 @@ +"""Check recorded control-axis evidence; this does not control or qualify a car.""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re + +AXES = ("lateral", "longitudinal") +MODES = {"off": (False, False), "lateral_only": (True, False), "longitudinal_only": (False, True), "combined": (True, True)} +SCENARIOS = tuple(f"mode_{mode}" for mode in MODES) + ( + "brake", "gas", "standstill", "gear_change", "steering_override", "lateral_fault", "longitudinal_fault", + "input_stale", "permission_revoked", "cancel", "policy_recovery", "restart", "stock_acc_handoff", +) +AXIS_SIGNALS = ("request", "permission", "effective", "command_active", "blocking_fault", "input_fresh", "displayed_active") +EXPECTED_SIGNALS = ("effective", "command_active", "displayed_active") +CONFIG_TEXT = ("platform", "device", "harness", "policy_id") +CONFIG_HASHES = ("car_params_sha256", "schema_sha256", "settings_sha256", "safety_binary_sha256", + "firmware_manifest_sha256", "policy_sha256", "adapter_sha256") + + +def configuration_id(configuration): + return hashlib.sha256(json.dumps(configuration, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def validate_trace(document): + """Return pass/failed/uncovered/error for this trace's contract, never fleet clearance. + + Null and absent observations are unknown. Expected recovery behavior comes from + a named, hashed vehicle policy and explicit frame expectations, not this module. + """ + errors, failures, gaps = [], [], [] + + def unknown(path): + gaps.append(f"{path}: unknown") + + def mapping(value, path): + if value is None: + unknown(path) + return {} + if not isinstance(value, dict): + errors.append(f"{path}: expected object") + return {} + return value + + def boolean(value, path): + if value is None: + unknown(path) + return None + if type(value) is not bool: + errors.append(f"{path}: expected boolean or null") + return None + return value + + def text(value, path, pattern=None): + if value is None or value == "": + unknown(path) + elif not isinstance(value, str) or (pattern and not re.fullmatch(pattern, value)): + errors.append(f"{path}: invalid string") + + document = mapping(document, "trace") + if document.get("schema_version") != 1 or type(document.get("schema_version")) is not int: + errors.append("schema_version: expected 1") + text(document.get("case_id"), "case_id") + scenario = document.get("scenario") + if scenario not in SCENARIOS: + errors.append("scenario: unknown scenario") + kind = document.get("evidence_kind") + if kind not in ("synthetic", "replay", "bench", "vehicle_recording"): + errors.append("evidence_kind: unknown evidence kind") + text(document.get("recording_sha256"), "recording_sha256", r"[0-9a-f]{64}") + config = mapping(document.get("configuration"), "configuration") + for key in CONFIG_TEXT: + text(config.get(key), f"configuration.{key}") + for key in CONFIG_HASHES: + text(config.get(key), f"configuration.{key}", r"[0-9a-f]{64}") + text(config.get("source_commit"), "configuration.source_commit", r"[0-9a-f]{40}") + owner = config.get("longitudinal_owner") + if owner is None: + unknown("configuration.longitudinal_owner") + elif owner not in ("system", "stock", "none"): + errors.append("configuration.longitudinal_owner: invalid owner") + supported = config.get("supported_modes") + if supported is None: + unknown("configuration.supported_modes") + supported = [] + elif not isinstance(supported, list) or any(not isinstance(m, str) or m not in MODES for m in supported): + errors.append("configuration.supported_modes: invalid modes") + supported = [] + elif len(supported) != len(set(supported)) or "off" not in supported: + errors.append("configuration.supported_modes: unique entries including off required") + + requirements = mapping(document.get("requirements"), "requirements") + minimum = requirements.get("minimum_frames") + max_gap = requirements.get("max_frame_gap_ns") + if type(minimum) is not int or minimum < 2: + errors.append("requirements.minimum_frames: integer >= 2 required") + minimum = 2 + if type(max_gap) is not int or max_gap <= 0: + errors.append("requirements.max_frame_gap_ns: positive integer required") + max_gap = None + required_events = requirements.get("required_events") + if not isinstance(required_events, list) or any(not isinstance(e, str) or not e for e in required_events): + errors.append("requirements.required_events: string list required") + required_events = [] + # Transition cases must identify the transition they claim to exercise. + if scenario in SCENARIOS and not scenario.startswith("mode_") and scenario not in required_events: + gaps.append(f"{scenario}: required event absent from scenario contract") + frames = document.get("frames") + if not isinstance(frames, list): + errors.append("frames: expected list") + frames = [] + if len(frames) < minimum: + gaps.append("insufficient frames") + events, modes = set(), Counter() + active = {mode: Counter() for mode in MODES} + stock_active = Counter() + previous_time = None + for index, raw in enumerate(frames): + prefix = f"frames[{index}]" + frame = mapping(raw, prefix) + time_ns = frame.get("time_ns") + if type(time_ns) is not int or time_ns < 0: + errors.append(f"{prefix}.time_ns: nonnegative integer required") + else: + if previous_time is not None: + if time_ns <= previous_time: + errors.append(f"{prefix}.time_ns: timestamps must strictly increase") + elif max_gap is not None and time_ns - previous_time > max_gap: + gaps.append(f"{prefix}: unobserved interval exceeds contract") + previous_time = time_ns + mode = frame.get("mode") + valid_mode = isinstance(mode, str) and mode in MODES + if not valid_mode: + errors.append(f"{prefix}.mode: invalid mode") + allowed_axes = (False, False) + else: + modes[mode] += 1 + allowed_axes = MODES[mode] + event = frame.get("event") + if event is not None: + if not isinstance(event, str) or not event: + errors.append(f"{prefix}.event: string or null required") + else: + events.add(event) + observed = mapping(frame.get("observed"), f"{prefix}.observed") + expected = mapping(frame.get("expected"), f"{prefix}.expected") + signals = {} + for name in AXIS_SIGNALS: + pair = mapping(observed.get(name), f"{prefix}.observed.{name}") + signals[name] = {axis: boolean(pair.get(axis), f"{prefix}.observed.{name}.{axis}") for axis in AXES} + can_observed = boolean(observed.get("control_can_observed"), f"{prefix}.observed.control_can_observed") + feedback_fresh = boolean(observed.get("safety_feedback_fresh"), f"{prefix}.observed.safety_feedback_fresh") + stock = boolean(observed.get("stock_acc_active"), f"{prefix}.observed.stock_acc_active") + if can_observed is False: + gaps.append(f"{prefix}: control CAN not observed; no-command cannot be inferred") + if feedback_fresh is False: + gaps.append(f"{prefix}: safety feedback stale") + for signal in EXPECTED_SIGNALS: + pair = mapping(expected.get(signal), f"{prefix}.expected.{signal}") + for axis in AXES: + desired = boolean(pair.get(axis), f"{prefix}.expected.{signal}.{axis}") + actual = signals[signal][axis] + if desired is not None and actual is not None and desired != actual: + failures.append(f"{prefix}: {signal}.{axis} differs from explicit policy expectation") + desired_stock = boolean(expected.get("stock_acc_active"), f"{prefix}.expected.stock_acc_active") + if stock is not None and desired_stock is not None and stock != desired_stock: + failures.append(f"{prefix}: stock ACC differs from explicit policy expectation") + for axis, selected in zip(AXES, allowed_axes, strict=True): + request, permission, effective, command, fault, fresh = (signals[s][axis] for s in AXIS_SIGNALS[:6]) + if any(signals[s][axis] is True for s in ("request", "effective", "command_active")) and not selected: + failures.append(f"{prefix}: {axis} activity outside selected mode") + if (effective is True or command is True) and mode not in supported: + failures.append(f"{prefix}: activity in unsupported mode") + if axis == "longitudinal" and owner in ("stock", "none") and (effective is True or command is True): + failures.append(f"{prefix}: host longitudinal actuation without system ownership") + if effective is True and any(value is False for value in (request, permission, fresh)): + failures.append(f"{prefix}: effective {axis} without request/permission/fresh input") + if command is True and any(value is False for value in (request, permission, effective, fresh)): + failures.append(f"{prefix}: active {axis} command without effective authorization") + if fault is True and (effective is True or command is True): + failures.append(f"{prefix}: {axis} active during a blocking axis fault") + if valid_mode and command is True and can_observed is True and feedback_fresh is True: + active[mode][axis] += 1 + if valid_mode and stock is True and can_observed is True and feedback_fresh is True: + stock_active[mode] += 1 + gaps.extend(f"required event not observed: {event}" for event in required_events if event not in events) + if isinstance(scenario, str) and scenario.startswith("mode_"): + target = scenario.removeprefix("mode_") + if target in MODES: + if modes[target] < minimum: + gaps.append(f"{target}: insufficient target-mode samples") + if target in supported: + for axis, selected in zip(AXES, MODES[target], strict=True): + if selected and (axis == "lateral" or owner == "system") and active[target][axis] < minimum: + gaps.append(f"{target}: insufficient observed active {axis} commands") + if MODES[target][1] and owner == "stock" and stock_active[target] < minimum: + gaps.append(f"{target}: insufficient observed stock ACC activity") + if MODES[target][1] and owner == "none": + gaps.append(f"{target}: requested longitudinal capability has no owner") + status = "error" if errors else "failed" if failures else "uncovered" if gaps else "pass" + return {"schema_version": 1, "scope": "recorded_control_trace_contract", "vehicle_qualification": "not_established", + "case_id": document.get("case_id"), "scenario": scenario, "evidence_kind": kind, + "configuration_id": configuration_id(config), "status": status, "errors": errors, "failures": failures, + "uncovered": gaps, "frames": len(frames), "mode_frames": dict(modes), + "observed_active_command_frames": {m: dict(v) for m, v in active.items()}} + + +def validate_matrix(documents, configurations): + """Require every named scenario for each exact caller-supplied configuration. + + Synthetic evidence cannot satisfy this coverage gate. Even complete recorded + coverage does not establish physical vehicle or release qualification. + """ + results = [validate_trace(document) for document in documents] + missing = [] + required_ids = [configuration_id(config) for config in configurations] + if not required_ids: + missing.append({"reason": "no required configurations"}) + for config_id in dict.fromkeys(required_ids): + for scenario in SCENARIOS: + rows = [r for r in results if r["configuration_id"] == config_id and r["scenario"] == scenario] + if not any(r["status"] == "pass" and r["evidence_kind"] != "synthetic" for r in rows): + missing.append({"configuration_id": config_id, "scenario": scenario, "reason": "no passing recorded evidence"}) + statuses = {r["status"] for r in results} + status = "error" if "error" in statuses else "failed" if "failed" in statuses else "uncovered" if missing or "uncovered" in statuses else "pass" + return {"schema_version": 1, "scope": "declared_configuration_coverage", "vehicle_qualification": "not_established", + "status": status, "required_configurations": len(set(required_ids)), "required_scenarios": list(SCENARIOS), + "results": results, "uncovered": missing} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("trace", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + try: + report = validate_trace(json.loads(args.trace.read_text())) + except (OSError, ValueError) as error: + report = {"status": "error", "scope": "recorded_control_trace_contract", "vehicle_qualification": "not_established", "errors": [str(error)]} + data = json.dumps(report, indent=2) + "\n" + if args.output: + args.output.write_text(data) + else: + print(data, end="") + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/openpilot/starpilot/validation/required-platforms.json b/openpilot/starpilot/validation/required-platforms.json new file mode 100644 index 0000000000..81d124beed --- /dev/null +++ b/openpilot/starpilot/validation/required-platforms.json @@ -0,0 +1,1744 @@ +{ + "opendbc_base_revision": "a3d3b7c6ca76ed4c97f606e9b2a0e371e8fa52af", + "platforms": [ + { + "family": "honda", + "id": "ACURA_ADX", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "ACURA_ILX", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "ACURA_INTEGRA", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "ACURA_MDX_3G", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "ACURA_MDX_3G_MMR", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "ACURA_MDX_4G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "ACURA_MDX_4G_MMR", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "ACURA_RDX", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "ACURA_RDX_3G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "ACURA_RDX_3G_MMR", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "ACURA_TLX_1G", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "ACURA_TLX_2G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "ACURA_TLX_2G_MMR", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "AUDI_A3_MK3", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "AUDI_Q2_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "AUDI_Q3_MK2", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "AUDI_Q4_MK1", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "AUDI_Q4_MK2", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "BUICK_BABYENCLAVE", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "BUICK_LACROSSE", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "BUICK_LACROSSE_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "BUICK_LACROSSE_ASCM_19US", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "BUICK_REGAL", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CADILLAC_ATS", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CADILLAC_CT6_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CADILLAC_ESCALADE", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CADILLAC_ESCALADE_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CADILLAC_ESCALADE_ESV", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CADILLAC_ESCALADE_ESV_2019", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CADILLAC_ESCALADE_ESV_2019_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CADILLAC_XT4", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CADILLAC_XT4_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CADILLAC_XT5", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CADILLAC_XT5_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CADILLAC_XT6", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_BLAZER", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_BOLT_ACC_2022_2023", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_BOLT_ACC_2022_2023_PEDAL", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_BOLT_CC_2017", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_BOLT_CC_2018_2021", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_BOLT_CC_2022_2023", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_EQUINOX", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CHEVROLET_EQUINOX_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_MALIBU", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CHEVROLET_MALIBU_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_MALIBU_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_MALIBU_HYBRID_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_MALIBU_SDGM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_SILVERADO", + "source_declaration": "declaration_changed" + }, + { + "family": "gm", + "id": "CHEVROLET_SILVERADO_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_SUBURBAN", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_SUBURBAN_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_SUBURBAN_CAMERA", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_SUBURBAN_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_TRAILBLAZER", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CHEVROLET_TRAILBLAZER_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_TRAVERSE", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CHEVROLET_TRAX", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_VOLT", + "source_declaration": "declaration_changed" + }, + { + "family": "gm", + "id": "CHEVROLET_VOLT_2019", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "CHEVROLET_VOLT_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_VOLT_CAMERA", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "CHEVROLET_VOLT_CC", + "source_declaration": "source_only" + }, + { + "family": "chrysler", + "id": "CHRYSLER_PACIFICA_2018", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "CHRYSLER_PACIFICA_2018_HYBRID", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "CHRYSLER_PACIFICA_2019_HYBRID", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "CHRYSLER_PACIFICA_2020", + "source_declaration": "same_declaration" + }, + { + "family": "body", + "id": "COMMA_BODY", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "CUPRA_BORN_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "chrysler", + "id": "DODGE_DURANGO", + "source_declaration": "same_declaration" + }, + { + "family": "ford", + "id": "FORD_BRONCO_SPORT_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_EDGE_MK2", + "source_declaration": "source_only" + }, + { + "family": "ford", + "id": "FORD_ESCAPE_MK4", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_ESCAPE_MK4_5", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_EXPEDITION_MK4", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_EXPLORER_MK6", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_FOCUS_MK4", + "source_declaration": "same_declaration" + }, + { + "family": "ford", + "id": "FORD_F_150_LIGHTNING_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_F_150_MK14", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_MAVERICK_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_MONDEO_MK5", + "source_declaration": "source_only" + }, + { + "family": "ford", + "id": "FORD_MUSTANG_MACH_E_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_RANGER_MK2", + "source_declaration": "declaration_changed" + }, + { + "family": "ford", + "id": "FORD_TRANSIT_MK5", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "GENESIS_G70", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "GENESIS_G70_2020", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "GENESIS_G70_2021_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "GENESIS_G80", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "GENESIS_G80_2ND_GEN_FL", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "GENESIS_G90", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "GENESIS_GV60_EV_1ST_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "GENESIS_GV70_1ST_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "GENESIS_GV70_2026", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "GENESIS_GV70_ELECTRIFIED_1ST_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "GENESIS_GV70_ELECTRIFIED_2ND_GEN", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "GENESIS_GV80", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "GENESIS_GV80_2025", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "GMC_ACADIA", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "GMC_ACADIA_ASCM", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "GMC_YUKON", + "source_declaration": "same_declaration" + }, + { + "family": "gm", + "id": "GMC_YUKON_CC", + "source_declaration": "source_only" + }, + { + "family": "gm", + "id": "HOLDEN_ASTRA", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_ACCORD", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_ACCORD_11G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_ACCORD_9G", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "HONDA_CITY_7G", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_CIVIC", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_CIVIC_2022", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_CIVIC_BOSCH", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_CIVIC_BOSCH_DIESEL", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_CLARITY", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "HONDA_CRV", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_CRV_5G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_CRV_6G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_CRV_EU", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_CRV_HYBRID", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_CRV_SA", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "HONDA_E", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_E_ADVANCE", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_FIT", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_FIT_4G", + "source_declaration": "source_only" + }, + { + "family": "honda", + "id": "HONDA_FREED", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_HRV", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_HRV_3G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_INSIGHT", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_NBOX_2G", + "source_declaration": "declaration_changed" + }, + { + "family": "honda", + "id": "HONDA_ODYSSEY", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_ODYSSEY_5G_MMR", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_ODYSSEY_TWN", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_PASSPORT_4G", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_PILOT", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_PILOT_4G", + "source_declaration": "same_declaration" + }, + { + "family": "honda", + "id": "HONDA_RIDGELINE", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_AZERA_6TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_AZERA_HEV_6TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_AZERA_HEV_7TH_GEN", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_BAYON_1ST_GEN_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_CUSTIN_1ST_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_2021", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_2022_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_2024", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_GT_I30", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_HEV_2021", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_HEV_2022_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_ELANTRA_HEV_2024", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_GENESIS", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_5", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_5_N", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_5_PE", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_6", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_9", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_EV_2020", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_EV_LTD", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_HEV_2022", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_PHEV", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_IONIQ_PHEV_2019", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_2022", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_2ND_GEN", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_EV", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_EV_2022", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_EV_2ND_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_EV_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_HEV", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_HEV_2ND_GEN", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_KONA_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_NEXO_1ST_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_PALISADE", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_PALISADE_2023", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_CRUZ_1ST_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_CRUZ_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_FE", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_FE_2022", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_FE_HEV_2022", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_FE_HEV_5TH_GEN", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SANTA_FE_PHEV_2022", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SONATA", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SONATA_2024", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SONATA_HEV_2024", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SONATA_HYBRID", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_SONATA_LF", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_STARIA_4TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_TUCSON", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_TUCSON_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_TUCSON_4TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "HYUNDAI_TUCSON_HEV_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_TUCSON_PHEV_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "HYUNDAI_VELOSTER", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "JEEP_GRAND_CHEROKEE", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "JEEP_GRAND_CHEROKEE_2019", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_CARNIVAL_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_CARNIVAL_4TH_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_CARNIVAL_HEV_4TH_GEN", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_CEED", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_CEED_PHEV_2022_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_EV6", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_EV6_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_EV9", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_FORTE", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_FORTE_2019_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_FORTE_2021_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_K4_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_K5_2021", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_K5_2025", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_K5_HEV_2020", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_K8_HEV_1ST_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_NIRO_EV", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_NIRO_EV_2ND_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_NIRO_HEV_2021", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_NIRO_HEV_2ND_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_NIRO_PHEV", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_NIRO_PHEV_2022", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_OPTIMA_G4", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_OPTIMA_G4_FL", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_OPTIMA_H", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_OPTIMA_H_G4_FL", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_RAY_EV", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_SELTOS", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_SELTOS_2023_NON_SCC", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_SORENTO", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_SORENTO_2024", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_SORENTO_4TH_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_SORENTO_HEV_4TH_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "hyundai", + "id": "KIA_SORENTO_HEV_4TH_GEN_LFA2", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_SPORTAGE_2026", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_SPORTAGE_5TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_SPORTAGE_HEV_2026", + "source_declaration": "source_only" + }, + { + "family": "hyundai", + "id": "KIA_STINGER", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_STINGER_2022", + "source_declaration": "same_declaration" + }, + { + "family": "hyundai", + "id": "KIA_XCEED_PHEV", + "source_declaration": "source_only" + }, + { + "family": "toyota", + "id": "LEXUS_CTH", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_ES", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_ES_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_GS_F", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_IS", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_IS_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_LC_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_NX", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_NX_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_RC", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_RC_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_RX", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "LEXUS_RX_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "mazda", + "id": "MAZDA_3", + "source_declaration": "same_declaration" + }, + { + "family": "mazda", + "id": "MAZDA_6", + "source_declaration": "same_declaration" + }, + { + "family": "mazda", + "id": "MAZDA_CX5", + "source_declaration": "same_declaration" + }, + { + "family": "mazda", + "id": "MAZDA_CX5_2022", + "source_declaration": "same_declaration" + }, + { + "family": "mazda", + "id": "MAZDA_CX9", + "source_declaration": "same_declaration" + }, + { + "family": "mazda", + "id": "MAZDA_CX9_2021", + "source_declaration": "same_declaration" + }, + { + "family": "mock", + "id": "MOCK", + "source_declaration": "same_declaration" + }, + { + "family": "nissan", + "id": "NISSAN_ALTIMA", + "source_declaration": "declaration_changed" + }, + { + "family": "nissan", + "id": "NISSAN_LEAF", + "source_declaration": "declaration_changed" + }, + { + "family": "nissan", + "id": "NISSAN_LEAF_IC", + "source_declaration": "declaration_changed" + }, + { + "family": "nissan", + "id": "NISSAN_ROGUE", + "source_declaration": "same_declaration" + }, + { + "family": "nissan", + "id": "NISSAN_XTRAIL", + "source_declaration": "same_declaration" + }, + { + "family": "volvo", + "id": "POLESTAR_2", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "PORSCHE_MACAN_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "psa", + "id": "PSA_PEUGEOT_208", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "RAM_1500_5TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "chrysler", + "id": "RAM_HD_5TH_GEN", + "source_declaration": "same_declaration" + }, + { + "family": "rivian", + "id": "RIVIAN_R1_GEN1", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "SEAT_ATECA_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "SKODA_ENYAQ_MK1", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "SKODA_ENYAQ_MK2", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "SKODA_FABIA_MK4", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "SKODA_KAMIQ_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "SKODA_KAROQ_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "SKODA_KODIAQ_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "SKODA_OCTAVIA_MK3", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "SKODA_SUPERB_MK3", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_ASCENT", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_ASCENT_2023", + "source_declaration": "declaration_changed" + }, + { + "family": "subaru", + "id": "SUBARU_CROSSTREK_2025", + "source_declaration": "source_only" + }, + { + "family": "subaru", + "id": "SUBARU_CROSSTREK_HYBRID", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_FORESTER", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_FORESTER_2022", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_FORESTER_HYBRID", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_FORESTER_PREGLOBAL", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_IMPREZA", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_IMPREZA_2020", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_LEGACY", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_LEGACY_2025", + "source_declaration": "source_only" + }, + { + "family": "subaru", + "id": "SUBARU_LEGACY_PREGLOBAL", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_OUTBACK", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_OUTBACK_2023", + "source_declaration": "declaration_changed" + }, + { + "family": "subaru", + "id": "SUBARU_OUTBACK_PREGLOBAL", + "source_declaration": "same_declaration" + }, + { + "family": "subaru", + "id": "SUBARU_OUTBACK_PREGLOBAL_2018", + "source_declaration": "same_declaration" + }, + { + "family": "tesla", + "id": "TESLA_MODEL_3", + "source_declaration": "declaration_changed" + }, + { + "family": "tesla", + "id": "TESLA_MODEL_S_HW1", + "source_declaration": "source_only" + }, + { + "family": "tesla", + "id": "TESLA_MODEL_S_PREAP", + "source_declaration": "source_only" + }, + { + "family": "tesla", + "id": "TESLA_MODEL_X", + "source_declaration": "same_declaration" + }, + { + "family": "tesla", + "id": "TESLA_MODEL_Y", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_ALPHARD_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_AVALON", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_AVALON_2019", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_AVALON_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_CAMRY", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_CAMRY_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_CHR", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_CHR_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_COROLLA", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_COROLLA_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_HIGHLANDER", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_HIGHLANDER_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_MATRIX_RETROFIT", + "source_declaration": "source_only" + }, + { + "family": "toyota", + "id": "TOYOTA_MIRAI", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_PRIUS", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_PRIUS_RETROFIT", + "source_declaration": "source_only" + }, + { + "family": "toyota", + "id": "TOYOTA_PRIUS_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_PRIUS_V", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_RAV4", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_RAV4H", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_RAV4_PRIME", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_RAV4_TSS2", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_RAV4_TSS2_2022", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_RAV4_TSS2_2023", + "source_declaration": "same_declaration" + }, + { + "family": "toyota", + "id": "TOYOTA_SIENNA", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_SIENNA_4TH_GEN", + "source_declaration": "declaration_changed" + }, + { + "family": "toyota", + "id": "TOYOTA_YARIS", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_ARTEON_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_ATLAS_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_CADDY_MK3", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_CRAFTER_MK2", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_GOLF_MK7", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_ID3_MK1", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_ID3_MK2", + "source_declaration": "source_only" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_ID4_MK1", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_ID4_MK2", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_JETTA_MK6", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_JETTA_MK7", + "source_declaration": "declaration_changed" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_PASSAT_MK8", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_PASSAT_NMS", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_POLO_MK6", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_SHARAN_MK2", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_TAOS_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_TCROSS_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_TIGUAN_MK2", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_TOURAN_MK2", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_TRANSPORTER_T61", + "source_declaration": "same_declaration" + }, + { + "family": "volkswagen", + "id": "VOLKSWAGEN_TROC_MK1", + "source_declaration": "same_declaration" + }, + { + "family": "volvo", + "id": "VOLVO_S60_RECHARGE", + "source_declaration": "source_only" + }, + { + "family": "volvo", + "id": "VOLVO_V40", + "source_declaration": "source_only" + }, + { + "family": "volvo", + "id": "VOLVO_XC40_RECHARGE", + "source_declaration": "source_only" + } + ], + "schema_version": 1, + "scope": "exact_source_platform_migration_obligations", + "source_inventory_sha256": "5cdb0fe01dd25bf87f0aeda7e5f71a10164d4d9bddf1c8b54e1ba6dec5140146", + "source_revision": "678af78347d9656bc2f5dacd4204b012db861fa7", + "upstream_additions": [ + "AUDI_Q5_MK1", + "CHEVROLET_BOLT_EUV", + "JEEP_CHEROKEE_5TH_GEN", + "KIA_K7_2017", + "LEXUS_LS", + "MG_5_EV", + "RIVIAN_R1" + ], + "upstream_base_revision": "521db4c825d37eb5f29acf955daa88da003e4433" +} diff --git a/openpilot/starpilot/validation/tests/test_control_modes.py b/openpilot/starpilot/validation/tests/test_control_modes.py new file mode 100644 index 0000000000..7bf171d125 --- /dev/null +++ b/openpilot/starpilot/validation/tests/test_control_modes.py @@ -0,0 +1,204 @@ +import copy +import unittest + +from openpilot.starpilot.validation.control_modes import ( + AXES, CONFIG_HASHES, MODES, SCENARIOS, configuration_id, validate_matrix, validate_trace, +) + + +def fixture(mode="combined", owner="system", kind="synthetic"): + """A validator fixture, not a recording from a supported vehicle.""" + configuration = { + "platform": "VALIDATOR_FIXTURE", "device": "host", "harness": "fixture", "policy_id": "fixture-policy-v1", + "source_commit": "a" * 40, "longitudinal_owner": owner, "supported_modes": list(MODES), + **dict.fromkeys(CONFIG_HASHES, "b" * 64), + } + selected = dict(zip(AXES, MODES[mode], strict=True)) + active = selected | {"longitudinal": selected["longitudinal"] and owner == "system"} + stock = selected["longitudinal"] and owner == "stock" + frame = { + "time_ns": 0, "mode": mode, "event": None, + "observed": { + "request": active.copy(), "permission": dict.fromkeys(AXES, True), "effective": active.copy(), + "command_active": active.copy(), "blocking_fault": dict.fromkeys(AXES, False), + "input_fresh": dict.fromkeys(AXES, True), "displayed_active": active.copy(), + "control_can_observed": True, "safety_feedback_fresh": True, "stock_acc_active": stock, + }, + "expected": {"effective": active.copy(), "command_active": active.copy(), "displayed_active": active.copy(), "stock_acc_active": stock}, + } + frames = [frame, copy.deepcopy(frame)] + frames[1]["time_ns"] = 10_000_000 + return { + "schema_version": 1, "case_id": "fixture-" + mode, "scenario": "mode_" + mode, "evidence_kind": kind, + "configuration": configuration, "recording_sha256": "c" * 64, + "requirements": {"minimum_frames": 2, "max_frame_gap_ns": 20_000_000, "required_events": []}, "frames": frames, + } + + +class TestControlModeContract(unittest.TestCase): + def test_four_modes_are_distinct(self): + for mode in MODES: + with self.subTest(mode=mode): + report = validate_trace(fixture(mode)) + self.assertEqual(report["status"], "pass", report) + self.assertEqual(report["vehicle_qualification"], "not_established") + + def test_lateral_only_rejects_longitudinal_command(self): + trace = fixture("lateral_only") + trace["frames"][0]["observed"]["command_active"]["longitudinal"] = True + self.assertEqual(validate_trace(trace)["status"], "failed") + + def test_longitudinal_only_rejects_lateral_command(self): + trace = fixture("longitudinal_only") + trace["frames"][0]["observed"]["command_active"]["lateral"] = True + self.assertEqual(validate_trace(trace)["status"], "failed") + + def test_off_rejects_either_axis_even_if_expected(self): + for axis in AXES: + trace = fixture("off") + trace["frames"][0]["observed"]["command_active"][axis] = True + trace["frames"][0]["expected"]["command_active"][axis] = True + self.assertEqual(validate_trace(trace)["status"], "failed") + + def test_stock_acc_is_separate_from_host_longitudinal(self): + trace = fixture(owner="stock") + self.assertEqual(validate_trace(trace)["status"], "pass") + trace["frames"][0]["observed"]["command_active"]["longitudinal"] = True + self.assertEqual(validate_trace(trace)["status"], "failed") + + def test_unknown_axis_is_uncovered_not_false(self): + for signal in ("permission", "command_active", "input_fresh"): + for missing in (False, True): + trace = fixture("off") + if missing: + del trace["frames"][0]["observed"][signal]["lateral"] + else: + trace["frames"][0]["observed"][signal]["lateral"] = None + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_missing_or_stale_feedback_is_uncovered(self): + for value in (None, False): + trace = fixture() + trace["frames"][0]["observed"]["safety_feedback_fresh"] = value + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_absent_control_can_cannot_prove_off(self): + for value in (None, False): + trace = fixture("off") + trace["frames"][0]["observed"]["control_can_observed"] = value + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_active_command_needs_axis_authorization(self): + for signal in ("permission", "request", "effective", "input_fresh"): + trace = fixture() + trace["frames"][0]["observed"][signal]["lateral"] = False + self.assertEqual(validate_trace(trace)["status"], "failed") + + def test_blocking_fault_is_axis_specific(self): + trace = fixture("longitudinal_only") + for frame in trace["frames"]: + frame["observed"]["blocking_fault"]["lateral"] = True + self.assertEqual(validate_trace(trace)["status"], "pass") + trace["frames"][0]["observed"]["blocking_fault"]["longitudinal"] = True + self.assertEqual(validate_trace(trace)["status"], "failed") + + def test_recovery_requires_explicit_policy_expectation(self): + trace = fixture("off") + trace["scenario"] = "policy_recovery" + trace["requirements"]["required_events"] = ["policy_recovery"] + trace["frames"][1]["event"] = "policy_recovery" + self.assertEqual(validate_trace(trace)["status"], "pass") + del trace["frames"][1]["expected"]["effective"] + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_selection_display_need_not_equal_authorization(self): + trace = fixture("lateral_only") + trace["scenario"] = "cancel" + trace["requirements"]["required_events"] = ["cancel"] + for frame in trace["frames"]: + frame["event"] = "cancel" + for signal in ("effective", "command_active"): + frame["observed"][signal]["lateral"] = False + frame["expected"][signal]["lateral"] = False + self.assertEqual(validate_trace(trace)["status"], "pass") + + def test_active_mode_needs_observed_activity(self): + trace = fixture() + for frame in trace["frames"]: + for signal in ("effective", "command_active"): + frame["observed"][signal] = dict.fromkeys(AXES, False) + frame["expected"][signal] = dict.fromkeys(AXES, False) + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_unsupported_mode_requires_denial(self): + trace = fixture("longitudinal_only") + trace["configuration"]["supported_modes"] = ["off", "lateral_only"] + self.assertEqual(validate_trace(trace)["status"], "failed") + for frame in trace["frames"]: + for signal in ("effective", "command_active", "displayed_active"): + frame["observed"][signal] = dict.fromkeys(AXES, False) + frame["expected"][signal] = dict.fromkeys(AXES, False) + self.assertEqual(validate_trace(trace)["status"], "pass") + + def test_time_gap_is_uncovered_and_reversal_is_error(self): + trace = fixture() + trace["frames"][1]["time_ns"] = 100_000_000 + self.assertEqual(validate_trace(trace)["status"], "uncovered") + trace["frames"][1]["time_ns"] = 0 + self.assertEqual(validate_trace(trace)["status"], "error") + + def test_malformed_mode_and_booleans_are_errors(self): + trace = fixture() + trace["frames"][0]["mode"] = [] + self.assertEqual(validate_trace(trace)["status"], "error") + trace = fixture() + trace["frames"][0]["observed"]["command_active"]["lateral"] = 0 + self.assertEqual(validate_trace(trace)["status"], "error") + + def test_required_event_cannot_be_omitted(self): + trace = fixture() + trace["scenario"] = "restart" + trace["requirements"]["required_events"] = ["restart"] + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_missing_configuration_provenance_is_uncovered(self): + trace = fixture() + del trace["configuration"]["policy_sha256"] + self.assertEqual(validate_trace(trace)["status"], "uncovered") + + def test_configuration_identity_includes_settings_and_firmware(self): + trace = fixture() + for key in ("settings_sha256", "firmware_manifest_sha256"): + changed = copy.deepcopy(trace["configuration"]) + changed[key] = "d" * 64 + self.assertNotEqual(configuration_id(changed), configuration_id(trace["configuration"])) + + def test_synthetic_cases_never_satisfy_recorded_coverage(self): + documents = [fixture(mode) for mode in MODES] + result = validate_matrix(documents, [documents[0]["configuration"]]) + self.assertEqual(result["status"], "uncovered") + self.assertEqual(len(result["uncovered"]), len(SCENARIOS)) + self.assertTrue(all(r["status"] == "pass" for r in result["results"])) + + def test_matrix_requires_all_scenarios_for_exact_configuration(self): + trace = fixture(kind="replay") + result = validate_matrix([trace], [trace["configuration"]]) + self.assertEqual(result["status"], "uncovered") + self.assertEqual(len(result["uncovered"]), len(SCENARIOS) - 1) + other = copy.deepcopy(trace["configuration"]) + other["harness"] = "another-fixture" + result = validate_matrix([trace], [other]) + self.assertEqual(len(result["uncovered"]), len(SCENARIOS)) + + def test_failure_is_not_hidden_by_another_passing_case(self): + failed = fixture("off", kind="replay") + failed["frames"][0]["observed"]["command_active"]["lateral"] = True + passed = fixture("off", kind="replay") + self.assertEqual(validate_matrix([failed, passed], [passed["configuration"]])["status"], "failed") + + def test_empty_matrix_does_not_pass(self): + self.assertEqual(validate_matrix([], [])["status"], "uncovered") + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/system/tests/test_timed_clocks.py b/openpilot/system/tests/test_timed_clocks.py new file mode 100644 index 0000000000..34eb489b49 --- /dev/null +++ b/openpilot/system/tests/test_timed_clocks.py @@ -0,0 +1,22 @@ +from openpilot.system import timed + + +def test_recorded_clock_pair_is_bound_to_event_timestamp(monkeypatch): + mono = iter((10_000_000, 10_100_000)) + monkeypatch.setattr(timed.time, 'monotonic_ns', lambda: next(mono)) + monkeypatch.setattr(timed.time, 'clock_gettime_ns', lambda clock: 20_000_000) + monkeypatch.setattr(timed.time, 'time_ns', lambda: 1_800_000_000_000_000_000) + monkeypatch.setattr(timed, 'system_time_valid', lambda: True) + event = timed.clock_message() + assert event.logMonoTime == event.clocks.deprecated.monotonicNanos == 10_000_000 + assert event.clocks.deprecated.bootTimeNanos == 20_000_000 + assert event.clocks.wallTimeNanos == 1_800_000_000_000_000_000 + + +def test_delayed_clock_sample_does_not_claim_a_precise_pair(monkeypatch): + mono = iter((10_000_000, 12_000_000)) + monkeypatch.setattr(timed.time, 'monotonic_ns', lambda: next(mono)) + monkeypatch.setattr(timed.time, 'clock_gettime_ns', lambda clock: 20_000_000) + monkeypatch.setattr(timed, 'system_time_valid', lambda: True) + event = timed.clock_message() + assert event.clocks.deprecated.monotonicNanos == event.clocks.deprecated.bootTimeNanos == 0 diff --git a/openpilot/tools/lib/tests/test_github_comments.py b/openpilot/tools/lib/tests/test_github_comments.py new file mode 100644 index 0000000000..f8eb930cb3 --- /dev/null +++ b/openpilot/tools/lib/tests/test_github_comments.py @@ -0,0 +1,40 @@ +from http import HTTPMethod +from unittest.mock import Mock + +import pytest + +from openpilot.tools.lib.github_utils import GithubUtils + + +@pytest.mark.parametrize("comments,marker,expected_path", [ + ([{"id": 1, "user": {"login": "bot"}, "body": "_model_tici_"}, + {"id": 2, "user": {"login": "bot"}, "body": "_model_chestnut_"}], "_model_chestnut_", "issues/comments/2"), + ([{"id": 1, "user": {"login": "bot"}, "body": "_model_chestnut_"}, + {"id": 2, "user": {"login": "bot"}, "body": "_model_tici_"}], "_model_tici_", "issues/comments/2"), + ([{"id": 1, "user": {"login": "other"}, "body": "_model_tici_"}], "_model_tici_", "issues/42/comments"), + ([{"id": 1, "user": {"login": "bot"}, "body": "old report"}], "", "issues/comments/1"), + ([], "_model_tici_", "issues/42/comments"), +]) +def test_comment_overwrite_is_scoped_to_author_and_marker(comments, marker, expected_path): + github = GithubUtils("", "") + github.get_pr_number = Mock(return_value=42) + github.api_call = Mock(return_value=Mock(json=Mock(return_value=comments))) + + github.comment_on_pr("new report", "branch", "bot", True, comment_marker=marker) + + calls = github.api_call.call_args_list + assert calls[0].args == ("issues/42/comments",) + assert calls[1].args == (expected_path,) + assert calls[1].kwargs["method"] == (HTTPMethod.PATCH if expected_path.startswith("issues/comments/") else HTTPMethod.POST) + assert calls[1].kwargs["data"] == '{"body": "new report"}' + assert len(calls) == 2 + + +def test_new_comment_does_not_read_or_replace_existing_comments(): + github = GithubUtils("", "") + github.get_pr_number = Mock(return_value=42) + github.api_call = Mock() + + github.comment_on_pr("new report", "branch", "bot", False, comment_marker="_model_tici_") + + github.api_call.assert_called_once_with("issues/42/comments", data='{"body": "new report"}', method=HTTPMethod.POST) diff --git a/openpilot/tools/plotjuggler/test_plotjuggler.py b/openpilot/tools/plotjuggler/test_plotjuggler.py index 10cd342dc8..4ba6b58943 100644 --- a/openpilot/tools/plotjuggler/test_plotjuggler.py +++ b/openpilot/tools/plotjuggler/test_plotjuggler.py @@ -2,6 +2,7 @@ import os import glob import shutil import signal +import selectors import subprocess import time @@ -9,7 +10,7 @@ import unittest from openpilot.common.test import OpenpilotTestCase from openpilot.common.basedir import BASEDIR -from openpilot.common.timeout import Timeout +from openpilot.common.timeout import Timeout, TimeoutException from openpilot.tools.plotjuggler.juggle import DEMO_ROUTE, install PJ_DIR = os.path.join(BASEDIR, "openpilot/tools/plotjuggler") @@ -22,19 +23,46 @@ class TestPlotJuggler(OpenpilotTestCase): pj = os.path.join(PJ_DIR, "juggle.py") with subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {pj} "{DEMO_ROUTE}/:2"', - stderr=subprocess.PIPE, shell=True, start_new_session=True) as p: - # Wait for "Done reading Rlog data" signal from the plugin - output = "\n" - with Timeout(180, error_msg=output): - while output.splitlines()[-1] != "Done reading Rlog data": - output += p.stderr.readline().decode("utf-8") - - # ensure plotjuggler didn't crash after exiting the plugin - time.sleep(2) - assert p.poll() is None - os.killpg(os.getpgid(p.pid), signal.SIGTERM) - - assert "Raw file read failed" not in output + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, start_new_session=True) as p: + output = "" + try: + assert p.stdout is not None + with selectors.DefaultSelector() as selector: + selector.register(p.stdout, selectors.EVENT_READ) + try: + with Timeout(180): + while "Done reading Rlog data" not in output.splitlines(): + for key, _ in selector.select(0.1): + chunk = os.read(key.fileobj.fileno(), 65536) + if chunk: + output += chunk.decode("utf-8", errors="replace") + else: + selector.unregister(key.fileobj) + if p.poll() is not None: + self.fail(f"PlotJuggler exited before startup (status {p.returncode}):\n{output}") + except TimeoutException: + self.fail(f"PlotJuggler did not report startup within 180 seconds:\n{output}") + time.sleep(2) + for key, _ in selector.select(0): + chunk = os.read(key.fileobj.fileno(), 65536) + output += chunk.decode("utf-8", errors="replace") + self.assertIsNone(p.poll(), f"PlotJuggler exited after startup (status {p.returncode}):\n{output}") + self.assertNotIn("Raw file read failed", output) + finally: + try: + os.killpg(p.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + p.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + finally: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + p.wait(timeout=5) # TODO: also test that layouts successfully load def test_layouts(self, subtests): diff --git a/tools/ci/FLEET_COVERAGE.md b/tools/ci/FLEET_COVERAGE.md new file mode 100644 index 0000000000..48e22c9c05 --- /dev/null +++ b/tools/ci/FLEET_COVERAGE.md @@ -0,0 +1,81 @@ +# Exact platform coverage gate + +`openpilot/starpilot/validation/required-platforms.json` freezes 345 exact source +platform IDs and seven distinct upstream additions. The source-inventory hash and +revisions make the obligation set reviewable. A shared name or declaration does not +mean that controller, firmware, settings, safety, or physical behavior is equivalent. + +Run the reporter with an interface suite's two companion files: + +```sh +python tools/ci/fleet_coverage.py \ + --interface-coverage ci-results/interfaces/coverage.json \ + --interface-results ci-results/interfaces/results.json \ + --output ci-results/fleet-coverage.json +``` + +The default `--require complete` exits zero only when both interface and independent +mode coverage pass. CI and predeploy use `--require interfaces` to require every +exact interface obligation and stable current execution provenance. That choice +does not clear mode gaps: the full report remains `uncovered` when mode evidence +is absent, and `vehicle_qualification` remains `not_established`. The separate +`interface_gate` records only interface errors, failures and gaps. It compares the +exact registry IDs and one concrete interface test per ID, checks every execution +record, verifies the companion results hash, and compares the reported checkout, +suite runner, test runner, dependency manifest, pinned opendbc commit, CAN sources, +and native parser binary with the current checkout. An older report remains visible +under `reported_interface_counts`, while `current_interface_counts` marks its +passing tests as `historical_pass_only`. A missing port or upstream addition remains +uncovered. A skip, missing test, failed test, duplicate, or malformed report never +becomes a current pass. Upstream additions remain independent obligations even if +they disappear from a later registry. +The command refuses to overwrite an existing output file, including a failed or +uncovered report. Use a fresh path for each run. A matching HEAD alone is not +enough for current execution: the recorded before/after source snapshots must +match the current snapshot, including relevant dirty source and native bytes. +Unrelated UI source changes do not invalidate vehicle execution. + +Recorded control evidence may be supplied through `--mode-evidence path.json`: + +```json +{ + "schema_version": 1, + "enumeration_complete": true, + "provenance": { + "configuration_inventory_sha256": "<64 lowercase hex characters>", + "trace_catalog_sha256": "<64 lowercase hex characters>" + }, + "configurations": [""], + "traces": [""] +} +``` + +The two SHA-256 values must be computed over their respective JSON arrays using +UTF-8, sorted object keys, compact separators `,` and `:`, no ASCII escaping, and +no trailing newline. For Python, this is +`sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8"))`. +The gate recomputes both digests; a digest-shaped label alone is insufficient. + +Each configuration is the exact object accepted by +`openpilot.starpilot.validation.control_modes`: platform, device, harness, policy, +firmware, settings, schema, safety binary and adapter identities belong in it. The +caller must enumerate every relevant configuration, including hardware and firmware +variants. Platform names alone are not a configuration inventory. All 17 named +scenarios must be covered for every configuration. Synthetic traces cannot satisfy +the matrix; earlier failures remain failures when later traces pass. The `off`, +`lateral_only`, `longitudinal_only`, and `combined` modes retain separate host-axis +and stock ACC ownership checks. A missing mode file, provenance, configuration, +recorded scenario or observation is uncovered. + +This gate checks declared data and reported provenance; hashes and recording labels +supplied by a caller do not authenticate a trace adapter or physical origin. Every +output says `vehicle_qualification: not_established`, including a passing coverage +contract. Recorded replay, adapter review, hardware, bus, firmware and release +qualification remain separate work. + +`--require interfaces` specifically gates the exact current production registry, +all concrete mapped interface tests, execution failures and current source/native +provenance. Inapplicable ancillary skips remain reported but do not count as skipped +concrete interfaces. Missing Dom ports, changed addition inventory and independent +mode gaps still keep full readiness uncovered. A green interface regression gate +does not mean complete Dom parity, complete fleet readiness or vehicle qualification. diff --git a/tools/ci/README.md b/tools/ci/README.md new file mode 100644 index 0000000000..aecd9bf1e1 --- /dev/null +++ b/tools/ci/README.md @@ -0,0 +1,337 @@ +# Local and release checks + +Run `./test` from the checkout for the focused pre-push checks. It uses the +isolated developer runtime, retains failure logs, and leaves device artifacts +unchanged. `./test --full` wraps the complete Linux pre-deployment command below. +Use `./test --help` for scope and reporting options. + +On Linux, use the managed host runtime to run the same source, unit, vehicle and pinned +recording checks and process replay as GitHub Actions: + +```sh +./dev python tools/ci/run_predeploy.py --output /tmp/starpilot-predeploy --download +``` + +The default run first builds the host native graph, then also runs the native +Panda protocol executables and required Go/Python IPC checks. +Use a new output directory per run. `--download` permits only the pinned public +recordings; omit it when they are already cached. Node.js 24 must be on PATH (or +set `STARPILOT_NODE`), and the pinned Go toolchain/dependencies are needed for map +provider checks. A failure stops the run with a named log and preserves its exit +status in `results.json`. `--stage build`, `source`, `host`, `native`, `vehicles`, `recorded`, or `replay` can +select a stage while diagnosing a failure; a selected-stage pass is not a full +release result. + +The replay stage runs the full process suite with its existing coverage and +reference comparisons. It can download replay logs independently of the pinned +recording `--download` flag. Unaccounted missing routes and unqualified historical schema caches fail. +Volvo C1/V40 is explicitly reported as lacking a recording, with its source and +native suites required separately; see the process replay README for details. Passing the pinned recorded vehicle checks does not qualify this +separate suite. + +The full runner requires Linux: its USB/native transport and executable map +IPC tests use Linux facilities. Run `tools/setup_dependencies.sh` first, provide +the Node and Go versions above, and fetch the pinned map dependencies with +`(cd mapd_repo && go mod download)`. + +On macOS, build host dependencies and run the source/host subset: + +```sh +./dev python -m SCons -j4 +./dev python tools/ci/run_predeploy.py --stage source --stage host --output /tmp/starpilot-host-checks +``` + +This subset does not qualify Linux native transport, vehicle/recorded suites, +or device artifacts. The full runner rejects an unsupported native-stage host +before executing checks. + +Before shipping device artifacts, run `./build 4` in the source checkout after +host validation. Host tests do not replace the device build, native firmware +qualification, device startup checks, or physical vehicle testing. GitHub also +runs platform-specific build, replay and rendering jobs. + +## Vehicle contract checks + +Run the rebuilt feature and CI tests locally after building native dependencies: + +```sh +python tools/ci/run_host_tests.py openpilot/starpilot tools/ci/tests --json-output feature-results.json +``` + +`tools/op.sh test` uses this wrapper too. It forwards test selection and reporting +options to `tools/test_runner.py`. +It creates disposable Params and messaging directories before any test imports, +then removes them after the test process exits, including on failure. Existing +developer settings and IPC namespaces are not used. It also sets `SCALE=1` so UI +imports do not query a desktop monitor. This is host logic testing; native visual +captures and device rendering need their own graphics context. The hosted unit +job uses the same wrapper for the complete `openpilot` test tree. Galaxy tests +require Node.js 24 on PATH or selected with `STARPILOT_NODE`. + +The source-integrity job also enforces cereal's custom-fork guidance with +`tools/ci/schema_policy.py`. The pinned `car.capnp` and other upstream schemas +must remain unchanged apart from the exact additive exceptions enumerated in +`schema-policy.json`; those exceptions are checked against the pinned source. Fork fields belong inside the reserved `custom.capnp` +structs; their type IDs stay fixed and `log.capnp` may rename only the associated +reserved event aliases, keeping their ordinals and type associations. The check +allows nested custom fields while rejecting edits to upstream-owned definitions, +reused IDs and retargeted events. `schema-policy.json` records the official source +pins and schema hashes; review it alongside `upstream-sync.json` when updating +upstream. Every `.capnp` file under `openpilot/cereal` and +`opendbc_repo/opendbc/car` requires explicit policy coverage; new schemas fail +until reviewed. This guard prevents upstream schema collisions; it does not replace +append-only evolution and compatibility testing of already shipped custom fields. +Slot5 remains empty under its original name. Slot1 restores the full historical +`StarPilotCarParams` fields0–8, nested `SafetyConfig`, and `pcmCruiseSpeed=true` +default at type ID `0xaedffd8f31e7b55d` and Event ordinal108. The optional +`autoSteeringLearning` Event alias still refers to that outer type; its versioned +`learning` child is appended at9 with a separately generated nested type ID. +Historical messages have a default version0 child and cannot advertise a fit. +The occupied-prefix guard protects both the historical and appended fields, +identity and enum meanings. The transient direct learner layout in commit +`c204f0b165` requires that exact schema for replay; this repair does not claim +compatibility with those incompatible development records. Keeping a type ID +alone does not preserve field meanings in the other restored historical slots. +Slot0 preserves the historical car-control HUD and steering-limit prefix and +appends versioned nested navigation fields. Navigation readers require version2; +old car-control records do not become navigation commands. +Slot4 restores `StarPilotModelDataV2` at Event111, including its original +turn-direction field and enum, then appends a typed Vision observation. Slot9 +retains all six historical model-status fields and appends SLC fields at6–30. +Native historical-reader fixtures protect both layouts and prove that an old +model-status record cannot advertise an active speed limit. This corrects the +unreleased Domathon slot9 reuse; earlier development recordings need their exact +source schema for replay. That one pre-release layout correction is explicit in +the compiled baseline; it does not permit future incompatible regeneration. +Slots 17/18/19 previously carried `MapdExtendedOut`/`MapdIn`/`MapdOut` at +Event ordinals 143/144/145; none is available for an unrelated new envelope. +Slot19 now restores `MapdOut` with all24 historical fields and three additive +fields from the pinned provider. Seven further append-only fields carry sample +status, source, GPS/computation times, generation and process session. V2 converts +the Python GPS MONOTONIC observation to BOOTTIME only after a startup/resume +fence; computation and output use BOOTTIME too. Older versions are unqualified +for diagnostics, and no version grants map-based control authority. Its referenced enums live inside the reserved +struct with their original explicit type IDs. Compiled-layout and real historical/Go +wire fixtures protect the restored contract; core `car.capnp` is unchanged. +Slots17/18 now restore the historical extended status and input layouts as well. +The newer provider adds position and loop-rate status, a JSON path input field, +and further input enum values. All older fields, helper type IDs and enum meanings +are preserved; helper types are nested inside their reserved roots. Opaque pointer +defaults are compared as serialized schema values in the compiled contract. +This schema restoration alone does not start the provider, expose settings +commands, or connect it to SLC control. +Development AOL safety and intent previously collided with 143/144 and are +retired from the compiled contract. Their transport now uses separate reserved +`Data` ordinals 124/125 with bounded, versioned flat nested payload layouts protected by the +compiled contract. The previous contract is archived in the external migration +audit. Old development recordings need source-aware conversion; merely matching +an Event ordinal must never select an AOL decoder. + +Lane Change Assist uses the separate reserved `Data` ordinal 126. Its bounded +typed payload is nested under `AolAxisState` as an independent schema declaration; +it adds no fields to the published AOL state. Modeld owns this optional diagnostic +service. Its frame-matched status changes pre-change alert wording and grants no +control authority; missing status does not block ordinary engagement. The compiled +contract protects the new payload and enums as well as every prior layout. + +The native StarPilot test suite also runs `test_custom_schema_evolution.py`. +`custom-schema-contract.json` records the compiled layouts of the first populated +custom messages. This check allows additions while preserving their field names, +types, offsets, defaults, union tags and enum ordinals. Field names are protected +as an application API in addition to the wire layout. Extend the contract from +a reviewed committed schema when a new message generation becomes a baseline; +never regenerate it just to silence an incompatible change. This does not convert +recordings made with older divergent vehicle schemas. +The reviewed baseline includes the SLC and independent-axis message families. +The unit job also executes the native Panda protocol test with fake transport; +it exercises the parser and negotiation state used by pandad without a device. +`test_aol_wire.cc` asserts the native safety encoder's bounded flat payload; +the matching Python fixture decodes its exact bytes. A full Linux pandad build +is still required for native call-site coverage on a Linux target. + +The default `tools/op.sh test` discovers the `openpilot/` tree. Vendored dependency +tests need explicit jobs. The root workflow runs these host-only suites: + +```sh +python tools/ci/run_vehicle_tests.py --suite interfaces --output ci-results/interfaces +python tools/ci/run_vehicle_tests.py --suite safety-debug --output ci-results/safety-debug +python tools/ci/run_vehicle_tests.py --suite safety-release --output ci-results/safety-release +``` + +Add `--collect-only` to inspect the selected IDs without running tests. Collection +is explicitly marked `not_run`. Each run writes `plan.json`, complete per-test +`results.json`, and `coverage.json` with source/dependency identity, native library +hashes, actual safety compiler commands and version, outcomes and uncovered areas. The safety +library used by behavioral tests is compiled once, loaded explicitly, and used in +one test process; no worker can silently rebuild a different loaded variant. +The runner also compares the selected vehicle/control source areas before and +after execution, including uncommitted patches and untracked files. A changed +snapshot fails the gate while retaining every individual test outcome. Keep +results outside those source areas. These endpoint checks do not detect a change +that is reverted during execution; qualification runs still require an isolated, +stable checkout. Native binary provenance remains separate from source snapshots. +The debug suite also retains the existing two compile-only variant tests. +`FUZZ_SEED` defaults to `0` for +reproducible input generation. An explicit `MAX_EXAMPLES` override is recorded. + +The interface suite recursively includes `test_*.py` files throughout `opendbc/car`, +including brand-specific tests. Only the exact route module `car/tests/test_models.py` +is excluded. Each module's source hash and collected count are recorded; a module +that silently collects zero tests fails the gate. Files containing module-level +pytest tests or plain pytest classes run once through pytest, including any unittest +classes in those mixed files; other files retain the unittest runner. Both runners +execute serially in the same process with the pinned native library. AST checks +reject declared test functions and methods without a collected counterpart, +including when other tests in that module were collected. +It asserts exact equality between the current platform registry and concrete +generated interface tests: a missing, duplicate or extra platform fails collection. +Abstract-class skips cannot satisfy platform coverage. +Every mapped concrete interface test must pass: a class-level skip stays visible +in the report and fails the interface coverage gate. Initialization and collection +errors also produce error reports, including when a native import fails. +The suite also runs the existing +platform, limits, fingerprint, docs and route-declaration checks. A route exemption +is reported as missing route coverage, not as a tested vehicle. + +`test_models.py` generates its route-backed classes only when directly executed. +Importing it with generic discovery would produce abstract skips, not a full +vehicle replay. Historical route/schema qualification remains a separate gate; +this workflow does not fetch those routes or regenerate expected outputs. + +## Recorded vehicle regressions + +`run_recorded_vehicle_tests.py` runs the pinned public Model Y CAN fixture through +current parser, controller and native safety tests. DEBUG includes stock and +alpha-long configurations; RELEASE includes stock only. Alpha-long exclusion is +recorded explicitly, and missing, skipped or errored selected cases fail the run. +This checks recorded CAN and synthetic controller behavior, not the user's +weekend failure, physical harness operation or complete vehicle identification. + +```sh +./dev python tools/ci/run_recorded_vehicle_tests.py --variant debug --cache /absolute/cache/path --output /absolute/results/path --download +``` + +Use `--variant release` for the separate RELEASE library. `--download` permits +only the manifest's public fixture URLs before execution; cached compressed +bytes must match their pinned size and SHA-256. Omit it for fully offline use. +Each process owns temporary Params and IPC, disables replay networking, records +actual controller configuration and native library identity, and rejects source +changes during the run. Use a new result directory to retain earlier evidence. +The hosted `recorded_vehicles` matrix runs both variants and retains reports even +on failure. A local pass does not establish the result of a later GitHub run. + +The debug safety suite includes all top-level safety test files with `ALLOW_DEBUG`. +The release safety suite compiles without that flag and checks the entire enum +against the explicit release hook registry. Its behavioral cases are deliberately +limited to no-output/silent, stock Tesla (including FSD14), Mazda, stock-longitudinal +Toyota torque, GM camera/EV and four additional manual camera-ACC variants, Bolt +Pedal, intercepted GM ASCM, stock SDGM, nine stock conventional-cruise gateway +variants, Ford ownership/Transit steering contracts, six manual Honda/Acura Nidec +variants, three manual radarless Honda/Acura Bosch variants, two manual Subaru +angle-steering variants, four manual MRR35 Hyundai/Genesis angle variants, three +manual Hyundai/Kia angle hybrids, two manual CCNC angle SUVs, and Carnival alternate-button resume. The Carnival tests use +the runner's recorded safety library and include a real stock-cruise RX sequence +with the exact frame emitted by the host controller. It also checks that the +DEBUG-only Honda Bosch longitudinal path cannot grant AOL requests in RELEASE, +including the classic Bosch family controller frames and brake-source variants. +The Ioniq 6 tests enumerate every bit-15 safety parameter and require RELEASE +to deny all dedicated longitudinal/AOL profiles. Their DEBUG controller and +takeover cases remain explicit skips in the RELEASE result. +The selected classes are listed in `RELEASE_TARGETS`; +other release behaviors are uncovered, not silently skipped into a green result. +The suite is not a whole-fleet release-mode safety qualification. + +Two separate hosted jobs compile Panda firmware with DEBUG and RELEASE flags. +Both use the checked-in development certificate. The recorder requires an actual +main-source compile command with matching flags and captures ELF/binary hashes and +the version stamp. These jobs do not use release credentials, flash hardware or +qualify firmware compatibility. Build logs and provenance are retained on failure. + +All jobs run on hosted Ubuntu without a device or self-hosted runner. A passing +interface test or safety hook unit test does not cover additional downstream car +ports, independent lateral/longitudinal modes, real CAN forwarding, hardware, +timing, logs from another schema, AGNOS, or deployment/rollback behavior. Skips +remain explicit in the JSON report; they do not count as passes. + +After the interface suite, the workflow runs the +[exact fleet coverage gate](FLEET_COVERAGE.md) and saves its report in the same +artifact. This step fails while required ports, current interface results or +recorded configuration/mode evidence are missing. It currently supplies no mode +evidence, so the fleet gate remains uncovered even if every interface test passes. +This is intentional during reconstruction: successful unit tests alone must not +produce a fleet-wide green result. Recorded mode evidence needs its own reviewed +producer before it can be connected here. The gate also runs after an interface +failure; available collection and execution records remain in the artifact. + +The small collection/provenance fixtures need only the standard library: + +```sh +python -m unittest discover -s tools/ci/tests -v +``` + +The map-provider job runs `tools/ci/run_map_tests.py` with the Go version pinned +in `mapd_repo/go.mod`. It records test outcomes, vet/build results, dependency +pin, source hashes and embedded build identifiers. It does not start the provider +or fetch maps. Runtime integration and a Linux ARM64/device build remain separate +checks; see `mapd_repo/INTEGRATION.md`. + +The unit job additionally requires the real Go-to-host map messaging test after +building the host's native dependencies. It supplies pinned Go 1.25.1 and the +repository Python environment, then runs this from `mapd_repo`: + +```sh +STARPILOT_SHADOW_IPC_REQUIRED=1 STARPILOT_SHADOW_EXEC_REQUIRED=1 \ +STARPILOT_PYTHON=/absolute/path/to/.venv/bin/python \ + go test -mod=readonly -race -run '^TestShadow(GoPythonIPC|ExecutableHostGpsIPC)$' -count=1 -v +``` + +Both tests use synthetic GPS/tiles and a disposable IPC namespace. The first +uses a Go pump and publisher harness; the second builds and starts the actual +shadow executable under isolated settings, then passes host GPS through the Go +subscriber and reads its output in Python. Neither uses installed maps or the +full provider mode. Missing host dependencies fail these required tests. The +standalone provider suite reports explicit skips when their required targets +are not selected; those skips are covered by the required unit-job step, not +counted as passes. + +The Ioniq 6 development branch retains custom AGNOS `19.8.1`, matching the +separately installed image. `AGNOS_UPDATE_POLICY=retain` prevents the launcher, background +updater, and development branch helper from replacing that OS. A version +mismatch blocks startup/update finalization and reports the required version. +Other branches without this policy keep their explicit upstream update behavior. +The retained version is an installation policy; it does not establish native +ABI, camera, model inference or device compatibility with the upstream baseline. +`openpilot/system/updated/tests/test_retained_agnos.py` exercises these paths +using disposable OS markers, fake hardware and intercepted firmware calls. + +For a read-only parked trial prerequisite report, run +`python3 tools/ci/run_device_preflight.py --font-dir /path/to/complete/reviewed/fonts --json-output /path/to/report.json`. +The standard-library driver reads the retained AGNOS and Python policy from +`launch_env.sh` and `pyproject.toml`, checks the UI's existing bitmap-font and +artwork manifests, and tries selected library imports in isolated, timed Python +children. It reads `/VERSION` only on a Linux ARM64 host; on a desktop, target +OS status is unavailable. It never starts services, accesses controls or +qualifies graphics, camera, CAN, model inference or driving. Missing optional +font input and absent device checks stay visible rather than becoming passes. +The import probes put this source tree's vendored Python roots first and reject +any loaded openpilot, msgq, opendbc or tinygrad module from another checkout. +The JSON `prerequisites` field is `pass`, `fail` or `incomplete`; command exit +codes are 0, 1 and 2 respectively. Even `pass` is a prerequisite report, never +a device or driving qualification. + +The separate safety-qualification workflow runs four nonmutation GitHub gates: +C-line coverage, pinned opendbc MISRA, Panda MISRA, and Panda host tests. +Mutation inventory and full mutation run locally in a disposable checkout: +`python tools/ci/run_safety_qualification.py --gate mutation-list --output /new/inventory/dir` +`python tools/ci/run_safety_qualification.py --gate mutation-full --output /new/mutation/dir` +Both the workflow and local gates use the root locked `safety` extra; +they do not call the nested opendbc or Panda setup scripts, which would sync or +upgrade separate environments. A local disposable checkout can run one gate with +`python tools/ci/run_safety_qualification.py --gate coverage --output /new/evidence/dir` +(other gates: `opendbc-misra`, `panda-misra`, `panda-host`, `mutation-list`, +`mutation-full`). Reports include source hashes, imported module origins, exact +commands, full logs, test/skip counts, and an explicit pass/fail result. Coverage +requires 100% of upstream checked C lines; full mutation requires zero surviving +or infrastructure-error mutants and first runs the full unmutated test catalog. +`HITL` is explicitly unset: Panda/Jungle/vehicle hardware tests remain a separate +bench gate and their skips are recorded, never labeled as host passes. diff --git a/tools/ci/__init__.py b/tools/ci/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci/fleet_coverage.py b/tools/ci/fleet_coverage.py new file mode 100644 index 0000000000..c6330f0a40 --- /dev/null +++ b/tools/ci/fleet_coverage.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Report exact source-platform obligations without qualifying any vehicle.""" + +import argparse +from collections import Counter +import hashlib +import importlib.machinery +import json +from pathlib import Path +import re +import subprocess + +from openpilot.starpilot.validation.control_modes import SCENARIOS, configuration_id, validate_matrix + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_MANIFEST = ROOT / "openpilot/starpilot/validation/required-platforms.json" +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +REVISION = re.compile(r"[0-9a-f]{40}\Z") +NATIVE_CAN_SOURCES = {"opendbc_repo/opendbc/can/packer.py", "opendbc_repo/opendbc/can/parser.py"} +RELEVANT_SOURCE_PATHS = ("opendbc_repo", "openpilot/common", "openpilot/cereal", "msgq_repo", + "tools/ci/run_vehicle_tests.py", "tools/test_runner.py", "upstream-sync.json") + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def canonical_sha256(value): + """Hash compact, key-sorted UTF-8 JSON without a trailing newline.""" + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest() + + +def relevant_dirty_paths(): + output = subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=all", "--", *RELEVANT_SOURCE_PATHS], + cwd=ROOT, text=True) + return output.splitlines() + + +def _relevant_status_lines(status): + if not isinstance(status, str): + return [] + return [line for line in status.splitlines() if len(line) >= 4 and any( + line[3:] == path or line[3:].startswith(path + "/") for path in RELEVANT_SOURCE_PATHS)] + + +def _ids(values, label, errors): + if not isinstance(values, list) or any(not isinstance(v, str) or not v for v in values): + errors.append(f"{label}: expected nonempty string IDs") + return [] + duplicates = sorted(k for k, count in Counter(values).items() if count > 1) + if duplicates: + errors.append(f"{label}: duplicate exact IDs {duplicates}") + return values + + +def evaluate(manifest, coverage, results, mode_evidence=None, *, expected_revision=None, hashes=None, dirty_relevant=None): + """Check independent registration, test execution and per-config mode evidence. + + All input documents are caller supplied; their physical origin is not asserted. + """ + errors, failures, uncovered = [], [], [] + if not isinstance(manifest, dict): + errors.append("required-platform manifest: expected object") + manifest = {} + if not isinstance(coverage, dict): + errors.append("interface coverage: expected object") + coverage = {} + if not isinstance(results, dict): + errors.append("interface results: expected object") + results = {} + if mode_evidence is not None and not isinstance(mode_evidence, dict): + errors.append("mode evidence: expected object") + mode_evidence = {} + if dirty_relevant is None: + dirty_relevant = relevant_dirty_paths() + if dirty_relevant: + uncovered.append("relevant vehicle/runner source differs from committed checkout") + platforms = manifest.get("platforms") + if manifest.get("schema_version") != 1 or not isinstance(platforms, list): + errors.append("required-platform manifest: expected schema v1 and platforms list") + platforms = [] + required = _ids([r.get("id") if isinstance(r, dict) else None for r in platforms], "required platforms", errors) + required_set = set(required) + if not required_set: + errors.append("required-platform manifest: at least one exact source ID required") + additions = _ids(manifest.get("upstream_additions"), "upstream additions", errors) + addition_set = set(additions) + if required_set & addition_set: + errors.append("required source platforms and upstream additions overlap") + all_obligations = required_set | addition_set + for row in platforms: + if isinstance(row, dict) and (not row.get("family") or row.get("source_declaration") not in ("source_only", "same_declaration", "declaration_changed")): + errors.append(f"required platform metadata invalid: {row.get('id')}") + if not REVISION.fullmatch(str(manifest.get("source_revision", ""))) or not SHA256.fullmatch(str(manifest.get("source_inventory_sha256", ""))): + errors.append("required-platform manifest source provenance missing") + + registered = _ids(coverage.get("platform_ids"), "reported registry", errors) + registered_set = set(registered) + mapped = coverage.get("interface_test_ids") + if not isinstance(mapped, dict): + errors.append("interface_test_ids: expected exact ID to test ID mapping") + mapped = {} + elif any(not isinstance(k, str) or not k for k in mapped): + errors.append("interface_test_ids: mapping keys must be exact string IDs") + mapped = {k: v for k, v in mapped.items() if isinstance(k, str) and k} + selected = _ids(coverage.get("selected_test_ids"), "selected tests", errors) + raw_records = results.get("results") + if not isinstance(raw_records, list): + errors.append("results: expected per-test records") + raw_records = [] + record_ids = [r.get("id") if isinstance(r, dict) else None for r in raw_records] + if any(not isinstance(v, str) for v in record_ids) or len(record_ids) != len({v for v in record_ids if isinstance(v, str)}): + errors.append("results: invalid or duplicate test IDs") + by_test = {r["id"]: r for r in raw_records if isinstance(r, dict) and isinstance(r.get("id"), str)} + nonpassing_records = [r for r in raw_records if isinstance(r, dict) and r.get("status") != "passed"] + for record in nonpassing_records: + issue = f"execution record {record.get('id')}: {record.get('status', 'missing status')}" + if record.get("status") == "skipped": + uncovered.append(issue) + else: + failures.append(issue) + if coverage.get("suite") != "interfaces" or coverage.get("schema_version") != 1: + errors.append("coverage: expected interface suite schema v1") + if results.get("schema_version") != 1: + errors.append("results: expected schema v1") + result_selected = results.get("selected_test_ids") + if not isinstance(result_selected, list) or any(not isinstance(v, str) for v in result_selected): + errors.append("results selected_test_ids: expected test IDs") + result_selected = [] + if set(selected) != set(by_test) or set(selected) != set(result_selected): + errors.append("selected tests and execution records differ") + mapped_values = list(mapped.values()) + unique_mapped = {v for v in mapped_values if isinstance(v, str)} + if set(mapped) != registered_set or len(unique_mapped) != len(mapped): + errors.append("registered platforms and unique interface test mapping differ") + if coverage.get("platform_count") != len(registered) or coverage.get("result_sha256") != (hashes or {}).get("results"): + errors.append("interface coverage count or companion results hash invalid") + if coverage.get("interface_results") != {p: by_test.get(t) if isinstance(t, str) else None for p, t in mapped.items()}: + errors.append("embedded interface results differ from execution records") + for p, test_id in mapped.items(): + if not isinstance(test_id, str) or not test_id.endswith(".test_car_interfaces_" + p) or test_id not in selected: + errors.append(f"{p}: concrete interface test mapping invalid") + if coverage.get("status") != "completed" or results.get("status") != "completed" or coverage.get("exit_code") != 0 or results.get("exit_code") != 0: + failures.append("interface suite did not complete successfully") + if coverage.get("collection_errors") or coverage.get("gate_errors") or results.get("collection_errors") or results.get("gate_errors"): + failures.append("interface suite reported collection or gate failures") + source = coverage.get("source") + source = source if isinstance(source, dict) else {} + if not isinstance(source.get("status"), str): + uncovered.append("interface report working-tree status missing or malformed") + reported_dirty = _relevant_status_lines(source.get("status")) + if reported_dirty: + uncovered.append("interface report recorded relevant dirty vehicle/runner source") + revision = source.get("revision") + dependencies = source.get("dependencies") + opendbc = [d for d in dependencies if isinstance(d, dict) and d.get("path") == "opendbc_repo"] if isinstance(dependencies, list) else [] + if not REVISION.fullmatch(str(revision or "")) or len(opendbc) != 1 or not REVISION.fullmatch(str(opendbc[0].get("commit", ""))): + errors.append("interface source/dependency provenance missing") + if expected_revision is not None and revision != expected_revision: + uncovered.append("interface report source revision differs from requested checkout") + current_manifest = ROOT / "upstream-sync.json" + current_runner = ROOT / "tools/ci/run_vehicle_tests.py" + current_test_runner = ROOT / "tools/test_runner.py" + for field, path in (("manifest_sha256", current_manifest), ("suite_runner_sha256", current_runner), ("runner_sha256", current_test_runner)): + if source.get(field) != sha256(path): + uncovered.append(f"interface report {field} differs from current source") + pinned = json.loads(current_manifest.read_text()).get("dependencies", []) + pinned_opendbc = [d for d in pinned if d.get("path") == "opendbc_repo"] + if len(pinned_opendbc) != 1 or len(opendbc) != 1 or opendbc[0].get("commit") != pinned_opendbc[0].get("commit"): + uncovered.append("interface report opendbc revision differs from current pin") + native = coverage.get("native") + if not isinstance(native, dict) or not isinstance(native.get("can_sources"), dict) or not isinstance(native.get("pycapnp"), dict): + uncovered.append("interface report native parser/packer provenance missing") + else: + if set(native["can_sources"]) != NATIVE_CAN_SOURCES: + uncovered.append("interface report native CAN source set incomplete or unexpected") + for source_path, digest in native["can_sources"].items(): + if not isinstance(source_path, str) or not (ROOT / source_path).is_file() or digest != sha256(ROOT / source_path): + uncovered.append(f"interface report native CAN source mismatch: {source_path}") + pycapnp = native["pycapnp"] + binary_name = pycapnp.get("path") + binary_path = Path(binary_name) if isinstance(binary_name, str) else None + if (binary_path is None or not any(str(binary_path).endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES) + or not binary_path.is_file() or pycapnp.get("sha256") != sha256(binary_path)): + uncovered.append("interface report pycapnp binary unavailable or changed") + if results.get("selected_test_ids") != selected: + errors.append("selected test order differs between coverage and results") + current_execution_compatible = not errors and not failures and not dirty_relevant and not reported_dirty + current_execution_compatible &= not any(issue.startswith("interface report ") for issue in uncovered) + + platform_rows = [] + for p in sorted(required_set): + if p not in registered_set: + interface = "missing_port" + elif p not in mapped: + interface = "missing_test" + else: + test_id = mapped[p] + status = by_test.get(test_id, {}).get("status") if isinstance(test_id, str) else None + interface = "passed" if status == "passed" else "missing_test" if status is None else status + current_interface = interface if interface != "passed" else "passed" if current_execution_compatible else "historical_pass_only" + platform_rows.append({"id": p, "obligation": "source", "interface": interface, + "current_interface": current_interface, "mode": "uncovered"}) + if interface != "passed": + uncovered.append(f"{p}: interface {interface}") + if interface not in ("missing_port", "missing_test", "skipped"): + failures.append(f"{p}: concrete interface test {interface}") + upstream_only = sorted(registered_set - required_set) + if set(upstream_only) != addition_set: + uncovered.append("upstream additions differ from frozen manifest") + # Additions remain independent obligations even if a future registry drops one. + for p in sorted(addition_set): + status = by_test.get(mapped.get(p), {}).get("status") if isinstance(mapped.get(p), str) else None + reported_interface = "passed" if status == "passed" else "missing_port" if p not in registered_set else status or "missing_test" + current_interface = "historical_pass_only" if reported_interface == "passed" and not current_execution_compatible else reported_interface + platform_rows.append({"id": p, "obligation": "upstream_addition", + "interface": reported_interface, "current_interface": current_interface, + "mode": "uncovered"}) + if status != "passed": + uncovered.append(f"upstream addition {p}: concrete interface test {status or 'missing'}") + if status not in (None, "skipped"): + failures.append(f"upstream addition {p}: concrete interface test {status}") + + mode_summary = {"input": "absent", "required_scenarios": list(SCENARIOS), "configuration_count": 0, "recorded_trace_count": 0} + if mode_evidence is None: + uncovered.append("independent per-configuration mode evidence absent") + else: + mode_summary["input"] = "supplied" + if mode_evidence.get("schema_version") != 1: + errors.append("mode evidence: expected schema v1") + provenance = mode_evidence.get("provenance") + provenance_keys = ("configuration_inventory_sha256", "trace_catalog_sha256") + if not isinstance(provenance, dict) or any(not SHA256.fullmatch(str(provenance.get(k, ""))) for k in provenance_keys): + uncovered.append("mode evidence: configuration inventory or trace catalog provenance missing") + configurations = mode_evidence.get("configurations") + traces = mode_evidence.get("traces") + if not isinstance(configurations, list) or not isinstance(traces, list): + errors.append("mode evidence: configurations and traces must be lists") + configurations, traces = [], [] + if isinstance(provenance, dict): + if provenance.get("configuration_inventory_sha256") != canonical_sha256(configurations): + errors.append("mode evidence: configuration inventory digest mismatch") + if provenance.get("trace_catalog_sha256") != canonical_sha256(traces): + errors.append("mode evidence: trace catalog digest mismatch") + mode_summary["configuration_count"] = len(configurations) + mode_summary["recorded_trace_count"] = len(traces) + config_ids = [configuration_id(c) for c in configurations if isinstance(c, dict)] + if len(config_ids) != len(configurations) or len(config_ids) != len(set(config_ids)): + errors.append("mode evidence: invalid or duplicate exact configurations") + if any(not isinstance(c, dict) or not isinstance(c.get("platform"), str) or c["platform"] not in all_obligations for c in configurations): + errors.append("mode evidence: configuration platform is not an exact obligated ID") + if any(not isinstance(t, dict) or not isinstance(t.get("configuration"), dict) or configuration_id(t["configuration"]) not in config_ids for t in traces): + errors.append("mode evidence: trace has no exact declared configuration") + complete = mode_evidence.get("enumeration_complete") + if complete is not True: + uncovered.append("mode evidence: exact configuration enumeration not asserted complete") + for platform in sorted(all_obligations): + row = next((r for r in platform_rows if r["id"] == platform), None) + configs = [c for c in configurations if isinstance(c, dict) and c.get("platform") == platform] + if not configs: + uncovered.append(f"{platform}: no exact configurations enumerated") + continue + platform_traces = [t for t in traces if isinstance(t, dict) and isinstance(t.get("configuration"), dict) + and t["configuration"].get("platform") == platform] + try: + matrix = validate_matrix(platform_traces, configs) + except (KeyError, TypeError, ValueError) as exc: + errors.append(f"{platform}: malformed mode matrix: {type(exc).__name__}") + continue + if row is not None: + row["mode"] = matrix["status"] + if matrix["status"] == "error": + errors.append(f"{platform}: malformed mode trace") + elif matrix["status"] == "failed": + failures.append(f"{platform}: mode invariant or expectation failed") + elif matrix["status"] == "uncovered": + uncovered.append(f"{platform}: {len(matrix['uncovered'])} mode scenario/configuration gaps") + + status = "error" if errors else "failed" if failures else "uncovered" if uncovered else "pass" + if errors or failures: + current_execution_compatible = False + for row in platform_rows: + if row["current_interface"] == "passed": + row["current_interface"] = "historical_pass_only" + return {"schema_version": 1, "scope": "exact_source_platform_and_recorded_configuration_coverage", + "vehicle_qualification": "not_established", "status": status, + "source_revision": manifest.get("source_revision"), "interface_report_revision": revision, + "hashes": hashes or {}, "required_platform_count": len(required_set), + "upstream_addition_count": len(addition_set), "registered_platform_count": len(registered_set), + "upstream_only": upstream_only, "required_upstream_additions": sorted(addition_set), + "evidence_limit": "Supplied recording labels and hashes do not authenticate trace adapters or physical origin.", + "relevant_dirty_paths": dirty_relevant, + "reported_relevant_dirty_paths": reported_dirty, + "interface_execution_context": "current_compatible" if current_execution_compatible else "historical_or_incompatible", + "reported_interface_counts": dict(Counter(r["interface"] for r in platform_rows)), + "current_interface_counts": dict(Counter(r["current_interface"] for r in platform_rows)), + "platforms": platform_rows, + "mode_evidence": mode_summary, "errors": errors, "failures": failures, "uncovered": uncovered} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) + parser.add_argument("--interface-coverage", type=Path, required=True) + parser.add_argument("--interface-results", type=Path, required=True) + parser.add_argument("--mode-evidence", type=Path) + parser.add_argument("--expected-revision", help="40-character checkout revision; defaults to current HEAD") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + parser.error(f"output already exists; refusing to replace evidence: {args.output}") + paths = {"manifest": args.manifest, "coverage": args.interface_coverage, "results": args.interface_results} + if args.mode_evidence: + paths["mode_evidence"] = args.mode_evidence + hashes = {k: sha256(v) for k, v in paths.items()} + checkout_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + if args.expected_revision is not None and args.expected_revision != checkout_revision: + parser.error("expected revision differs from current checkout HEAD") + expected = checkout_revision + report = evaluate(json.loads(args.manifest.read_text()), json.loads(args.interface_coverage.read_text()), + json.loads(args.interface_results.read_text()), + json.loads(args.mode_evidence.read_text()) if args.mode_evidence else None, + expected_revision=expected, hashes=hashes, dirty_relevant=relevant_dirty_paths()) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + stream.write(json.dumps(report, indent=2, sort_keys=True) + "\n") + summary = (f"fleet coverage: {report['status']}; required={report['required_platform_count']}; " + + f"registered={report['registered_platform_count']}; missing={report['reported_interface_counts'].get('missing_port', 0)}; " + + f"current passes={report['current_interface_counts'].get('passed', 0)}; " + + f"mode configurations={report['mode_evidence']['configuration_count']}; report={args.output}") + print(summary) + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/record_firmware.py b/tools/ci/record_firmware.py new file mode 100644 index 0000000000..5acbba0ade --- /dev/null +++ b/tools/ci/record_firmware.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Record firmware build commands/artifacts without installing or flashing them.""" +import argparse +import json +from pathlib import Path +import shlex +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.ci.run_vehicle_tests import ROOT, sha256, source_provenance + + +def compile_commands(text, variant): + commands = [] + for line in text.splitlines(): + if "arm-none-eabi-gcc" not in line: + continue + command = shlex.split(line) + if "-c" in command and any(token.endswith("board/main.c") for token in command): + commands.append(command) + if not commands: + raise ValueError("Build log has no actual Panda main compilation command") + if any(("-DALLOW_DEBUG" in command) != (variant == "debug") for command in commands): + raise ValueError("Panda compiled flags do not match the requested variant") + return commands + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--variant", choices=("debug", "release"), required=True) + parser.add_argument("--build-log", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + commands = compile_commands(args.build_log.read_text(), args.variant) + files = [ROOT / "panda/board/obj" / name for name in ( + "panda.bin.signed", "panda/main.elf", "bootstub.panda.bin", + "panda_h7.bin.signed", "panda_h7/main.elf", "bootstub.panda_h7.bin", "version")] + version = files[-1].read_text() + if f"-{args.variant.upper()}" not in version: + raise ValueError("Panda version stamp does not match the compiled variant") + report = {"schema_version": 1, "variant": args.variant, "source": source_provenance(), "commands": commands, + "compiler_version": subprocess.check_output(["arm-none-eabi-gcc", "--version"], text=True), + "artifacts": {str(path.relative_to(ROOT)): sha256(path) for path in files}, + "version_stamp": version, "build_log_sha256": sha256(args.build_log), + "certificate": "checked-in development certificate, including the RELEASE build", + "certificate_sha256": sha256(ROOT / "panda/board/crypto/certs/debug"), + "scope": "compile/link only; no release-key signing, flashing, device execution, or firmware compatibility qualification"} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tools/ci/run_device_preflight.py b/tools/ci/run_device_preflight.py new file mode 100644 index 0000000000..c15cb8386d --- /dev/null +++ b/tools/ci/run_device_preflight.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Read-only parked-device prerequisites; never starts StarPilot or probes hardware.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import stat +import subprocess +import sys +import tomllib + + +SOURCE = Path(__file__).resolve().parents[2] +UI = Path("openpilot/starpilot/ui") +MANIFESTS = ("bitmap-fonts.json", "sora-brand-font.json", "home-assets.json", "settings-assets.json", + "toggles-assets.json", "onroad-assets.json") +IMPORTS = ("pyray", "capnp", "numpy", "cv2", "tinygrad", "openpilot.cereal.messaging") +MAX_MANIFEST = 1 << 20 +MAX_ASSET = 16 << 20 +PROBE_TIMEOUT = 8 + + +def _read_regular(path: Path, limit: int) -> bytes: + flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(path, flags) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_size > limit: + raise ValueError("not a bounded regular file") + with os.fdopen(fd, "rb", closefd=False) as stream: + data = stream.read(limit + 1) + if len(data) != info.st_size or len(data) > limit: + raise ValueError("file changed or exceeds limit") + return data + finally: + os.close(fd) + + +def _check(status: str, detail: str) -> dict[str, str]: + return {"status": status, "detail": detail} + + +def _policy(source: Path) -> tuple[dict, str]: + launch = _read_regular(source / "launch_env.sh", MAX_MANIFEST).decode("utf-8") + project = tomllib.loads(_read_regular(source / "pyproject.toml", MAX_MANIFEST).decode("utf-8")) + version = re.findall(r'^[ \t]*export AGNOS_VERSION="([^"]+)"$', launch, re.MULTILINE) + policy = re.findall(r'^[ \t]*export AGNOS_UPDATE_POLICY="([^"]+)"$', launch, re.MULTILINE) + required_python = project["project"]["requires-python"] + if version != ["19.8.1"] or policy != ["retain"]: + raise ValueError("retained AGNOS source policy differs from reviewed trial") + if required_python != ">= 3.12.3, < 3.13": + raise ValueError("project Python requirement differs from reviewed trial") + return {"agnos_version": version[0], "update_policy": policy[0], + "requires_python": required_python}, hashlib.sha256(launch.encode()).hexdigest() + + +def _manifest_records(source: Path, name: str) -> list[dict]: + data = json.loads(_read_regular(source / UI / name, MAX_MANIFEST)) + records = data["files"] + if not isinstance(records, list) or not records: + raise ValueError("empty asset manifest") + names = set() + for record in records: + filename = record["file"] + relative = Path(filename) + if (not isinstance(filename, str) or relative.is_absolute() or + any(part in ("", ".", "..") for part in relative.parts) or filename in names or + not isinstance(record["bytes"], int) or isinstance(record["bytes"], bool) or + not 0 <= record["bytes"] <= MAX_ASSET or + not re.fullmatch(r"[0-9a-f]{64}", record["sha256"])): + raise ValueError("invalid asset manifest entry") + names.add(filename) + return records + + +def _assets(source: Path, font_dir: Path | None) -> dict[str, dict]: + result = {} + for name in MANIFESTS: + try: + records = _manifest_records(source, name) + if name == "bitmap-fonts.json": + root = font_dir if font_dir is not None else source / UI / "assets/fonts" + elif name == "sora-brand-font.json": + root = source / UI / "assets/fonts" + else: + root = source / "openpilot/selfdrive/assets" + for record in records: + data = _read_regular(root / record["file"], record["bytes"]) + if len(data) != record["bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]: + raise ValueError("missing or mismatched asset") + result[name] = _check("pass", f"{len(records)} manifest-pinned files match") + except (OSError, ValueError, KeyError, TypeError, UnicodeError, json.JSONDecodeError): + result[name] = _check("fail", "missing, malformed or mismatched reviewed asset") + return result + + +def _probe_import(source: Path, python: str, module: str) -> dict[str, str]: + # The child imports one named library only. It creates no service, camera, + # Params instance or model runner. Do not include child stderr in the report. + program = """import importlib, pathlib, sys +source = pathlib.Path(sys.argv[1]).resolve() +vendor = {'openpilot': source / 'openpilot', + 'msgq': source / 'msgq_repo', + 'opendbc': source / 'opendbc_repo', + 'tinygrad': source / 'tinygrad_repo', + 'jetlink': source / 'jetlink_repo'} +sys.path[:0] = [str(source / path) for path in + ('', 'msgq_repo', 'opendbc_repo', 'rednose_repo', 'teleoprtc_repo', 'tinygrad_repo', 'jetlink_repo')] +try: importlib.import_module(sys.argv[2]) +except Exception as error: + print(type(error).__name__) + sys.exit(1) +for name, loaded in tuple(sys.modules.items()): + prefix = name.split('.', 1)[0] + if prefix in vendor: + origin = getattr(loaded, '__file__', None) + if origin is None or not pathlib.Path(origin).resolve().is_relative_to(vendor[prefix]): + print('OriginMismatch') + sys.exit(1) +""" + env = {key: os.environ[key] for key in ("PATH", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH") if key in os.environ} + env["PYTHONDONTWRITEBYTECODE"] = "1" + try: + run = subprocess.run([python, "-I", "-B", "-c", program, str(source), module], + cwd=source, env=env, capture_output=True, timeout=PROBE_TIMEOUT, check=False) + except subprocess.TimeoutExpired: + return _check("fail", "isolated import timed out") + except OSError: + return _check("fail", "Python executable unavailable") + if run.returncode == 0: + return _check("pass", "isolated import succeeded") + error = (run.stdout or b"").decode("ascii", errors="ignore").strip() + allowed = {"ImportError", "ModuleNotFoundError", "OSError", "RuntimeError", "ValueError", "SyntaxError", "OriginMismatch"} + return _check("fail", f"isolated import failed: {error}" if error in allowed else "isolated import failed") + + +def collect(source: Path = SOURCE, font_dir: Path | None = None, *, + python: str = sys.executable, os_version_file: Path = Path("/VERSION"), + system: str | None = None, machine: str | None = None, + python_version: tuple[int, int, int] | None = None, + probe=_probe_import) -> dict: + source = source.resolve() + system = platform.system() if system is None else system + machine = platform.machine() if machine is None else machine + python_version = sys.version_info[:3] if python_version is None else python_version + try: + revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=source, capture_output=True, + text=True, timeout=3, check=True).stdout.strip() + except (OSError, subprocess.SubprocessError): + revision = "unavailable" + checks: dict[str, dict] = {} + try: + policy, launch_sha = _policy(source) + checks["source_policy"] = _check("pass", "retained AGNOS and Python source policy match") + except (OSError, ValueError, KeyError, TypeError, UnicodeError): + policy, launch_sha = None, None + checks["source_policy"] = _check("fail", "retained AGNOS or Python source policy unavailable/mismatched") + checks["python_version"] = _check("pass" if (3, 12, 3) <= python_version < (3, 13, 0) else "fail", + ".".join(map(str, python_version))) + if system == "Linux" and machine in ("aarch64", "arm64"): + try: + actual = _read_regular(os_version_file, 128).decode("ascii").strip() + checks["target_agnos"] = _check("pass" if policy and actual == policy["agnos_version"] else "fail", + "retained version matches" if policy and actual == policy["agnos_version"] else "version unavailable/mismatched") + except (OSError, ValueError, UnicodeError): + checks["target_agnos"] = _check("unavailable", "target OS marker unavailable") + else: + checks["target_agnos"] = _check("unavailable", "target-device OS check unavailable on this host") + checks.update(_assets(source, font_dir)) + checks["runtime_imports"] = {module: probe(source, python, module) for module in IMPORTS} + statuses = [item["status"] for key, item in checks.items() if key != "runtime_imports"] + statuses.extend(item["status"] for item in checks["runtime_imports"].values()) + prerequisites = "fail" if "fail" in statuses else "incomplete" if "unavailable" in statuses else "pass" + source_hashes = {} + for relative in ("launch_env.sh", "pyproject.toml", *(str(UI / name) for name in MANIFESTS)): + try: + source_hashes[relative] = hashlib.sha256(_read_regular(source / relative, MAX_MANIFEST)).hexdigest() + except (OSError, ValueError): + source_hashes[relative] = None + return {"schema_version": 1, "source_commit": revision, "launch_env_sha256": launch_sha, + "source_files_sha256": source_hashes, + "host": {"system": system, "machine": machine}, "checks": checks, + "prerequisites": prerequisites, + "device_qualification": "not_assessed", + "scope": "Read-only prerequisites; no graphics, camera, CAN, model inference, hardware timing or driving check"} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--font-dir", type=Path, help="complete existing font directory; never copied") + parser.add_argument("--json-output", type=Path, help="write a local JSON report") + args = parser.parse_args(argv) + report = collect(font_dir=args.font_dir) + encoded = json.dumps(report, indent=2) + "\n" + if args.json_output: + args.json_output.write_text(encoded) + else: + print(encoded, end="") + return {"pass": 0, "fail": 1, "incomplete": 2}[report["prerequisites"]] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/run_host_tests.py b/tools/ci/run_host_tests.py new file mode 100644 index 0000000000..7c8528849b --- /dev/null +++ b/tools/ci/run_host_tests.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Run host tests with disposable Params and a private messaging namespace.""" + +import ast +import os +from pathlib import Path +import platform +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[2] +RUNNER = ROOT / 'tools/test_runner.py' +UI_PYTEST_FILES = ( + 'openpilot/system/ui/lib/tests/test_text_caches.py', + 'openpilot/system/ui/widgets/tests/test_label_cache.py', + 'openpilot/selfdrive/ui/tests/test_onboarding_textures.py', +) + + +def pytest_files(root=ROOT): + """Select custom files containing pytest functions or plain Test classes.""" + files = [] + for path in sorted((root / 'openpilot/starpilot').rglob('test_*.py')): + module = ast.parse(path.read_text(), filename=str(path)) + if any((isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith('test_')) or + (isinstance(node, ast.ClassDef) and node.name.startswith('Test') and not node.bases) + for node in module.body): + files.append(path.relative_to(root).as_posix()) + return files + [path for path in UI_PYTEST_FILES if (root / path).is_file()] + + +def main(arguments=None): + arguments = sys.argv[1:] if arguments is None else arguments + pytest_mode = bool(arguments and arguments[0] == '--pytest') + if pytest_mode: + files = pytest_files() + if not files: + print('No StarPilot pytest functions were discovered.', file=sys.stderr) + return 5 + shared_memory = '/tmp' if platform.system() == 'Darwin' else '/dev/shm' + with tempfile.TemporaryDirectory(prefix='starpilot-host-params-') as params, \ + tempfile.TemporaryDirectory(prefix='msgq_starpilot-host-', dir=shared_memory) as messaging: + # Set these before imports in the child. UIState constructs Params and + # subscribers at import time; SCALE avoids querying a desktop monitor. + environment = dict(os.environ, PARAMS_ROOT=params, SCALE='1', + OPENPILOT_PREFIX=Path(messaging).name.removeprefix('msgq_')) + if pytest_mode: + environment.pop('PYTEST_ADDOPTS', None) + environment.pop('PYTEST_PLUGINS', None) + environment['PYTEST_DISABLE_PLUGIN_AUTOLOAD'] = '1' + command = ([sys.executable, '-m', 'pytest', '-q', '--import-mode=importlib', *arguments[1:], *files] if pytest_mode + else [sys.executable, str(RUNNER), *arguments]) + result = subprocess.run(command, cwd=ROOT, env=environment, check=False) + return result.returncode if result.returncode >= 0 else 128 - result.returncode + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/ci/run_local_tests.py b/tools/ci/run_local_tests.py new file mode 100755 index 0000000000..25421ff0a8 --- /dev/null +++ b/tools/ci/run_local_tests.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run pre-push regression checks in the existing isolated developer runtime.""" + +import argparse +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from tools.ci.run_predeploy import commands, run +from tools.ci.run_host_tests import UI_PYTEST_FILES + + +QUICK_TARGETS = ( + 'openpilot/common/tests/test_params.py', + 'openpilot/common/tests/test_params_registry_usage.py', + 'openpilot/selfdrive/selfdrived/tests', + 'openpilot/starpilot/tests', + 'openpilot/starpilot/aol/tests', + 'openpilot/starpilot/analytics/tests', + 'openpilot/starpilot/gps/tests', + 'opendbc_repo/opendbc/car/tests/test_gps.py', + 'openpilot/starpilot/bluetooth/tests/test_owner.py', + 'openpilot/starpilot/flm/tests/test_controller_replay.py', + 'openpilot/starpilot/controllers/tests', + 'openpilot/starpilot/car/tesla/tests', + 'openpilot/starpilot/lateral/tests', + 'openpilot/starpilot/longitudinal/tests', + 'openpilot/starpilot/speed_limits/tests/test_runtime_replay.py', + 'openpilot/starpilot/drive_state/tests', + 'openpilot/starpilot/models/tests', + 'openpilot/starpilot/software/tests/test_fast_update.py', + 'openpilot/starpilot/software/tests/test_update_process.py', + 'openpilot/starpilot/software/tests/test_history.py', + 'openpilot/starpilot/ui/tests', + 'openpilot/starpilot/galaxy/tests/test_access.py', + 'openpilot/starpilot/galaxy/tests/test_pairing_migration.py', + 'openpilot/starpilot/galaxy/tests/test_settings_defaults.py', + 'openpilot/starpilot/galaxy/tests/test_vehicle_configuration.py', + 'openpilot/starpilot/galaxy/tests/test_tesla_screen_settings.py', + 'opendbc_repo/opendbc/car/gm/tests/test_bolt_pedal.py', + 'opendbc_repo/opendbc/car/gm/tests/test_camera_acc_pedal.py', + 'opendbc_repo/opendbc/car/gm/tests/test_volt_transitions.py', + 'opendbc_repo/opendbc/car/gm/tests/test_ordinary_cc.py::TestMalibuHybridCc', + 'opendbc_repo/opendbc/safety/tests/test_gm_bolt_pedal.py', + 'opendbc_repo/opendbc/safety/tests/test_gm_camera_acc_pedal.py', + 'opendbc_repo/opendbc/safety/tests/test_gm_cc_pedal.py', + 'opendbc_repo/opendbc/safety/tests/test_gm_hybrid_cc.py', + 'opendbc_repo/opendbc/car/tesla/tests/test_screen_button.py', + 'opendbc_repo/opendbc/safety/tests/test_tesla_screen_button.py', + 'opendbc_repo/opendbc/safety/tests/test_toyota.py', + 'opendbc_repo/opendbc/car/toyota/tests/test_retrofit.py', + 'opendbc_repo/opendbc/car/honda/tests/test_stock_aol.py', + 'openpilot/selfdrive/car/tests/test_alpha_long_saved_intent.py', + 'openpilot/selfdrive/car/tests/test_bolt_cc_integration.py', + 'openpilot/selfdrive/car/tests/test_bolt_cruise_event.py', + 'openpilot/selfdrive/car/tests/test_carparams_first_publish.py', + 'openpilot/selfdrive/car/tests/test_conditional_manual_event.py', + 'openpilot/selfdrive/car/tests/test_cruise_intervals.py', + 'openpilot/selfdrive/car/tests/test_cruise_speed.py', + 'openpilot/selfdrive/car/tests/test_curve_press_event.py', + 'openpilot/selfdrive/car/tests/test_distance_personality_claim.py', + 'openpilot/selfdrive/car/tests/test_honda_aol.py', + 'openpilot/selfdrive/car/tests/test_hyundai_aol.py', + 'openpilot/selfdrive/car/tests/test_init_lifecycle.py', + 'openpilot/selfdrive/car/tests/test_lkas_long_only.py', + 'opendbc_repo/opendbc/car/gm/tests/test_suburban_gateway.py', + 'opendbc_repo/opendbc/car/gm/tests/test_volt_alternate_brake.py', + 'opendbc_repo/opendbc/car/hyundai/tests/test_ioniq6_handoff.py', + 'opendbc_repo/opendbc/car/hyundai/tests/test_ioniq6_stock_parser.py', + 'opendbc_repo/opendbc/car/hyundai/tests/test_ioniq6_engagement_messages.py', + 'opendbc_repo/opendbc/safety/tests/test_hyundai_ioniq6_long.py', +) + UI_PYTEST_FILES + + +def main(arguments=None): + parser = argparse.ArgumentParser(description=__doc__, epilog=( + 'Default: lint, packaging/CI/updater checks, startup/migration, Params, controls, models, UI, and Bolt/Ioniq regressions. ' + + 'First use prepares host dependencies; later runs reuse them. No device build, flash, push, or road qualification.')) + parser.add_argument('--full', action='store_true', help='Run the existing full Linux pre-deployment suite, including recorded routes and replay') + parser.add_argument('--download', action='store_true', help='Allow pinned recorded-route downloads with --full (replay may also fetch its logs)') + parser.add_argument('--output', type=Path, help='New directory for per-check logs; defaults to a temporary report directory') + args = parser.parse_args(arguments) + if args.download and not args.full: + parser.error('--download requires --full; the default checks do not fetch routes') + if args.full and platform.system() != 'Linux': + parser.error('--full requires Linux for native transport and map IPC; run ./test for local checks on macOS') + output = args.output.expanduser().resolve() if args.output else Path(tempfile.mkdtemp(prefix='starpilot-tests-')) / 'results' + if output.exists(): + parser.error('--output must be new so previous reports are preserved') + if output.is_relative_to(ROOT): + parser.error('--output must be outside the source checkout') + if os.environ.get('SP_HOST_RUNTIME') != '1': + options = ['--output', str(output), *(['--full'] if args.full else []), *(['--download'] if args.download else [])] + print(f'Preparing isolated host tests. Reports: {output}', flush=True) + started = time.monotonic() + result = subprocess.run([str(ROOT / 'dev'), 'python', 'tools/ci/run_local_tests.py', *options], cwd=ROOT, check=False) + print(f'Total including host preparation: {time.monotonic() - started:.1f}s', flush=True) + return result.returncode if result.returncode >= 0 else 128 - result.returncode + environment = dict(os.environ, SCALE='1', RAYLIB_BACKEND='desktop' if platform.system() == 'Darwin' else 'headless', + FUZZ_SEED='0', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1') + for key in ('FAST', 'RUN', 'SKIP', 'PYTEST_ADDOPTS', 'PYTEST_PLUGINS'): + environment.pop(key, None) + if args.full: + return subprocess.call([sys.executable, 'tools/ci/run_predeploy.py', '--output', str(output), + *(['--download'] if args.download else [])], cwd=ROOT, env=environment) + + if platform.system() == 'Darwin': + for formula in ('coreutils', 'gnu-sed'): + for prefix in ('/opt/homebrew', '/usr/local'): + directory = Path(prefix) / f'opt/{formula}/libexec/gnubin' + if directory.is_dir(): + environment['PATH'] = str(directory) + os.pathsep + environment.get('PATH', '') + break + sed = shutil.which('sed', path=environment.get('PATH')) + if sed is None or 'GNU sed' not in subprocess.run( + [sed, '--version'], env=environment, capture_output=True, text=True, check=False).stdout: + parser.error('GNU sed is required for the updater checks; install gnu-sed (brew install gnu-sed on macOS)') + if shutil.which('timeout', path=environment.get('PATH')) is None: + parser.error('GNU timeout is required for the updater checks; install coreutils (brew install coreutils on macOS)') + shared_memory = '/tmp' if platform.system() == 'Darwin' else '/dev/shm' + with tempfile.TemporaryDirectory(prefix='starpilot-test-params-') as params, \ + tempfile.TemporaryDirectory(prefix='msgq_starpilot-test-', dir=shared_memory) as messaging: + environment.update(PARAMS_ROOT=params, OPENPILOT_PREFIX=Path(messaging).name.removeprefix('msgq_')) + plan = commands(output, Path('/unused'))['source'] + aol_protocol = output / 'test_aol_protocol' + plan.extend((('aol-protocol-build', ['c++', '-std=c++17', '-I.', '-Iopenpilot', '-Iopendbc_repo', + 'openpilot/selfdrive/pandad/tests/test_aol_protocol.cc', '-o', str(aol_protocol)]), + ('aol-protocol', [str(aol_protocol)]))) + plan.append(('regressions', [sys.executable, '-m', 'pytest', '-q', '--import-mode=importlib', + '--junitxml', str(output / 'regressions.xml'), *QUICK_TARGETS])) + result = run(plan, output, environment=environment) + print(f'{"PASS" if result == 0 else "FAIL"}: local pre-push checks. Reports: {output}', flush=True) + return result + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/ci/run_predeploy.py b/tools/ci/run_predeploy.py new file mode 100755 index 0000000000..15f34bfd7b --- /dev/null +++ b/tools/ci/run_predeploy.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Run the source, host, vehicle and recorded checks used by GitHub Actions.""" + +import argparse +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +STAGES = ('build', 'source', 'host', 'native', 'vehicles', 'recorded', 'replay') + + +def commands(output, cache, download=False): + python = sys.executable + source = [('mapd-package', [python, 'tools/release/stage_mapd_provider.py', '--source', str(ROOT), '--require-tracked']), + ('lint', ['bash', 'scripts/lint/lint.sh']), + ('dependencies', [python, 'tools/vendor/check.py', '--revision', 'HEAD']), + ('schemas', [python, 'tools/ci/schema_policy.py'])] + for name, path in (('vendor', 'tools/vendor/tests'), ('packaging', 'tools/release/tests'), + ('ci-tools', 'tools/ci/tests'), ('updater', 'openpilot/system/updated/tests')): + command = [python, '-m', 'pytest', '-q', path] if name == 'packaging' else [python, '-m', 'unittest', 'discover', '-s', path, '-v'] + source.append((name, command)) + host = [('unittest', [python, 'tools/ci/run_host_tests.py', '--json-output', str(output / 'host.json')]), + ('pytest', [python, 'tools/ci/run_host_tests.py', '--pytest'])] + vehicles = [(suite, [python, 'tools/ci/run_vehicle_tests.py', '--suite', suite, '--output', str(output / suite)]) + for suite in ('interfaces', 'safety-debug', 'safety-release')] + vehicles.append(('fleet-coverage', [python, 'tools/ci/fleet_coverage.py', '--require', 'interfaces', + '--interface-coverage', str(output / 'interfaces/coverage.json'), + '--interface-results', str(output / 'interfaces/results.json'), + '--output', str(output / 'interfaces/fleet-coverage.json')])) + recorded = [(f'recorded-{variant}', [python, 'tools/ci/run_recorded_vehicle_tests.py', '--variant', variant, + '--cache', str(cache), '--output', str(output / f'recorded-{variant}'), *(['--download'] if download else [])]) + for variant in ('debug', 'release')] + native = [(name, [str(ROOT / 'openpilot/selfdrive/pandad/tests' / name)]) + for name in ('test_panda_usb', 'test_pandad_canprotocol', 'test_aol_protocol', 'test_aol_wire')] + if platform.system() == 'Linux': + native.extend((name, [str(ROOT / 'openpilot/selfdrive/pandad/tests' / name)]) + for name in ('test_producer_timing', 'test_producer_sink')) + native.append(('map-ipc', ['go', '-C', str(ROOT / 'mapd_repo'), 'test', '-mod=readonly', '-race', + '-run', '^Test(MsgqNativeLayout|Shadow(GoPythonIPC|ExecutableHostGpsIPC))$', '-count=1', '-v'])) + return {'build': [('native-build', [python, '-m', 'SCons', '-j4'])], + 'source': source, 'host': host, 'native': native, 'vehicles': vehicles, 'recorded': recorded, + 'replay': [('process-replay', [python, 'openpilot/selfdrive/test/process_replay/test_processes.py', + '-j', str(os.cpu_count() or 1), '--output', str(output / 'process-replay')])]} + + +def run(plan, output, *, environment=None): + output.mkdir(parents=True, exist_ok=False) + records = [] + for name, command in plan: + print(f'{name}: starting', flush=True) + started = time.monotonic() + with (output / f'{name}.log').open('w') as log: + try: + result = subprocess.run(command, cwd=ROOT, env=environment, stdout=log, stderr=subprocess.STDOUT, check=False) + code = result.returncode + except OSError as error: + log.write(str(error) + '\n') + code = 127 + records.append({'check': name, 'command': command, 'returncode': code, 'seconds': round(time.monotonic() - started, 3)}) + (output / 'results.json').write_text(json.dumps(records, indent=2) + '\n') + print(f'{name}: {"passed" if code == 0 else "FAILED"} ({output / (name + ".log")})', flush=True) + if code: + return code if code > 0 else 128 - code + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True, help='New directory for logs and per-check results') + parser.add_argument('--cache', type=Path, default=Path.home() / '.cache/starpilot/recorded-fixtures') + parser.add_argument('--stage', choices=STAGES, action='append', help='Run selected stages; default runs all stages') + parser.add_argument('--download', action='store_true', help='Allow the pinned public recording downloads') + args = parser.parse_args() + output, cache = args.output.expanduser().resolve(), args.cache.expanduser().resolve() + if output.exists(): + parser.error('--output must be new so earlier evidence is preserved') + if output.is_relative_to(ROOT): + parser.error('--output must be outside the source tree') + selected = set(args.stage or STAGES) + if "native" in selected and platform.system() != "Linux": + parser.error("the native predeploy stage requires Linux; on macOS build host dependencies, then select --stage source --stage host") + plan = commands(output, cache, args.download) + environment = dict(os.environ, SCALE='1', RAYLIB_BACKEND='headless', FUZZ_SEED='0', GOTOOLCHAIN='local', + STARPILOT_SHADOW_IPC_REQUIRED='1', STARPILOT_SHADOW_EXEC_REQUIRED='1', STARPILOT_PYTHON=sys.executable) + # An inherited fast-lint flag must not silently remove type/spelling checks. + environment.pop('FAST', None) + environment['PATH'] = str(Path(sys.executable).parent) + os.pathsep + environment.get('PATH', '') + return run([entry for stage in STAGES if stage in selected for entry in plan[stage]], output, environment=environment) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/ci/run_safety_qualification.py b/tools/ci/run_safety_qualification.py new file mode 100644 index 0000000000..c17a8c5e52 --- /dev/null +++ b/tools/ci/run_safety_qualification.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python3 +"""Run isolated, source-pinned safety qualification gates without nested uv setup.""" + +import argparse +import codecs +from contextlib import ExitStack +import io +import hashlib +import json +import math +import os +from pathlib import Path +import re +import selectors +import shutil +import signal +import stat +import subprocess +import sys +import time + + +ROOT = Path(__file__).resolve().parents[2] +SAFETY_TESTS = ROOT / "opendbc_repo/opendbc/safety/tests" +SAFETY_SOURCE_ROOTS = (SAFETY_TESTS.parent, ROOT / "opendbc_repo/opendbc/bluepilot_lateral/safety") +MAX_MUTATION_ARTIFACT_BYTES = 256 * 1024 * 1024 +SOURCE_PREFIXES = ( + "opendbc_repo/", "panda/", "openpilot/", "msgq_repo/", "rednose_repo/", + "teleoprtc_repo/", "tinygrad_repo/", "jetlink_repo/", "tools/", "scripts/", +) +SOURCE_EXACT = {".github/workflows/safety.yaml", "pyproject.toml", "uv.lock", "SConstruct", + "panda/SConscript", "panda/SConstruct", "opendbc_repo/SConscript", "opendbc_repo/pyproject.toml", + "openpilot/cereal/SConscript", "openpilot/common/SConscript", "panda/tests/libpanda/SConscript", + "panda/tests/misra/coverage_table", + "opendbc_repo/opendbc/safety/tests/misra/coverage_table"} +SOURCE_SUFFIXES = {".c", ".h", ".cc", ".cpp", ".py", ".sh", ".capnp", ".dbc", ".toml", ".yaml", ".txt", ".json", + ".s", ".S", ".ld", ".mk", ".pyx", ".pxd", ".hpp"} +SOURCE_NAMES = {"SConscript", "SConstruct", "Makefile", "coverage_table"} +# These four tracked outputs are declared by the longitudinal MPC SConscript. +# They may regenerate only during the catalog import prerequisite build; their +# generators, templates, schemas, build definitions, and all safety inputs stay pinned. +MPC_PREREQUISITE_OUTPUTS = { + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/acados_ocp_long.json": "long_mpc.py (SConscript generated_long)", + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/Makefile": "long_mpc.py (SConscript generated_long)", + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_solver_long.c": "long_mpc.py (SConscript generated_long)", + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.c": "Cython (SConscript libacados_ocp_solver_c)", +} + + +def prerequisite_transition(before, after, name): + """Pin real inputs before build and admit only the four declared output rewrites.""" + allowed = MPC_PREREQUISITE_OUTPUTS if name in ("coverage", "mutation-full") else {} + if before.keys() != after.keys(): + return False + return all(before[path] == after[path] or path in allowed for path in before) + + +UNITTEST_ACCOUNTING = """ +def test_ids(group): + for item in group: + if isinstance(item, unittest.TestSuite): + yield from test_ids(item) + else: + yield item.id() +targets = list(test_ids(suite)) +print(f'Safety qualification collected {len(targets)} tests', flush=True) +result = unittest.TextTestRunner(verbosity=2).run(suite) +suppressed = set() +for test, _reason in result.skipped: + skipped_id = test.id() + match = re.fullmatch(r'setUp(?:Class|Module) \\(([^)]+)\\)', skipped_id) + if match: + matched = {index for index, target in enumerate(targets) if target.startswith(match.group(1) + '.')} + if not matched: + raise RuntimeError(f'unmatched skip holder: {skipped_id}') + suppressed.update(matched) + elif skipped_id not in targets: + raise RuntimeError(f'unexpected skip holder: {skipped_id}') +print(f'Safety qualification suppressed {len(suppressed)} methods', flush=True) +sys.exit(0 if result.wasSuccessful() else 1) +""" + + +def source_identity(): + listed = subprocess.check_output(["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], cwd=ROOT) + names = (Path(name.decode()) for name in listed.split(b"\0") if name) + files = sorted(name for name in names if (name.as_posix() in SOURCE_EXACT or + (name.as_posix().startswith(SOURCE_PREFIXES) and (name.suffix in SOURCE_SUFFIXES or name.name in SOURCE_NAMES))) and + "obj" not in name.parts and "gen" not in name.parts and (ROOT / name).is_file()) + return {path.as_posix(): hashlib.sha256((ROOT / path).read_bytes()).hexdigest() for path in files} + + +def prerequisite_artifact_paths(prerequisites, name): + paths = list(prerequisites) + if name in ("coverage", "mutation-full"): + generated = "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/" + extension = ".dylib" if sys.platform == "darwin" else ".so" + paths.append(generated + "libacados_ocp_solver_long" + extension) + if sys.platform != "darwin": + paths.extend(generated + lib for lib in ("libacados.so", "libblasfeo.so", "libhpipm.so", "libqpOASES_e.so.3.1")) + return paths + + +def prerequisite_artifact_identity(paths): + return {path: hashlib.sha256((ROOT / path).read_bytes()).hexdigest() for path in paths} + + +def generated_identity(): + files = sorted(path for pattern in ("openpilot/cereal/gen/**/*", "opendbc_repo/opendbc/can/*.so", + "panda/tests/libpanda/*.so", "panda/board/obj/**/*.elf") + for path in ROOT.glob(pattern) if path.is_file()) + return {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in files} + + +def source_head(): + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def parse_unittest_summary(output): + matches = list(re.finditer(r"Ran (\d+) tests? in[^\n]*\n\s*\n(OK|FAILED)(?: \([^\n]*\))?", output)) + match = matches[-1] if matches else None + status = match.group() if match else "" + skipped = re.search(r"skipped=(\d+)", status) + return {"ran": int(match.group(1)) if match else None, + "skipped": int(skipped.group(1)) if skipped else 0, + "status": match.group(2) if match else "missing", + "collected": int(found.group(1)) if (found := re.search(r"^Safety qualification collected (\d+) tests$", output, re.MULTILINE)) else None, + "suppressed": int(found.group(1)) if (found := re.search(r"^Safety qualification suppressed (\d+) methods$", output, re.MULTILINE)) else None} + + +def complete_unittest(summary): + return (summary["status"] == "OK" and summary["collected"] is not None and + summary["suppressed"] is not None and summary["ran"] + summary["suppressed"] == summary["collected"] and + summary["ran"] > summary["skipped"]) + + +def parse_mutation_summary(output): + plain = re.sub(r"\x1b\[[0-9;]*m", "", output) + summary = {"candidates": int(match.group(1)) if (match := re.search(r"Found (\d+) unique candidates", plain)) else None} + for label in ("total", "killed", "survived", "infra_error", "pruned_build_incompatible"): + match = re.search(rf"^ {label}: (\d+)$", plain, re.MULTILINE) + summary[label] = int(match.group(1)) if match else None + match = re.search(r"^ proven_equivalent: (\d+)$", plain, re.MULTILINE) + summary["proven_equivalent"] = int(match.group(1)) if match else 0 + return summary + + +def complete_mutation_artifact(path, summary, *, digest_out=None): + """A recorded candidate set must agree with the subprocess summary.""" + try: + flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(path, flags) + try: + metadata = os.fstat(fd) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_MUTATION_ARTIFACT_BYTES: + return False + with os.fdopen(fd, "rb", closefd=False) as stream: + raw = stream.read(MAX_MUTATION_ARTIFACT_BYTES + 1) + if len(raw) > MAX_MUTATION_ARTIFACT_BYTES: + return False + finally: + os.close(fd) + artifact = json.loads(raw) + results = artifact["results"] + pruned = artifact["pruned_build_incompatible"] + equivalent = artifact.get("proven_equivalent", []) + version = artifact["schema_version"] + target_sets = artifact["target_sets"] + sources = artifact["source_sha256"] + def digest(value): + return type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value) is not None + if (type(version) is not int or version not in (1, 2) or + (version == 1 and bool(equivalent)) or + (version == 2 and "proven_equivalent" not in artifact) or + type(equivalent) is not list or len(equivalent) != summary.get("proven_equivalent", 0) or + type(artifact["discovered"]) is not int or artifact["discovered"] != summary["candidates"] or + type(results) is not list or type(pruned) is not list or type(target_sets) is not list or + len(pruned) != summary["pruned_build_incompatible"] or + len(results) != summary["total"] or + not digest(artifact["preprocessed_source_sha256"]) or + artifact["safety_input_sha256"] != hashlib.sha256((SAFETY_TESTS / "libsafety/safety.c").read_bytes()).hexdigest() or + artifact["mutation_runner_sha256"] != hashlib.sha256((SAFETY_TESTS / "mutation.py").read_bytes()).hexdigest() or + type(artifact["baseline_sec"]) not in (int, float) or not math.isfinite(artifact["baseline_sec"]) or artifact["baseline_sec"] < 0 or + type(sources) is not dict or not sources or + any(type(source) is not str or not digest(source_hash) for source, source_hash in sources.items()) or + any(type(item) is not list or not item or any(type(target) is not str or not target for target in item) + for item in target_sets)): + return False + safety_roots = tuple(source_root.resolve() for source_root in SAFETY_SOURCE_ROOTS) + for source, claimed_hash in sources.items(): + source_path = (ROOT / "opendbc_repo" / source).resolve() + if not any(source_path.is_relative_to(source_root) for source_root in safety_roots) or not source_path.is_file() or \ + hashlib.sha256(source_path.read_bytes()).hexdigest() != claimed_hash: + return False + ids = [item["site_id"] for item in results] + pruned_ids = [item["site_id"] for item in pruned] + equivalent_ids = [item["site_id"] for item in equivalent] + if (any(type(site_id) is not int for site_id in ids + pruned_ids + equivalent_ids) or + len(ids) != len(set(ids)) or len(pruned_ids) != len(set(pruned_ids)) or + len(equivalent_ids) != len(set(equivalent_ids)) or set(equivalent_ids) & set(ids + pruned_ids) or + set(ids) & set(pruned_ids) or len(ids) + len(pruned_ids) + len(equivalent_ids) != artifact["discovered"] or + set(ids + pruned_ids + equivalent_ids) != set(range(artifact["discovered"]))): + return False + for label in ("killed", "survived", "infra_error"): + if sum(item["outcome"] == label for item in results) != summary[label]: + return False + valid = (all(type(item["selected_test_set"]) is int and + 0 <= item["selected_test_set"] < len(target_sets) and + type(item["selected_test_count"]) is int and + item["selected_test_count"] == len(target_sets[item["selected_test_set"]]) and + item["source"] in sources and type(item["line"]) is int and item["line"] > 0 and + item["outcome"] in ("killed", "survived", "infra_error") and + (item["outcome"] == "infra_error" or (item["failure_kind"] is None and item["exit_code"] is None)) and + all(type(item[key]) is str and item[key] for key in ("mutator", "original_op")) and + type(item["mutated_op"]) is str and + (bool(item["mutated_op"]) or (item["mutator"] == "remove_negation" and item["original_op"] == "!")) and + all(type(item[key]) is str for key in ("details", "stderr_tail", "stdout_tail", "unittest_output_tail")) and + (item["failure_kind"] is None or type(item["failure_kind"]) is str) and + (item["exit_code"] is None or type(item["exit_code"]) is int) + for item in results) and + all(item["source"] in sources and type(item["line"]) is int and item["line"] > 0 + for item in pruned)) + if valid and version == 2: + # Independent rediscovery binds ID/offset/context/source/compiler/value, + # including an empty claimed partition. Never reinterpret v1 survivors. + proof = subprocess.run([sys.executable, str(SAFETY_TESTS / "mutation.py"), + "--verify-equivalence-artifact", str(path)], + cwd=ROOT / "opendbc_repo", capture_output=True, text=True, timeout=60) + valid = proof.returncode == 0 and proof.stdout.strip() == hashlib.sha256(raw).hexdigest() + if valid and digest_out is not None: + digest_out.append(hashlib.sha256(raw).hexdigest()) + return valid + except (OSError, ValueError, TypeError, KeyError, IndexError, RecursionError, OverflowError, subprocess.SubprocessError): + return False + + +def check_import_origins(python, env, output): + script = """ +import json +from pathlib import Path +import importlib.util +import opendbc, openpilot, msgq +import opendbc.can.parser, opendbc.can.packer, opendbc.can.dbc +from openpilot.cereal import log as cereal_log +import openpilot.cereal.messaging +import msgq.ipc_pyx +from opendbc.safety.tests.libsafety import libsafety_py +print(json.dumps({name: str(Path(module.__file__).resolve()) for name, module in + [('opendbc', opendbc), ('openpilot', openpilot), + ('msgq', msgq), ('libsafety_py', libsafety_py), + ('can_parser', opendbc.can.parser), ('can_packer', opendbc.can.packer), + ('can_dbc', opendbc.can.dbc), ('cereal_log', cereal_log), + ('cereal_messaging', openpilot.cereal.messaging), ('msgq_ipc', msgq.ipc_pyx)]} | + {'panda': importlib.util.find_spec('panda').origin})) +""" + result = subprocess.run([python, "-c", script], cwd=output.parent, env=env, capture_output=True, text=True) + output.write_text(result.stdout + result.stderr) + if result.returncode: + return False + try: + origins = json.loads(result.stdout) + except json.JSONDecodeError: + return False + expected = {"opendbc": ROOT / "opendbc_repo", "panda": ROOT / "panda", "msgq": ROOT / "msgq_repo", + "openpilot": ROOT / "openpilot", "libsafety_py": ROOT / "opendbc_repo", + "can_parser": ROOT / "opendbc_repo", "can_packer": ROOT / "opendbc_repo", + "can_dbc": ROOT / "opendbc_repo", "cereal_log": ROOT / "openpilot/cereal", + "cereal_messaging": ROOT / "openpilot/cereal", "msgq_ipc": ROOT / "msgq_repo"} + return all(isinstance(origins.get(name), str) and Path(origins[name]).is_relative_to(path) + for name, path in expected.items()) + + +def imported_module_hashes(output): + origins = json.loads(output.read_text().splitlines()[0]) + return {name: {"path": path, "sha256": hashlib.sha256(Path(path).read_bytes()).hexdigest()} + for name, path in origins.items()} + + +class Gate: + def __init__(self, name, output, python): + self.name = name + self.output = output + self.python = str(python) + self.env = dict(os.environ) + self.env.update({"PYTHONPATH": os.pathsep.join(str(ROOT / part) for part in + ("", "opendbc_repo", "msgq_repo", "rednose_repo", "teleoprtc_repo", "tinygrad_repo", "jetlink_repo")), + "PARAMS_ROOT": str(output / "params"), "OPENPILOT_PREFIX": f"safety_qual_{os.getpid()}", + "PYTHONDONTWRITEBYTECODE": "1", "FUZZ_SEED": "0", "SCALE": "1", "PWD": str(ROOT)}) + for inherited_key in ("HITL", "RELEASE", "CERT", "SKIP_BUILD", "SKIP_TABLES_DIFF", + "LIBSAFETY_PREBUILT", "GCOV_PREFIX", "GCOV_PREFIX_STRIP", "CC", "CXX", + "CFLAGS", "CXXFLAGS", "LDFLAGS", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"): + self.env.pop(inherited_key, None) + self.env.pop("SAFETY_COVERAGE", None) + if name == "coverage": + self.env["SAFETY_COVERAGE"] = "1" + self.env["PATH"] = str(Path(self.python).parent) + os.pathsep + self.env.get("PATH", "") + (output / "params").mkdir() + self.commands = [] + self.artifacts = {} + + @staticmethod + def stream_output(proc, log, timeout): + assert proc.stdout is not None + decoder = io.IncrementalNewlineDecoder( + codecs.getincrementaldecoder(proc.stdout.encoding)(errors=proc.stdout.errors), translate=True) + chunks = [] + deadline = time.monotonic() + timeout + timed_out = False + + def emit(text): + chunks.append(text) + log.write(text) + log.flush() + sys.stdout.write(text) + sys.stdout.flush() + + try: + with selectors.DefaultSelector() as selector: + selector.register(proc.stdout, selectors.EVENT_READ) + while selector.get_map(): + remaining = deadline - time.monotonic() + if not timed_out and remaining <= 0: + os.killpg(proc.pid, signal.SIGKILL) + timed_out = True + for key, _ in selector.select(None if timed_out else max(0, remaining)): + data = os.read(key.fd, 65536) + emit(decoder.decode(data, final=not data)) + if not data: + selector.unregister(key.fileobj) + try: + proc.wait(timeout=None if timed_out else max(0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + timed_out = True + except BaseException: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + raise + finally: + proc.stdout.close() + if timed_out: + emit("\nTimed out; process group killed\n") + return (124 if timed_out else proc.returncode), "".join(chunks) + + def run(self, label, argv, *, cwd=ROOT, timeout=3600): + start = time.monotonic() + log = self.output / f"{label}.log" + try: + with ExitStack() as stack: + stream_log = (stack.enter_context(log.open("w")) + if label == "mutation" and self.name in ("mutation-list", "mutation-full") else None) + proc = subprocess.Popen(argv, cwd=cwd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, start_new_session=True) + if stream_log is not None: + code, content = self.stream_output(proc, stream_log, timeout) + else: + try: + content, _ = proc.communicate(timeout=timeout) + code = proc.returncode + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + content, _ = proc.communicate() + code = 124 + content += "\nTimed out; process group killed\n" + except OSError as exc: + code, content = 127, f"{type(exc).__name__}: {exc}\n" + log.write_text(content) + self.commands.append({"label": label, "argv": [str(arg) for arg in argv], "cwd": str(cwd), + "exit_code": code, "seconds": round(time.monotonic() - start, 3), "log": log.name}) + return code, content + + def hash_artifact(self, path): + path = Path(path) + if path.is_file(): + self.artifacts[path.relative_to(ROOT).as_posix() if path.is_relative_to(ROOT) else path.name] = hashlib.sha256(path.read_bytes()).hexdigest() + + def cppcheck_dir(self): + code, output = self.run("cppcheck-location", [self.python, "-c", "import cppcheck; print(cppcheck.DIR)"]) + if code: + return None + location = Path(output.strip()) + version, text = self.run("cppcheck-version", [str(location / "cppcheck"), "--version"]) + if version or re.fullmatch(r"Cppcheck 2\.21(?:\.0)?", text.strip()) is None: + return None + return location + + def misra(self, panda): + directory = self.cppcheck_dir() + if directory is None: + return False + misra = directory / "addons/misra.py" + check_dir = ROOT / ("panda/tests/misra" if panda else "opendbc_repo/opendbc/safety/tests/misra") + table, text = self.run("misra-table", [self.python, str(misra), "-generate-table"]) + if table or text != (check_dir / "coverage_table").read_text(): + return False + include = ROOT / ("panda" if panda else "opendbc_repo") + compiler = "arm-none-eabi-gcc" if panda else "cc" + status, compiler_include = self.run("compiler-include", [compiler, "-print-file-name=include"]) + if status: + return False + common = [str(directory / "cppcheck"), "--inline-suppr", "-I", str(include), + "-I", compiler_include.strip(), "--suppressions-list=" + str(check_dir / "suppressions.txt"), + "--error-exitcode=2", "--check-level=exhaustive", "--safety", "--platform=arm32-wchar_t4", + "--std=c11", "--enable=all", "--addon=misra"] + if panda: + common += ["--disable=unusedFunction", "--suppress=*:*inc/*", "--suppress=*:*include/*", + "-D__GNUC__=9", "-UCMSIS_NVIC_VIRTUAL", "-UCMSIS_VECTAB_VIRTUAL", "-UBOOTSTUB", + "-DSTM32H7", "-DSTM32H725xx", "-I", str(ROOT / "opendbc_repo"), + "-I", str(ROOT / "panda/board/stm32h7/inc"), + str(ROOT / "panda/board/main.c")] + else: + common += ["--enable=unusedFunction", "--suppress=missingIncludeSystem", "--suppress=*:*include/*", + "-D__GNUC__=9", "-D__has_include_next(x)=0", + str(ROOT / "opendbc_repo/opendbc/safety/tests/misra/main.c")] + outcomes = {} + for variant in ("debug", "release"): + argv = [*common[:-1], "--checkers-report=" + str(self.output / f"checkers-{variant}.txt")] + if variant == "debug": + argv.append("-DALLOW_DEBUG") + argv.append(common[-1]) + code, text = self.run(f"misra-{variant}", argv, timeout=5400) + outcomes[variant] = code == 0 and re.search(r"misra violation|error|style: ", text) is None + self.summary = {"variants": outcomes} + return all(outcomes.values()) + + def execute(self): + if self.name == "coverage": + version, _ = self.run("gcovr-version", ["gcovr", "--version"]) + if version: + return False + build_code = "from opendbc.safety.tests.libsafety import libsafety_py; print(libsafety_py._build_libsafety())" + built, text = self.run("build-libsafety", [self.python, "-c", build_code]) + if built: + return False + native = self.output / "libsafety-debug.so" + shutil.copy2(Path(text.strip().splitlines()[-1]), native) + self.hash_artifact(native) + # The caller supplies a disposable checkout. Only generated coverage files are cleared. + for path in (SAFETY_TESTS / "libsafety").glob("*.gcda"): + path.unlink() + suite_script = """import re, sys, unittest +from opendbc.safety.tests.libsafety import libsafety_py +libsafety_py.load(sys.argv[1]) +suite = unittest.TestLoader().discover('.', pattern='test_*.py') +suite.addTests(unittest.TestLoader().loadTestsFromName('opendbc.bluepilot_lateral.tests.test_angle_native')) +""" + UNITTEST_ACCOUNTING + code, text = self.run("unittest", [self.python, "-c", suite_script, str(native)], cwd=SAFETY_TESTS) + summary = parse_unittest_summary(text) + self.summary = summary + if sys.platform == "darwin": + found, llvm_cov = self.run("llvm-cov-path", ["xcrun", "--find", "llvm-cov"]) + if found: + return False + gcov_tool = llvm_cov.strip() + " gcov" + else: + gcov_tool = "gcov" + coverage_argv = ["gcovr", "-r", str(SAFETY_TESTS.parent), "--gcov-executable", gcov_tool, + "-d", "--fail-under-line=100", "-e", "^libsafety"] + for source_root in SAFETY_SOURCE_ROOTS: + coverage_argv += ["--filter", re.escape(source_root.resolve().as_posix() + "/")] + gcovr, _ = self.run("coverage", coverage_argv, cwd=SAFETY_TESTS) + return code == 0 and complete_unittest(summary) and gcovr == 0 + if self.name in ("opendbc-misra", "panda-misra"): + if self.name == "panda-misra": + built, _ = self.run("build-h7", ["scons", "-j4", "panda/board/obj/panda_h7/main.elf", + "panda/board/obj/panda_h7/bootstub.elf"], timeout=3600) + if built: + return False + self.hash_artifact(ROOT / "panda/board/obj/panda_h7/main.elf") + self.hash_artifact(ROOT / "panda/board/obj/panda_h7/bootstub.elf") + return self.misra(self.name == "panda-misra") + if self.name == "panda-host": + built, _ = self.run("build", ["scons", "-j4", "panda/tests/libpanda/libpanda.so", + "panda/board/obj/panda_h7/main.elf", "panda/board/obj/panda_h7/bootstub.elf"], timeout=3600) + if built: + return False + self.hash_artifact(ROOT / "panda/tests/libpanda/libpanda.so") + self.hash_artifact(ROOT / "panda/board/obj/panda_h7/main.elf") + self.hash_artifact(ROOT / "panda/board/obj/panda_h7/bootstub.elf") + suite_script = """import re, sys, unittest +suite = unittest.TestLoader().discover('panda/tests', pattern='test_*.py') +""" + UNITTEST_ACCOUNTING + code, text = self.run("unittest", [self.python, "-W", "error", "-c", suite_script], timeout=5400) + self.summary = parse_unittest_summary(text) + return code == 0 and complete_unittest(self.summary) + if self.name in ("mutation-list", "mutation-full"): + argv = [self.python, str(SAFETY_TESTS / "mutation.py")] + artifact = self.output / "mutation-results.json" + if self.name == "mutation-full" and os.path.lexists(artifact): + self.summary = {"artifact_valid": False, "error": "mutation result path already exists"} + return False + argv += ["--list-only"] if self.name == "mutation-list" else ["-j", "4", "--results-json", str(artifact)] + code, text = self.run("mutation", argv, cwd=ROOT / "opendbc_repo", timeout=21600) + self.summary = parse_mutation_summary(text) + if self.name == "mutation-full": + artifact_digest = [] + artifact_valid = complete_mutation_artifact(artifact, self.summary, digest_out=artifact_digest) + self.summary["artifact_valid"] = artifact_valid + if artifact_valid: + self.artifacts[artifact.name] = artifact_digest[0] + counts = self.summary + reconciled = (all(counts[key] is not None for key in ("candidates", "pruned_build_incompatible", "total", "killed", "survived", "infra_error")) and + counts["candidates"] - counts["pruned_build_incompatible"] - counts["proven_equivalent"] == counts["total"] == + counts["killed"] + counts["survived"] + counts["infra_error"]) + return code == 0 and reconciled and artifact_valid and counts["survived"] == 0 and counts["infra_error"] == 0 + return code == 0 and self.summary["candidates"] is not None and self.summary["candidates"] > 0 + raise ValueError(self.name) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gate", required=True, choices=("coverage", "opendbc-misra", "panda-misra", "panda-host", "mutation-list", "mutation-full")) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + output = args.output.resolve() + if output.exists(): + parser.error("Output directory must be new") + output.mkdir(parents=True) + python = ROOT / ".venv/bin/python" + gate = None + before = None + after = None + bootstrap = None + bootstrap_ok = False + prerequisite_outputs = {} + prerequisite_artifacts = {} + prerequisite_artifacts_after = {} + head_before = None + head_after = None + origins_ok = False + imported_before = {} + imported_after = {} + generated_before = {} + passed = False + error = None + try: + if not python.is_file(): + raise RuntimeError("root locked environment is missing") + gate = Gate(args.gate, output, python) + head_before = source_head() + before = source_identity() + (output / "source-before.json").write_text(json.dumps(before, indent=2) + "\n") + generated_before = generated_identity() + (output / "generated-before.json").write_text(json.dumps(generated_before, indent=2) + "\n") + extension = ".dylib" if sys.platform == "darwin" else ".so" + prerequisite = f"openpilot/common/libparams_c{extension}" + prerequisites = [prerequisite, "msgq_repo/msgq/ipc_pyx.so"] + if args.gate in ("coverage", "mutation-full"): + # The full catalog includes actual Card consumers importing the longitudinal planner. + prerequisites.append("openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so") + built, _ = gate.run("build-imports", ["scons", "--minimal", "-j4", *prerequisites], timeout=3600) + if built: + raise RuntimeError("native host import prerequisites failed") + bootstrap = source_identity() + (output / "source-after-prerequisites.json").write_text(json.dumps(bootstrap, indent=2) + "\n") + bootstrap_ok = head_before == source_head() and prerequisite_transition(before, bootstrap, args.gate) + if args.gate in ("coverage", "mutation-full"): + for path, producer in MPC_PREREQUISITE_OUTPUTS.items(): + if path not in before or path not in bootstrap: + raise RuntimeError(f"declared MPC prerequisite output missing: {path}") + prerequisite_outputs[path] = {"producer": producer, "before_sha256": before[path], "built_sha256": bootstrap[path]} + gate.hash_artifact(ROOT / path) + (output / "prerequisite-outputs.json").write_text(json.dumps(prerequisite_outputs, indent=2) + "\n") + if not bootstrap_ok: + raise RuntimeError("source inputs or HEAD changed during native import prerequisite build") + bound_artifacts = prerequisite_artifact_paths(prerequisites, args.gate) + for artifact in bound_artifacts: + gate.hash_artifact(ROOT / artifact) + prerequisite_artifacts = prerequisite_artifact_identity(bound_artifacts) + origins_ok = check_import_origins(gate.python, gate.env, output / "imports.log") + if origins_ok: + imported_before = imported_module_hashes(output / "imports.log") + version_codes = [gate.run("python-version", [gate.python, "--version"])[0], + gate.run("compiler-version", ["cc", "--version"])[0], + gate.run("scons-version", ["scons", "--version"])[0]] + if args.gate in ("panda-misra", "panda-host"): + version_codes.append(gate.run("arm-compiler-version", ["arm-none-eabi-gcc", "--version"])[0]) + passed = origins_ok and all(code == 0 for code in version_codes) and gate.execute() + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + try: + after = source_identity() + head_after = source_head() + (output / "source-after.json").write_text(json.dumps(after, indent=2) + "\n") + except Exception as exc: + error = f"source finalization failed: {type(exc).__name__}: {exc}" + try: + if prerequisite_artifacts: + prerequisite_artifacts_after = prerequisite_artifact_identity(prerequisite_artifacts) + except Exception as exc: + error = f"prerequisite artifact finalization failed: {type(exc).__name__}: {exc}" + generated = {} + try: + generated = generated_identity() + (output / "generated-inputs.json").write_text(json.dumps(generated, indent=2) + "\n") + except Exception as exc: + error = f"generated input finalization failed: {type(exc).__name__}: {exc}" + try: + imported_after = {name: {"path": item["path"], "sha256": hashlib.sha256(Path(item["path"]).read_bytes()).hexdigest()} + for name, item in imported_before.items()} + except Exception as exc: + error = f"imported module finalization failed: {type(exc).__name__}: {exc}" + report = {"gate": args.gate, "head_before": head_before, "head_after": head_after, + "source_unchanged": bootstrap_ok and bootstrap is not None and bootstrap == after, + "prerequisite_inputs_unchanged": bootstrap_ok, "prerequisite_outputs": prerequisite_outputs, + "prerequisite_artifacts_before": prerequisite_artifacts, "prerequisite_artifacts_after": prerequisite_artifacts_after, + "prerequisite_artifacts_unchanged": bool(prerequisite_artifacts) and prerequisite_artifacts == prerequisite_artifacts_after, + "import_origins_ok": origins_ok, + "commands": gate.commands if gate else [], "artifacts": gate.artifacts if gate else {}, + "generated_before": len(generated_before), "generated_after": len(generated), + "imported_modules": imported_before, + "imported_modules_unchanged": imported_before == imported_after, + "summary": getattr(gate, "summary", {}), "error": error, + "passed": (passed and error is None and bool(prerequisite_artifacts) and prerequisite_artifacts == prerequisite_artifacts_after and + bootstrap_ok and bootstrap is not None and bootstrap == after and head_before == head_after and imported_before == imported_after)} + (output / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({key: report[key] for key in ("gate", "passed", "summary", "source_unchanged", "import_origins_ok")})) + return int(not report["passed"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/run_vehicle_tests.py b/tools/ci/run_vehicle_tests.py new file mode 100644 index 0000000000..f90aaf3aaf --- /dev/null +++ b/tools/ci/run_vehicle_tests.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Run explicitly scoped vehicle suites with collection and native-build evidence.""" +import argparse +import ast +from collections import Counter +import hashlib +import importlib.machinery +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +import traceback +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from tools import test_runner + +RELEASE_TARGETS = ( + "opendbc_repo/opendbc/safety/tests/test_hyundai_blended.py", + "opendbc_repo/opendbc/car/hyundai/tests/test_palisade_2023.py", + "openpilot/starpilot/longitudinal/tests/test_output_max.py", + "openpilot/starpilot/ui/tests/test_output_max_feature.py", + "openpilot/starpilot/tests/test_output_max_namespace.py", + "opendbc_repo/opendbc/car/gm/tests/test_conventional_pedal_transitions.py", + "opendbc_repo/opendbc/car/gm/tests/test_conventional_pedal_disabled.py", + "opendbc_repo/opendbc/safety/tests/test_gm_cc_pedal_disabled.py", + "opendbc_repo/opendbc/car/gm/tests/test_silverado_cc_pedal.py", + "opendbc_repo/opendbc/safety/tests/test_gm_silverado_pedal.py", + "opendbc_repo/opendbc/safety/tests/test_gm_cc_pedal.py", + "opendbc_repo/opendbc/car/gm/tests/test_conventional_pedal.py", + "openpilot/starpilot/tests/test_gm_conventional_pedal_startup.py", + "tools/ci/test_release_safety.py", + "opendbc_repo/opendbc/safety/tests/test_defaults.py::TestNoOutput", + "opendbc_repo/opendbc/safety/tests/test_defaults.py::TestSilent", + "opendbc_repo/opendbc/safety/tests/test_tesla.py::TestTeslaStockSafety", + "opendbc_repo/opendbc/safety/tests/test_tesla_hw1.py", + "opendbc_repo/opendbc/safety/tests/test_volvo_c1.py", + "opendbc_repo/opendbc/safety/tests/test_tesla.py::TestTeslaFSD14StockSafety", + "opendbc_repo/opendbc/safety/tests/test_mazda.py::TestMazdaSafety", + "opendbc_repo/opendbc/safety/tests/test_toyota.py::TestToyotaStockLongitudinalTorque", + "opendbc_repo/opendbc/safety/tests/test_toyota_auto_hold.py", + "opendbc_repo/opendbc/safety/tests/test_gm.py::TestGmCameraSafety", + "opendbc_repo/opendbc/safety/tests/test_gm.py::TestGmCameraEVSafety", + "opendbc_repo/opendbc/safety/tests/test_gm_bolt_pedal.py", + "opendbc_repo/opendbc/safety/tests/test_gm_bolt_cc.py", + "openpilot/starpilot/tests/test_bolt_disable_active_pedal.py::TestBoltDisableActivePedal", + "opendbc_repo/opendbc/car/gm/tests/test_bolt_acc_cc.py", + "opendbc_repo/opendbc/car/gm/tests/test_bolt_factory_acc.py", + "opendbc_repo/opendbc/safety/tests/test_gm_bolt_factory_stock.py", + "opendbc_repo/opendbc/safety/tests/test_gm_bolt_euv.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_cc.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_cc_gas.py", + "opendbc_repo/opendbc/safety/tests/test_gm_aol.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ascm_intercept.py", + "opendbc_repo/opendbc/safety/tests/test_gm_sdgm_stock.py", + "opendbc_repo/opendbc/safety/tests/test_gm_cc_gateway_stock.py", + "opendbc_repo/opendbc/safety/tests/test_gm_camera_stock_four.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_gateway_mapping.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_alternate_brake.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_ascm.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ordinary_ascm.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ordinary_sdgm.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ordinary_cc.py", + "opendbc_repo/opendbc/car/gm/tests/test_ordinary_cc.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ordinary_camera.py", + "opendbc_repo/opendbc/car/gm/tests/test_ordinary_camera.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ordinary_camera_removed.py", + "opendbc_repo/opendbc/car/gm/tests/test_ordinary_camera_removed.py", + "opendbc_repo/opendbc/safety/tests/test_gm_ascm_aol.py", + "opendbc_repo/opendbc/car/gm/tests/test_ordinary_ascm.py", + "openpilot/starpilot/longitudinal/tests/test_gm_suburban_outer.py", + "opendbc_repo/opendbc/car/gm/tests/test_ordinary_sdgm.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_camera.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_camera_removed.py", + "opendbc_repo/opendbc/car/gm/tests/test_volt_camera_removed.py", + "opendbc_repo/opendbc/safety/tests/test_gm_volt_sdgm.py", + "opendbc_repo/opendbc/car/gm/tests/test_volt_sdgm_control.py", + "opendbc_repo/opendbc/car/gm/tests/test_volt_camera_control.py", + "opendbc_repo/opendbc/safety/tests/test_hyundai_mrr35_angle_four.py", + "opendbc_repo/opendbc/safety/tests/test_kia_ev6_2025_angle.py", + "opendbc_repo/opendbc/safety/tests/test_hyundai_angle_hybrids_three.py", + "opendbc_repo/opendbc/safety/tests/test_hyundai_ccnc_angle_two.py", + "opendbc_repo/opendbc/safety/tests/test_hyundai_ioniq6_long.py", + "opendbc_repo/opendbc/safety/tests/test_hyundai_ray_pedal.py", + "opendbc_repo/opendbc/safety/tests/test_honda_aol_family.py", + "opendbc_repo/opendbc/safety/tests/test_aol_defensive_config.py", + "opendbc_repo/opendbc/safety/tests/test_honda_nidec_six.py", + "opendbc_repo/opendbc/safety/tests/test_honda_bosch_radarless_three.py", + "opendbc_repo/opendbc/safety/tests/test_subaru_gen2_angle_pair.py", + "opendbc_repo/opendbc/safety/tests/test_subaru_ascent_angle.py", + "opendbc_repo/opendbc/safety/tests/test_ford_three.py", + "opendbc_repo/opendbc/safety/tests/test_hyundai_canfd_carnival.py", +) + +# Vehicle tests import both the vendored car code and these host control owners. +# Presentation work can proceed independently; native binaries have separate +# provenance below. Keep result directories outside these source areas. +SOURCE_INPUT_PATHS = ( + "opendbc_repo", "openpilot/cereal", "openpilot/common", "openpilot/selfdrive/car", + "openpilot/selfdrive/controls", "openpilot/selfdrive/locationd", "openpilot/selfdrive/selfdrived", + "openpilot/starpilot/aol", "openpilot/starpilot/car", "openpilot/starpilot/lateral", "openpilot/starpilot/longitudinal", + "openpilot/starpilot/speed_limits", "openpilot/starpilot/schema_cache.py", "msgq_repo", "rednose_repo", + "openpilot/starpilot/vehicle_preferences.py", + "openpilot/starpilot/vehicle_startup.py", "openpilot/starpilot/controller_extensions.py", + "tools/ci", "tools/test_runner.py", "pyproject.toml", "uv.lock", "upstream-sync.json", +) + + +def source_input_snapshot(): + def git(*args): + return subprocess.check_output(["git", *args], cwd=ROOT) + + untracked = {} + for raw_path in git("ls-files", "--others", "--exclude-standard", "-z", "--", *SOURCE_INPUT_PATHS).split(b"\0"): + if raw_path: + name = os.fsdecode(raw_path) + path = ROOT / name + data = os.fsencode(os.readlink(path)) if path.is_symlink() else path.read_bytes() + untracked[name] = hashlib.sha256(data).hexdigest() + patch_bytes = git("diff", "--no-ext-diff", "--no-textconv", "--binary", "HEAD", "--", *SOURCE_INPUT_PATHS) + return {"revision": git("rev-parse", "HEAD").decode().strip(), "paths": list(SOURCE_INPUT_PATHS), + "tracked_patch_sha256": hashlib.sha256(patch_bytes).hexdigest(), "untracked_sha256": untracked} + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def targets_for(suite): + if suite == "interfaces": + car_root = ROOT / "opendbc_repo/opendbc/car" + return [str(path.relative_to(ROOT)) for path in sorted(car_root.rglob("test_*.py")) + if path != car_root / "tests/test_models.py"] + if suite == "safety-debug": + return [str(path.relative_to(ROOT)) for path in sorted((ROOT / "opendbc_repo/opendbc/safety/tests").glob("test_*.py"))] + if suite == "safety-release": + return list(RELEASE_TARGETS) + raise ValueError(f"Unknown vehicle test suite: {suite}") + + +def audit_interface_modules(targets, test_ids): + """Expose empty modules and test declarations unsupported by unittest discovery.""" + modules, errors = [], [] + for target in targets: + path = ROOT / target + module = ".".join(path.relative_to(ROOT).with_suffix("").parts) + selected = [test_id for test_id in test_ids if test_id.startswith(module + ".")] + declarations = ast.parse(path.read_text(), filename=str(path)) + functions, missing_methods = [], [] + for node in declarations.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"): + functions.append(node.name) + elif isinstance(node, ast.ClassDef): + for method in node.body: + if isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)) and method.name.startswith("test_"): + expected = f"{module}.{node.name}.{method.name}" + if not any(test_id == expected or test_id.startswith(expected + "_") for test_id in selected): + missing_methods.append(f"{node.name}.{method.name}") + if not selected: + errors.append(f"Interface test module collected zero tests: {target}") + if functions: + errors.append(f"Unsupported module-level tests require an explicit runner: {target}: {functions}") + if missing_methods: + errors.append(f"Declared interface tests were not collected: {target}: {missing_methods}") + modules.append({"path": target, "source_sha256": sha256(path), "collected_count": len(selected), + "module_level_test_functions": functions, "uncollected_test_methods": missing_methods}) + return modules, errors + + +def interface_coverage(test_ids, platforms): + prefix = ".TestCarInterfaces.test_car_interfaces_" + matches = [(test_id.split(prefix, 1)[1], test_id) for test_id in test_ids if prefix in test_id] + counts = Counter(platform for platform, _ in matches) + if set(counts) != set(platforms) or any(count != 1 for count in counts.values()): + missing, extra = sorted(set(platforms) - counts.keys()), sorted(counts.keys() - set(platforms)) + duplicates = sorted(platform for platform, count in counts.items() if count != 1) + raise ValueError(f"Concrete interface collection mismatch: missing={missing}, extra={extra}, duplicates={duplicates}") + return dict(matches) + + +def interface_execution_errors(platform_tests, records): + by_id = {record["id"]: record for record in records} + return [f"Concrete interface test for {platform} must pass: {by_id.get(test_id, {}).get('status', 'missing')} ({test_id})" + for platform, test_id in platform_tests.items() if by_id.get(test_id, {}).get("status") != "passed"] + + +def vehicle_inventory(): + from opendbc.car.values import PLATFORMS + from opendbc.car.tests.routes import non_tested_cars, routes + return {"platform_ids": sorted(PLATFORMS), "platform_count": len(PLATFORMS), + "platforms_without_route": sorted(set(PLATFORMS) - {str(route.car_model) for route in routes}), + "declared_route_exemptions": sorted(str(platform) for platform in non_tested_cars)} + + +def source_provenance(): + def git(*args): + return subprocess.check_output(["git", *args], cwd=ROOT).decode().strip() + manifest = ROOT / "upstream-sync.json" + return {"revision": git("rev-parse", "HEAD"), "status": git("status", "--porcelain"), + "inputs": source_input_snapshot(), + "manifest_sha256": sha256(manifest), "dependencies": json.loads(manifest.read_text())["dependencies"], + "runner_sha256": sha256(ROOT / "tools/test_runner.py"), "suite_runner_sha256": sha256(__file__)} + + +def native_provenance(): + import capnp.lib.capnp as native_capnp + from opendbc.can import packer, parser + path = Path(native_capnp.__file__).resolve() + if not any(str(path).endswith(suffix) for suffix in importlib.machinery.EXTENSION_SUFFIXES): + raise RuntimeError("Vehicle tests require the native pycapnp extension") + expected = ROOT / "opendbc_repo/opendbc/can" + if Path(packer.__file__).resolve().parent != expected or Path(parser.__file__).resolve().parent != expected: + raise RuntimeError("Vehicle tests imported CAN code from outside the tracked dependency") + return {"pycapnp": {"path": str(path), "sha256": sha256(path)}, + "can_implementation": "tracked upstream Python parser/packer", "can_sources": { + str(Path(module.__file__).relative_to(ROOT)): sha256(module.__file__) for module in (packer, parser)}} + + +def build_safety_library(release, output): + from opendbc.car.structs import CarParams + from opendbc.safety.tests.libsafety import libsafety_py + commands = [] + check_call = subprocess.check_call + + def record_command(command, *args, **kwargs): + commands.append(list(command)) + return check_call(command, *args, **kwargs) + + with patch.object(libsafety_py.subprocess, "check_call", side_effect=record_command): + built = Path(libsafety_py._build_libsafety(release=release)) + path = output / ("libsafety-release.so" if release else "libsafety-debug.so") + shutil.copyfile(built, path) + built.unlink() + defines_debug = any("-DALLOW_DEBUG" in command for command in commands) + if defines_debug == release or len(commands) != 2: + raise RuntimeError("Safety build did not use the requested compile variant") + libsafety_py.load(path) + status = libsafety_py.libsafety.set_safety_hooks(CarParams.SafetyModel.allOutput, 0) + if (status == 0) != (not release): + raise RuntimeError("Loaded safety library does not match the requested debug/release registry") + return {"variant": "release" if release else "ALLOW_DEBUG", "path": str(path), "sha256": sha256(path), + "commands": commands, "compiler": shutil.which(commands[0][0]), + "compiler_version": subprocess.check_output([commands[0][0], "--version"], text=True).strip(), + "all_output_hook_available": status == 0, + "loader_sha256": sha256(libsafety_py.__file__)} + + +def run(suite, output, collect_only=False): + output = Path(output).resolve() + output.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + tests, errors = [], [] + records, platform_tests, gate_errors = [], {}, [] + phase = "initialization" + plan = {"schema_version": 1, "suite": suite, "status": "initializing", "workers": 1, + "scope": "host synthetic interfaces" if suite == "interfaces" else "host safety hooks", + "uncovered": ["historical route replay", "hardware and bus integration", "additional downstream vehicles", + "independent lateral/longitudinal feature transitions", "device firmware execution"], + "targets": [], "selected_test_ids": [], "collection_errors": [], + "fuzz_seed": os.environ.get("FUZZ_SEED"), "max_examples_override": os.environ.get("MAX_EXAMPLES")} + if suite == "safety-release": + plan["uncovered"].append("full release-mode behavioral safety suite; only explicit selected classes run") + (output / "plan.json").write_text(json.dumps(plan, sort_keys=True, indent=2) + "\n") + try: + plan["targets"] = targets_for(suite) + plan["source"] = source_provenance() + plan.update(vehicle_inventory()) + plan["native"] = native_provenance() + if suite != "interfaces" and not collect_only: + plan["native"]["safety"] = build_safety_library(suite == "safety-release", output) + phase = "collection" + tests, errors = test_runner.collect(plan["targets"], None) + ids = [test.id() for test in tests] + if suite == "interfaces": + plan["module_collection"], module_errors = audit_interface_modules(plan["targets"], ids) + errors.extend(module_errors) + platform_tests = interface_coverage(ids, plan["platform_ids"]) + if not ids: + errors.append("Vehicle suite collected no tests") + plan.update(selected_test_ids=ids, interface_test_ids=platform_tests, collection_errors=errors, + status="error" if errors else "not_run" if collect_only else "ready") + (output / "plan.json").write_text(json.dumps(plan, sort_keys=True, indent=2) + "\n") + print(f"{suite}: {len(ids)} tests; {len(platform_tests)} concrete platform interface tests; one process", flush=True) + if collect_only and not errors: + return 0 + if not errors: + phase = "execution" + records = test_runner.run_batch(ids, capture_output=True) + phase = "source verification" + plan["source_after"] = source_provenance() + if plan["source"].get("inputs") != plan["source_after"].get("inputs"): + gate_errors.append("Vehicle/control source inputs changed during the suite; rerun against stable source") + except Exception: + errors.append(f"{phase} error:\n{traceback.format_exc()}") + plan["error_phase"] = phase + ids = [test.id() for test in tests] + records = test_runner.account_for_tests(ids, records, tests) + if not errors and suite == "interfaces": + gate_errors.extend(interface_execution_errors(platform_tests, records)) + elapsed = time.monotonic() - started + exit_code = test_runner.report(records, errors + gate_errors, 10, elapsed) + status = "error" if errors else "failed" if exit_code else "completed" + test_runner.write_json_report(output / "results.json", ids, records, errors, elapsed, exit_code) + result = json.loads((output / "results.json").read_text()) + result.update(status=status, gate_errors=gate_errors) + if "error_phase" in plan: + result["error_phase"] = plan["error_phase"] + (output / "results.json").write_text(json.dumps(result, sort_keys=True, indent=2) + "\n") + by_id = {record["id"]: record for record in records} + plan.update(status=status, exit_code=exit_code, collection_errors=errors, gate_errors=gate_errors, duration_seconds=elapsed, + selected_test_ids=ids, interface_test_ids=platform_tests, + result_counts=dict(Counter(record["status"] for record in records)), + interface_results={platform: by_id[test_id] for platform, test_id in platform_tests.items()}, + result_sha256=sha256(output / "results.json")) + if errors: + (output / "plan.json").write_text(json.dumps(plan, sort_keys=True, indent=2) + "\n") + (output / "coverage.json").write_text(json.dumps(plan, sort_keys=True, indent=2) + "\n") + return exit_code + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite", choices=("interfaces", "safety-debug", "safety-release"), required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--collect-only", action="store_true") + args = parser.parse_args() + os.chdir(ROOT) + # Keep the same generated examples across collection, execution, and reruns. + os.environ.setdefault("FUZZ_SEED", "0") + return run(args.suite, args.output, args.collect_only) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/test_release_safety.py b/tools/ci/test_release_safety.py new file mode 100644 index 0000000000..16c6aa7f4c --- /dev/null +++ b/tools/ci/test_release_safety.py @@ -0,0 +1,59 @@ +"""Explicit release hook-registry contract; run through run_vehicle_tests.py.""" +import unittest + +from opendbc.car.structs import CarParams +from opendbc.safety.tests.libsafety import libsafety_py + +RELEASE_MODES = frozenset({ + "silent", "hondaNidec", "toyota", "elm327", "gm", "hondaBosch", "hyundai", "chrysler", "subaru", "volkswagen", + "volkswagenMeb", "nissan", "noOutput", "hyundaiLegacy", "mazda", "body", "ford", "rivian", "tesla", "hyundaiCanfd", "teslaPreap", +}) + + +def registry_test(name, value): + def test(self): + self.assertEqual(libsafety_py.libsafety.set_safety_hooks(value, 0) == 0, name in RELEASE_MODES, name) + return test + + +class TestReleaseRegistry(unittest.TestCase): + pass + + +class TestReleaseAxisCapabilities(unittest.TestCase): + def test_experimental_hyundai_longitudinal_payloads_remain_closed_in_release(self): + safety = libsafety_py.libsafety + # Exact whole-controller SCC11, ADRV and FCA samples; generic steering + # and tester-present overlap are deliberately outside this assertion. + packets = ( + (0x420, 0, "49005dedaa470600"), + (0x420, 1, "620009ecaa470600"), + (0x51, 0, "00" * 32), + (0x38D, 1, "8d000000c03f7f00"), + (0x7D0, 0, "0328030100000000"), + (0x730, 1, "0328030100000000"), + ) + for param in (0x2004, 0x2014): + self.assertEqual(safety.set_safety_hooks(CarParams.SafetyModel.hyundai, param), 0) + safety.init_tests() + safety.set_controls_allowed(True) + for address, bus, payload in packets: + with self.subTest(safety_param=param, address=hex(address), bus=bus): + packet = libsafety_py.make_CANPacket(address, bus, bytes.fromhex(payload)) + self.assertFalse(safety.safety_tx_hook(packet)) + + def test_honda_debug_only_longitudinal_cannot_grant_aol_in_release(self): + safety = libsafety_py.libsafety + for param in (0, 2, 32, 34, 42, 50): + with self.subTest(safety_param=param): + self.assertEqual(safety.set_safety_hooks(CarParams.SafetyModel.hondaBosch, param), 0) + safety.init_tests() + safety.set_controls_allowed(True) + safety.set_aol_test_heartbeat(True) + safety.aol_set_host_request(3) + self.assertEqual(safety.aol_get_request_mask(), 0) + self.assertEqual(safety.aol_get_permission_mask(), 0) + + +for mode_name, mode_value in CarParams.SafetyModel.schema.enumerants.items(): + setattr(TestReleaseRegistry, f"test_release_registry_{mode_name}", registry_test(mode_name, mode_value)) diff --git a/tools/ci/tests/__init__.py b/tools/ci/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci/tests/test_build_entry.py b/tools/ci/tests/test_build_entry.py new file mode 100644 index 0000000000..2df20abc7c --- /dev/null +++ b/tools/ci/tests/test_build_entry.py @@ -0,0 +1,609 @@ +"""The build wrapper's CLI contract, without a container or device.""" + +import os +import hashlib +from pathlib import Path +import pickle +import shutil +import struct +import subprocess +import sys +import tempfile +import types +import unittest + + +SOURCE = Path(__file__).resolve().parents[3] / "build" +HELPER = SOURCE.parent / "scripts/laptop_device_build.sh" +VALIDATOR = SOURCE.parent / "tools/laptop_device_build/validate_artifacts.py" +MAPD_HELPER = SOURCE.parent / "scripts/build_mapd_provider.sh" +NATIVE_HELPER = SOURCE.parent / "scripts/native_device_build.sh" + + +def captured_pickle(backend: str, *, host_backend: str | None = None) -> bytes: + """A small valid pickle graph with the current TinyJit class references.""" + names = ("tinygrad", "tinygrad.engine", "tinygrad.engine.jit") + saved = {name: sys.modules.get(name) for name in names} + try: + root, engine, jit = (types.ModuleType(name) for name in names) + root.engine, engine.jit = engine, jit + sys.modules.update(zip(names, (root, engine, jit), strict=True)) + tiny = type("_TinyJit", (), {"__module__": "tinygrad.engine.jit"}) + captured = type("CapturedJit", (), {"__module__": "tinygrad.engine.jit"}) + jit._TinyJit, jit.CapturedJit = tiny, captured + run = tiny() + run.state = captured() + if host_backend is not None: + run.state.host_backend = host_backend + return pickle.dumps({"metadata": {"output_slices": {}, "input_shapes": {}}, "run": run, + "input_specs": {}, "output_specs": {}, "device": backend}) + finally: + for name, module in saved.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + +class BuildEntryTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + shutil.copy2(SOURCE, self.root / "build") + helper = self.root / "scripts/laptop_device_build.sh" + helper.parent.mkdir() + helper.write_text("""#!/usr/bin/env bash +printf '%s\\n' "$@" > "$ARGS_FILE" +mkdir -p panda/board/obj +printf changed > panda/board/obj/gitversion.h +printf changed > panda/board/obj/version +exit "${HELPER_EXIT:-0}" +""") + helper.chmod(0o755) + native = self.root / "scripts/native_device_build.sh" + native.write_text('#!/usr/bin/env bash\nprintf "native\\n" > "$ARGS_FILE"\nprintf "%s\\n" "$@" >> "$ARGS_FILE"\n') + native.chmod(0o755) + mapd = self.root / "scripts/build_mapd_provider.sh" + mapd.write_text('#!/usr/bin/env bash\nprintf "mapd\\n" > "$ARGS_FILE"\n') + mapd.chmod(0o755) + verifier = self.root / "tools/release/stage_mapd_provider.py" + verifier.parent.mkdir(parents=True) + verifier.write_text('import os, sys\nfrom pathlib import Path\n' + + 'Path("mapd-check").write_text("\\n".join(sys.argv[1:]))\n' + + 'sys.exit(int(os.getenv("MAPD_CHECK_EXIT", "0")))\n') + self.env = os.environ.copy() + self.env["ARGS_FILE"] = str(self.root / "args") + + def run_build(self, *args, helper_exit=0): + env = dict(self.env, HELPER_EXIT=str(helper_exit)) + return subprocess.run([str(self.root / "build"), *args], cwd=self.root, env=env, check=False) + + def args(self): + return (self.root / "args").read_text().splitlines() + + def test_shortcuts_and_numeric_jobs(self): + self.assertEqual(self.run_build("--params", "6").returncode, 0) + self.assertEqual(self.args(), ["scons", "--no-scrub", "openpilot/common/libparams_c.so", "-j6"]) + self.assertEqual(self.run_build("--cereal").returncode, 0) + self.assertEqual(self.args(), ["scons", "--no-scrub", "openpilot/cereal/libcereal.a", + "openpilot/cereal/libsocketmaster.a", "openpilot/cereal/messaging/bridge", + "openpilot/cereal/services.h"]) + self.assertEqual(self.run_build("--panda", "8").returncode, 0) + self.assertEqual(self.args(), ["scons", "--no-scrub", "panda/board/obj/panda.bin.signed", "panda/board/obj/panda_h7.bin.signed", + "panda/board/obj/body_h7.bin.signed", "-j8"]) + + def test_default_and_passthrough(self): + self.assertEqual(self.run_build().returncode, 0) + self.assertEqual(self.args(), ["build"]) + self.assertEqual(self.run_build("--verbose", "4").returncode, 0) + self.assertEqual(self.args(), ["build", "4", "--verbose"]) + self.assertEqual(self.run_build("4", "--verbose").returncode, 0) + self.assertEqual(self.args(), ["build", "4", "--verbose"]) + self.assertEqual(self.run_build("--verbose", "4", "FEATURE=value", "8").returncode, 0) + self.assertEqual(self.args(), ["build", "8", "--verbose", "FEATURE=value"]) + self.assertEqual(self.run_build("8").returncode, 0) + self.assertEqual(self.args(), ["build", "8"]) + + def test_explicit_scons_jobs_options_keep_their_value(self): + for option in ("-j", "--jobs"): + with self.subTest(option=option): + self.assertEqual(self.run_build("--verbose", option, "4").returncode, 0) + self.assertEqual(self.args(), ["build", "--verbose", option, "4"]) + self.assertEqual(self.run_build("--panda", option, "4").returncode, 0) + self.assertEqual(self.args(), ["scons", "--no-scrub", "panda/board/obj/panda.bin.signed", + "panda/board/obj/panda_h7.bin.signed", "panda/board/obj/body_h7.bin.signed", option, "4"]) + + def test_zero_jobs_fail_before_package_or_helper_mutation(self): + (self.root / "prebuilt").write_text("preserve") + for args in (("0",), ("--verbose", "00"), ("--panda", "0"), ("--params", "0")): + with self.subTest(args=args): + self.assertEqual(self.run_build(*args).returncode, 2) + self.assertFalse((self.root / "args").exists()) + self.assertFalse((self.root / "mapd-check").exists()) + self.assertFalse((self.root / "panda").exists()) + self.assertEqual((self.root / "prebuilt").read_text(), "preserve") + + def test_missing_map_package_stops_full_build_but_allows_repair(self): + self.env["MAPD_CHECK_EXIT"] = "9" + self.assertEqual(self.run_build().returncode, 9) + self.assertFalse((self.root / "args").exists()) + self.assertEqual((self.root / "mapd-check").read_text().splitlines(), + ["--source", str(self.root), "--require-tracked"]) + (self.root / "mapd-check").unlink() + self.assertEqual(self.run_build("--mapd").returncode, 0) + self.assertEqual(self.args(), ["mapd"]) + self.assertFalse((self.root / "mapd-check").exists()) + + def test_firmware_assignments_require_environment_before_helper_dispatch(self): + for prefix in (("--panda", "4"), ("4",)): + for assignment in ("RELEASE=1", "CERT=debug", "RELEASE="): + for release_env in ({}, {"RELEASE": "1", "CERT": "environment key"}): + with self.subTest(prefix=prefix, assignment=assignment, environment=release_env): + result = subprocess.run([str(self.root / "build"), *prefix, assignment], cwd=self.root, + env=dict(self.env, **release_env), check=False, capture_output=True, text=True) + self.assertEqual(result.returncode, 2) + self.assertIn("environment variables", result.stderr) + self.assertIn("RELEASE=1 CERT=/path/to/certificate ./build --panda 4", result.stderr) + self.assertFalse((self.root / "args").exists()) + self.assertFalse((self.root / "panda").exists()) + + def test_environment_and_unrelated_scons_assignments_remain_supported(self): + helper = self.root / "scripts/laptop_device_build.sh" + helper.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$RELEASE" "$CERT" "$@" > "$ARGS_FILE"\n') + result = subprocess.run([str(self.root / "build"), "--panda", "4", "FEATURE=value"], cwd=self.root, + env=dict(self.env, RELEASE="1", CERT="certificate with spaces"), check=False) + self.assertEqual(result.returncode, 0) + self.assertEqual(self.args(), ["1", "certificate with spaces", "scons", "--no-scrub", + "panda/board/obj/panda.bin.signed", "panda/board/obj/panda_h7.bin.signed", "panda/board/obj/body_h7.bin.signed", + "-j4", "FEATURE=value"]) + + def test_mapd_is_explicit_and_cannot_be_mixed_with_scons(self): + self.assertEqual(self.run_build("--mapd").returncode, 0) + self.assertEqual(self.args(), ["mapd"]) + (self.root / "args").unlink() + self.assertEqual(self.run_build("--mapd", "--panda").returncode, 2) + self.assertFalse((self.root / "args").exists()) + self.assertEqual(self.run_build().returncode, 0) + self.assertEqual(self.args(), ["build"]) + + def test_help_does_not_invoke_builder(self): + result = subprocess.run([str(self.root / "build"), "--help"], cwd=self.root, env=self.env, + check=False, capture_output=True, text=True) + self.assertEqual(result.returncode, 0) + self.assertIn("--panda", result.stdout) + self.assertIn("--mapd", result.stdout) + self.assertIn("RELEASE=1 CERT=/path/to/certificate ./build --panda 4", result.stdout) + self.assertFalse((self.root / "args").exists()) + + def test_actual_agnos_arm64_dispatch_uses_native_helper(self): + # Only the temporary copy changes the immutable /AGNOS probe; production + # has no environment override that can redirect a build to native mode. + build = self.root / "build" + build.write_text(build.read_text().replace("-f /AGNOS", f"-f {self.root}/AGNOS")) + (self.root / "AGNOS").touch() + fake_uname = self.root / "fake-bin/uname" + fake_uname.parent.mkdir() + fake_uname.write_text('#!/usr/bin/env bash\n[[ "$1" == -s ]] && echo Linux || echo aarch64\n') + fake_uname.chmod(0o755) + self.env["PATH"] = f"{fake_uname.parent}:{self.env['PATH']}" + self.assertEqual(self.run_build("--panda", "2").returncode, 0) + self.assertEqual(self.args(), ["native", "scons", "--no-scrub", "panda/board/obj/panda.bin.signed", "panda/board/obj/panda_h7.bin.signed", + "panda/board/obj/body_h7.bin.signed", "-j2"]) + self.assertEqual(self.run_build().returncode, 0) + self.assertEqual(self.args(), ["native", "build"]) + + def test_non_panda_shortcut_restores_metadata_even_on_failure(self): + obj = self.root / "panda/board/obj" + obj.mkdir(parents=True) + (obj / "gitversion.h").write_text("original") + self.assertEqual(self.run_build("--params", helper_exit=7).returncode, 7) + self.assertEqual((obj / "gitversion.h").read_text(), "original") + self.assertFalse((obj / "version").exists()) + + def test_panda_shortcut_retains_generated_metadata(self): + self.assertEqual(self.run_build("--panda").returncode, 0) + self.assertEqual((self.root / "panda/board/obj/version").read_text(), "changed") + + +class NativeDeviceBuildTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() + helper = self.root / "scripts/native_device_build.sh" + helper.parent.mkdir() + shutil.copy2(NATIVE_HELPER, helper) + helper.write_text(helper.read_text().replace("-f /AGNOS", f"-f {self.root}/AGNOS") + .replace("/usr/local/venv/bin/scons", f"{self.root}/missing/scons")) + (self.root / "AGNOS").touch() + fake_uname = self.root / "fake-bin/uname" + fake_uname.parent.mkdir() + fake_uname.write_text('#!/usr/bin/env bash\n[[ "$1" == -s ]] && echo Linux || echo aarch64\n') + fake_uname.chmod(0o755) + scons = self.root / ".venv/bin/scons" + scons.parent.mkdir(parents=True) + scons.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "$SCONS_ARGS_FILE"\n' + + 'printf "%s" "$PYTHONPATH" > "$SCONS_PYTHONPATH_FILE"\nexit "${SCONS_EXIT:-0}"\n') + scons.chmod(0o755) + validator = self.root / "tools/laptop_device_build/validate_artifacts.py" + validator.parent.mkdir(parents=True) + validator.write_text('import os, sys\nsys.exit(int(os.getenv("VALIDATOR_EXIT", "0")))\n') + (validator.parent / 'package_model_chunks.py').write_text('') + (validator.parent / 'validate_params_registry.py').write_text('') + manifest = self.root / 'openpilot/common/prebuilt_manifest.py' + manifest.parent.mkdir(parents=True) + manifest.write_text('') + self.env = dict(os.environ, PATH=f"{fake_uname.parent}:{os.environ['PATH']}", + SCONS_ARGS_FILE=str(self.root / "scons-args"), + SCONS_PYTHONPATH_FILE=str(self.root / "scons-pythonpath")) + + def run_helper(self, *args, **extra_env): + return subprocess.run([str(self.root / "scripts/native_device_build.sh"), *args], cwd=self.root, + env=dict(self.env, **extra_env), check=False, capture_output=True, text=True) + + def test_complete_build_validates_before_prebuilt_and_binds_checkout(self): + self.assertEqual(self.run_helper("build", "2", VALIDATOR_EXIT="1").returncode, 1) + self.assertFalse((self.root / "prebuilt").exists()) + self.assertEqual((self.root / "scons-args").read_text().splitlines(), ["-j2"]) + self.assertEqual((self.root / "scons-pythonpath").read_text().split(":"), + [str(self.root / name) for name in ("", "msgq_repo", "opendbc_repo", "rednose_repo", + "teleoprtc_repo", "tinygrad_repo")]) + result = self.run_helper("build", "2") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((self.root / "prebuilt").exists()) + self.assertEqual(self.run_helper("build", "2", "--dry-run").returncode, 0) + self.assertFalse((self.root / "prebuilt").exists()) + + def test_zero_jobs_fail_before_scons_or_prebuilt_change(self): + (self.root / "prebuilt").write_text("preserve") + self.assertEqual(self.run_helper("build", "0").returncode, 2) + self.assertFalse((self.root / "scons-args").exists()) + self.assertEqual((self.root / "prebuilt").read_text(), "preserve") + + def test_firmware_assignments_fail_before_scons_or_prebuilt_change(self): + (self.root / "prebuilt").write_text("preserve") + for mode in ("build", "scons"): + for assignment in ("RELEASE=1", "CERT=debug", "RELEASE="): + with self.subTest(mode=mode, assignment=assignment): + result = self.run_helper(mode, "4", assignment, RELEASE="1", CERT="environment key") + self.assertEqual(result.returncode, 2) + self.assertIn("environment variables", result.stderr) + self.assertFalse((self.root / "scons-args").exists()) + self.assertEqual((self.root / "prebuilt").read_text(), "preserve") + + def test_unrelated_scons_assignments_are_forwarded(self): + self.assertEqual(self.run_helper("scons", "FEATURE=value").returncode, 0) + self.assertEqual((self.root / "scons-args").read_text().splitlines(), ["FEATURE=value"]) + + def test_shortcut_never_marks_full_build(self): + self.assertEqual(self.run_helper("scons", "--no-scrub", "openpilot/common/libparams_c.so").returncode, 0) + self.assertEqual((self.root / "scons-args").read_text().splitlines(), + ["openpilot/common/libparams_c.so"]) + self.assertFalse((self.root / "prebuilt").exists()) + + +class LaptopReleaseForwardingTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="laptop build ") + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() + helper = self.root / "scripts/laptop_device_build.sh" + helper.parent.mkdir() + helper.write_text(HELPER.read_text().replace('\nmain "$@"\n', '\n')) + self.engine = self.root / "fake engine" + self.engine.write_text('#!/usr/bin/env bash\nprintf "%s\\0" "$@" > "$CAPTURE_FILE"\n') + self.engine.chmod(0o755) + self.capture = self.root / "container args" + + def run_scons(self, **environment): + env = os.environ.copy() + env.pop("RELEASE", None) + env.pop("CERT", None) + env.update(environment, CAPTURE_FILE=str(self.capture), FAKE_ENGINE=str(self.engine)) + command = ''.join(('source "$1"; detect_engine() { printf "%s" "$FAKE_ENGINE"; }; ', + 'ensure_image_exists() { :; }; ensure_sysroot_layout() { :; }; ', + 'scrub_mixed_arch_artifacts() { :; }; ', + 'run_larch64_scons 4 --no-scrub panda/board/obj/panda_h7.bin.signed ', + 'panda/board/obj/body_h7.bin.signed FEATURE=value --verbose "--tag=two words"')) + return subprocess.run(["bash", "-c", command, "bash", str(self.root / "scripts/laptop_device_build.sh")], + cwd=self.root, env=env, check=False, capture_output=True, text=True) + + def args(self): + return [value.decode() for value in self.capture.read_bytes().split(b'\0')[:-1]] + + def test_direct_modes_reject_firmware_assignments_before_build(self): + (self.root / "prebuilt").write_text("preserve") + command = ''.join(('source "$1"; shift; ', + 'run_larch64_build() { touch "$CAPTURE_FILE"; }; ', + 'run_larch64_scons() { touch "$CAPTURE_FILE"; }; main "$@"')) + env = dict(os.environ, CAPTURE_FILE=str(self.capture), RELEASE="1", CERT="environment key") + for mode in ("build", "scons"): + for assignment in ("RELEASE=1", "CERT=debug", "RELEASE="): + with self.subTest(mode=mode, assignment=assignment): + result = subprocess.run(["bash", "-c", command, "bash", + str(self.root / "scripts/laptop_device_build.sh"), mode, "4", assignment], + cwd=self.root, env=env, check=False, capture_output=True, text=True) + self.assertEqual(result.returncode, 2) + self.assertIn("environment variables", result.stderr) + self.assertFalse(self.capture.exists()) + self.assertEqual((self.root / "prebuilt").read_text(), "preserve") + + def test_direct_build_zero_jobs_fail_before_builder_mutation(self): + (self.root / "prebuilt").write_text("preserve") + command = 'source "$1"; run_larch64_build() { touch "$CAPTURE_FILE"; }; main build 0' + result = subprocess.run(["bash", "-c", command, "bash", str(self.root / "scripts/laptop_device_build.sh")], + cwd=self.root, env=dict(os.environ, CAPTURE_FILE=str(self.capture)), + check=False, capture_output=True, text=True) + self.assertEqual(result.returncode, 2) + self.assertIn("jobs must be positive", result.stderr) + self.assertFalse(self.capture.exists()) + self.assertEqual((self.root / "prebuilt").read_text(), "preserve") + + def test_release_certificate_with_spaces_is_mounted_and_forwarded(self): + cert = self.root / "certs with spaces/release key" + cert.parent.mkdir() + cert.write_text("test certificate") + result = self.run_scons(RELEASE="1", CERT=str(cert.relative_to(self.root))) + self.assertEqual(result.returncode, 0, result.stderr) + args = self.args() + self.assertIn(f"{cert}:/tmp/starpilot-signing-cert:ro", args) + self.assertEqual(args[args.index("-e") + 1], "RELEASE=1") + self.assertEqual(args[args.index("-e") + 3], "CERT=/tmp/starpilot-signing-cert") + self.assertIn("--tag=two\\ words", args[-1]) + self.assertIn("FEATURE=value", args[-1]) + + def test_debug_does_not_inherit_release_and_invalid_release_fails_early(self): + result = self.run_scons() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("-e", self.args()) + self.capture.unlink() + result = self.run_scons(RELEASE="") + self.assertNotEqual(result.returncode, 0) + self.assertIn("RELEASE must be nonempty", result.stderr) + self.assertFalse(self.capture.exists()) + result = self.run_scons(RELEASE="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("RELEASE requires CERT", result.stderr) + self.assertFalse(self.capture.exists()) + result = self.run_scons(RELEASE="1", CERT="missing key") + self.assertNotEqual(result.returncode, 0) + self.assertIn("CERT", result.stderr) + self.assertFalse(self.capture.exists()) + result = self.run_scons(RELEASE="1", CERT=str(self.root)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("readable regular file", result.stderr) + self.assertFalse(self.capture.exists()) + with tempfile.TemporaryDirectory(prefix="external key ") as external: + outside = Path(external) / "release key" + outside.write_text("test certificate") + result = self.run_scons(RELEASE="1", CERT=str(outside)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"{outside.resolve()}:/tmp/starpilot-signing-cert:ro", self.args()) + self.capture.unlink() + (self.root / "linked key").symlink_to(outside) + result = self.run_scons(RELEASE="1", CERT="linked key") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"{outside.resolve()}:/tmp/starpilot-signing-cert:ro", self.args()) + + +class MapdHelperSafetyTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name).resolve() + helper = self.root / 'scripts/build_mapd_provider.sh' + helper.parent.mkdir() + shutil.copy2(MAPD_HELPER, helper) + self.helper = helper + + def test_missing_offline_cache_fails_without_creating_package(self): + result = subprocess.run([str(self.helper)], cwd=self.root, capture_output=True, text=True, + env=dict(os.environ, COMMA_MAPD_MODULE_CACHE=str(self.root / 'missing'))) + self.assertEqual(result.returncode, 1) + self.assertIn('offline Go module cache missing', result.stderr) + self.assertFalse((self.root / 'openpilot/starpilot/maps/provider').exists()) + + def test_external_build_cache_rejected_before_mutation(self): + modules = self.root / 'modules' + modules.mkdir() + with tempfile.TemporaryDirectory() as external: + outside = Path(external) / 'unsafe-cache' + result = subprocess.run([str(self.helper)], cwd=self.root, capture_output=True, text=True, + env=dict(os.environ, COMMA_MAPD_MODULE_CACHE=str(modules), + COMMA_MAPD_BUILD_CACHE=str(outside))) + self.assertEqual(result.returncode, 1) + self.assertIn('must resolve beneath this checkout', result.stderr) + self.assertFalse(outside.exists()) + + +class ArtifactCheckTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + helper = self.root / "scripts/laptop_device_build.sh" + helper.parent.mkdir() + shutil.copy2(HELPER, helper) + validator = self.root / "tools/laptop_device_build/validate_artifacts.py" + validator.parent.mkdir(parents=True) + shutil.copy2(VALIDATOR, validator) + chunks = self.root / "openpilot/common/file_chunker.py" + chunks.parent.mkdir(parents=True) + (self.root / "openpilot/__init__.py").touch() + shutil.copy2(SOURCE.parent / "openpilot/common/file_chunker.py", chunks) + for name in ("tools/laptop_device_build/jetlink_artifacts.py", "tools/laptop_device_build/jetlink_pickle.py", + "openpilot/system/camerad/cameras/nv12_info.py", "tinygrad_repo/tinygrad/uop/__init__.py"): + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(SOURCE.parent / name, target) + from tools.laptop_device_build.jetlink_artifacts import MODEL_DIR, warp_names + for name in warp_names(): + target = self.root / MODEL_DIR / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(SOURCE.parent / MODEL_DIR / name, target) + shipped = b"verified RDF artifact fixture" + self.write("openpilot/selfdrive/modeld/models/rdf43_driving_tinygrad.pkl", shipped) + self.write("openpilot/starpilot/models/catalog.py", + (f"DEFAULT_SMALL='rdf43'\nDEFAULT_SMALL_SHA256='{hashlib.sha256(shipped).hexdigest()}'\n" + + f"DEFAULT_SMALL_SIZE={len(shipped)}\n").encode()) + + elf = bytearray(128) + elf[:6] = b"\x7fELF\x02\x01" + struct.pack_into("\n" + archive_header + bytes(obj) + + for name in ("openpilot/common/libparams_c.so", "openpilot/selfdrive/pandad/pandad", + "openpilot/system/loggerd/loggerd", "openpilot/system/loggerd/encoderd", + "msgq_repo/msgq/ipc_pyx.so", "msgq_repo/msgq/visionipc/visionipc_pyx.so", + "rednose_repo/rednose/helpers/ekf_sym_pyx.so", "openpilot/tools/replay/replay", + "openpilot/tools/replay/libmedia_mux.so"): + self.write(name, self.elf) + for name in ("openpilot/cereal/libcereal.a", "openpilot/cereal/libsocketmaster.a"): + self.write(name, archive) + for name in ("dmonitoring_model_tinygrad.pkl",): + code = captured_pickle("QCOM", host_backend="CPU") + self.write(f"openpilot/selfdrive/modeld/models/{name}", struct.pack("&2\nexit 1\n') + sed.chmod(0o755) + with (patch.dict(os.environ, {'SP_HOST_RUNTIME': '1'}), + patch.object(run_local_tests.platform, 'system', return_value='Darwin'), + patch.object(Path, 'is_dir', return_value=False), + patch.object(run_local_tests.shutil, 'which', return_value=str(sed)), + patch.object(run_local_tests, 'run') as invoke, + patch('sys.stderr') as stderr): + with self.assertRaises(SystemExit) as error: + run_local_tests.main(['--output', str(Path(temporary) / 'results')]) + self.assertEqual(error.exception.code, 2) + self.assertIn('brew install gnu-sed', ''.join(call.args[0] for call in stderr.write.call_args_list)) + invoke.assert_not_called() + + def test_local_command_preserves_failure_and_uses_isolated_host(self): + with tempfile.TemporaryDirectory() as temporary, \ + patch.dict(os.environ, {'SP_HOST_RUNTIME': '0'}), \ + patch.object(run_local_tests.subprocess, 'run') as invoke: + invoke.return_value.returncode = 7 + output = (Path(temporary) / 'results').resolve() + self.assertEqual(run_local_tests.main(['--output', str(output)]), 7) + self.assertEqual(invoke.call_args.args[0], [str(run_local_tests.ROOT / 'dev'), 'python', + 'tools/ci/run_local_tests.py', '--output', str(output)]) + + def test_local_regressions_use_disposable_state_and_full_lint(self): + observed = {} + def invoke(plan, output, *, environment): + observed.update(environment) + self.assertTrue(Path(environment['PARAMS_ROOT']).is_dir()) + self.assertNotIn('FAST', environment) + self.assertNotIn('SKIP', environment) + self.assertNotIn('PYTEST_ADDOPTS', environment) + self.assertEqual(plan[:2], [('mapd-package', [sys.executable, 'tools/release/stage_mapd_provider.py', + '--source', str(run_local_tests.ROOT), '--require-tracked']), + ('lint', ['bash', 'scripts/lint/lint.sh'])]) + self.assertIn('openpilot/starpilot/tests', plan[-1][1]) + self.assertIn('opendbc_repo/opendbc/safety/tests/test_gm_bolt_pedal.py', plan[-1][1]) + self.assertIn('opendbc_repo/opendbc/safety/tests/test_hyundai_ioniq6_long.py', plan[-1][1]) + return 9 + with tempfile.TemporaryDirectory() as temporary, \ + patch.dict(os.environ, {'SP_HOST_RUNTIME': '1', 'FAST': '1', 'SKIP': 'ty', 'PYTEST_ADDOPTS': '--collect-only'}), \ + patch.object(run_local_tests, 'run', side_effect=invoke): + self.assertEqual(run_local_tests.main(['--output', str(Path(temporary) / 'results')]), 9) + self.assertFalse(Path(observed['PARAMS_ROOT']).exists()) + + def test_full_local_command_delegates_to_existing_suite(self): + with tempfile.TemporaryDirectory() as temporary, \ + patch.dict(os.environ, {'SP_HOST_RUNTIME': '1', 'PYTEST_ADDOPTS': '--collect-only'}), \ + patch.object(run_local_tests.platform, 'system', return_value='Linux'), \ + patch.object(run_local_tests.subprocess, 'call', return_value=3) as invoke: + output = (Path(temporary) / 'results').resolve() + self.assertEqual(run_local_tests.main(['--full', '--download', '--output', str(output)]), 3) + self.assertEqual(invoke.call_args.args[0], [sys.executable, 'tools/ci/run_predeploy.py', + '--output', str(output), '--download']) + self.assertNotIn('PYTEST_ADDOPTS', invoke.call_args.kwargs['env']) + + def test_failure_is_preserved_and_later_check_does_not_run(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + marker = root / 'unexpected' + plan = [('broken', [sys.executable, '-c', 'print("diagnostic"); raise SystemExit(7)']), + ('later', [sys.executable, '-c', f'from pathlib import Path; Path({str(marker)!r}).touch()'])] + self.assertEqual(run(plan, root / 'results'), 7) + self.assertFalse(marker.exists()) + self.assertIn('diagnostic', (root / 'results/broken.log').read_text()) + self.assertEqual(json.loads((root / 'results/results.json').read_text())[0]['returncode'], 7) + + def test_recorded_download_is_explicit_and_both_variants_run(self): + plan = commands(Path('/results'), Path('/cache')) + self.assertEqual([name for name, _ in plan['recorded']], ['recorded-debug', 'recorded-release']) + self.assertTrue(all('--download' not in command for _, command in plan['recorded'])) + self.assertTrue(all('--download' in command for _, command in commands(Path('/results'), Path('/cache'), True)['recorded'])) + + def test_full_plan_builds_and_requires_native_protocols(self): + plan = commands(Path('/results'), Path('/cache')) + self.assertEqual(plan['build'][0][1][1:], ['-m', 'SCons', '-j4']) + self.assertEqual([name for name, _ in plan['native']], [ + 'test_panda_usb', 'test_pandad_canprotocol', 'test_aol_protocol', 'test_aol_wire', + *(['test_producer_timing', 'test_producer_sink'] if platform.system() == 'Linux' else []), 'map-ipc']) + self.assertIn('-race', plan['native'][-1][1]) + + def test_full_plan_requires_unfiltered_process_replay(self): + with patch('tools.ci.run_predeploy.os.cpu_count', return_value=8): + plan = commands(Path('/results'), Path('/cache')) + self.assertIn('replay', STAGES) + self.assertEqual(plan['replay'], [('process-replay', [sys.executable, + 'openpilot/selfdrive/test/process_replay/test_processes.py', '-j', '8', '--output', '/results/process-replay'])]) + + def test_default_run_includes_process_replay(self): + with tempfile.TemporaryDirectory() as temporary, \ + patch.object(sys, 'argv', ['run_predeploy.py', '--output', str(Path(temporary) / 'results')]), \ + patch('tools.ci.run_predeploy.platform.system', return_value='Linux'), \ + patch('tools.ci.run_predeploy.run', return_value=0) as invoke: + self.assertEqual(main(), 0) + self.assertIn('process-replay', [name for name, _ in invoke.call_args.args[0]]) + + def test_reports_cannot_be_overwritten(self): + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaises(FileExistsError): + run([], Path(temporary)) + + + def test_full_source_stage_does_not_inherit_fast_lint(self): + with tempfile.TemporaryDirectory() as temporary, \ + patch.dict(os.environ, {'FAST': '1'}), \ + patch.object(sys, 'argv', ['run_predeploy.py', '--output', str(Path(temporary) / 'results'), '--stage', 'source']), \ + patch('tools.ci.run_predeploy.run', return_value=0) as invoke: + self.assertEqual(main(), 0) + self.assertNotIn('FAST', invoke.call_args.kwargs['environment']) + self.assertIn('lint', [name for name, _ in invoke.call_args.args[0]]) + + + def test_native_stage_rejects_non_linux_before_running(self): + with tempfile.TemporaryDirectory() as temporary, \ + patch.object(sys, 'argv', ['run_predeploy.py', '--output', str(Path(temporary) / 'results')]), \ + patch('tools.ci.run_predeploy.platform.system', return_value='Darwin'), \ + patch('tools.ci.run_predeploy.run') as invoke, \ + patch('sys.stderr'): + with self.assertRaises(SystemExit) as error: + main() + self.assertEqual(error.exception.code, 2) + invoke.assert_not_called() diff --git a/tools/ci/tests/test_safety_qualification.py b/tools/ci/tests/test_safety_qualification.py new file mode 100644 index 0000000000..c1f74c004b --- /dev/null +++ b/tools/ci/tests/test_safety_qualification.py @@ -0,0 +1,499 @@ +from contextlib import redirect_stdout +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import signal +import threading +import time +import tempfile +from unittest import TestCase, skipUnless +from unittest.mock import patch + +from tools.ci import run_safety_qualification as qualification + + +class TestSafetyQualificationRunner(TestCase): + @staticmethod + def mutation_artifact(): + source = "opendbc/safety/helpers.h" + def digest(path): + return qualification.hashlib.sha256(path.read_bytes()).hexdigest() + return {"schema_version": 1, "discovered": 1, "pruned_build_incompatible": [], + "safety_input_sha256": digest(qualification.SAFETY_TESTS / "libsafety/safety.c"), + "preprocessed_source_sha256": "1" * 64, + "mutation_runner_sha256": digest(qualification.SAFETY_TESTS / "mutation.py"), + "source_sha256": {source: digest(qualification.ROOT / "opendbc_repo" / source)}, + "baseline_sec": 1.0, + "target_sets": [["test_mod.TestCase.test_one"]], + "results": [{"site_id": 0, "source": source, "line": 40, "mutator": "boundary", + "original_op": "0", "mutated_op": "1", "outcome": "killed", + "selected_test_set": 0, "selected_test_count": 1, "details": "assertion", + "failure_kind": None, "exit_code": None, "stderr_tail": "", "stdout_tail": "", + "unittest_output_tail": ""}]} + + def test_coverage_instrumentation_is_required_and_does_not_leak_to_other_gates(self): + for name, inherited in (("coverage", "0"), ("coverage", "1"), ("mutation-full", "1"), ("mutation-list", "1")): + with self.subTest(gate=name, inherited=inherited), tempfile.TemporaryDirectory() as td, \ + patch.dict(os.environ, {"SAFETY_COVERAGE": inherited}): + gate = qualification.Gate(name, Path(td), sys.executable) + if name == "coverage": + self.assertEqual(gate.env["SAFETY_COVERAGE"], "1") + else: + self.assertNotIn("SAFETY_COVERAGE", gate.env) + + def test_source_identity_includes_gate_and_workflow_inputs(self): + identity = qualification.source_identity() + self.assertFalse(any("/obj/" in path or "/gen/" in path for path in identity)) + for path in ("tools/ci/run_safety_qualification.py", "tools/ci/tests/test_safety_qualification.py", + ".github/workflows/safety.yaml", "pyproject.toml", "uv.lock", + "opendbc_repo/opendbc/safety/tests/libsafety/safety.c", + "openpilot/starpilot/aol/intent.py", "openpilot/cereal/SConscript", + "msgq_repo/msgq/ipc_pyx.pyx", "msgq_repo/SConscript", "tools/setup_dependencies.sh", + "openpilot/common/SConscript", "panda/tests/libpanda/SConscript", + "panda/board/stm32h7/stm32h7x5_flash.ld", "panda/board/stm32h7/startup_stm32h7x5xx.s", + "panda/tests/misra/coverage_table", "opendbc_repo/opendbc/safety/tests/misra/coverage_table"): + self.assertEqual(identity[path], qualification.hashlib.sha256((qualification.ROOT / path).read_bytes()).hexdigest()) + + def test_prerequisite_transition_rejects_every_unexpected_bootstrap_change(self): + outputs = dict.fromkeys(qualification.MPC_PREREQUISITE_OUTPUTS, "old-output") + inputs = dict.fromkeys(("openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py", + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "openpilot/cereal/car.capnp", "opendbc_repo/opendbc/safety/safety.h", + "msgq_repo/msgq/ipc_pyx.pyx", "tools/ci/run_safety_qualification.py"), "input") + before = {**outputs, **inputs} + regenerated = {**dict.fromkeys(outputs, "new-output"), **inputs} + for name in ("coverage", "mutation-full"): + with self.subTest(gate=name): + self.assertTrue(qualification.prerequisite_transition(before, regenerated, name)) + for path in inputs: + with self.subTest(changed_input=path): + self.assertFalse(qualification.prerequisite_transition(before, {**regenerated, path: "changed"}, name)) + self.assertFalse(qualification.prerequisite_transition(before, {**regenerated, "openpilot/undeclared.c": "new"}, name)) + removed = dict(regenerated) + removed.pop(next(iter(outputs))) + self.assertFalse(qualification.prerequisite_transition(before, removed, name)) + for name in ("panda-host", "panda-misra", "opendbc-misra", "mutation-list"): + with self.subTest(gate=name): + self.assertFalse(qualification.prerequisite_transition(before, regenerated, name)) + self.assertTrue(qualification.prerequisite_transition(before, before, name)) + + def test_clean_checkout_bootstrap_hashes_real_files_and_rejects_generator_change(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + generator = "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py" + for path in (*qualification.MPC_PREREQUISITE_OUTPUTS, generator): + file = root / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text("checked-in source fixture\n") + subprocess.run(["git", "-c", "gc.auto=0", "init", "--quiet", str(root)], check=True) + subprocess.run(["git", "-c", "gc.auto=0", "add", "."], cwd=root, check=True) + with patch.object(qualification, "ROOT", root): + before = qualification.source_identity() + for path in qualification.MPC_PREREQUISITE_OUTPUTS: + (root / path).write_text("regenerated output fixture\n") + built = qualification.source_identity() + self.assertEqual(set(before), set(built)) + self.assertEqual({path for path in before if before[path] != built[path]}, set(qualification.MPC_PREREQUISITE_OUTPUTS)) + self.assertTrue(qualification.prerequisite_transition(before, built, "coverage")) + (root / generator).write_text("unexpected changed generator\n") + self.assertFalse(qualification.prerequisite_transition(before, qualification.source_identity(), "coverage")) + + def test_main_pins_inputs_before_bootstrap_and_all_outputs_after_bootstrap(self): + before = {**dict.fromkeys(qualification.MPC_PREREQUISITE_OUTPUTS, "old-output"), + "opendbc_repo/opendbc/safety/safety.h": "input"} + regenerated = {**dict.fromkeys(qualification.MPC_PREREQUISITE_OUTPUTS, "new-output"), + "opendbc_repo/opendbc/safety/safety.h": "input"} + cases = (("fresh bootstrap", regenerated, regenerated, "commit", True, True), + ("input mutation", {**regenerated, "opendbc_repo/opendbc/safety/safety.h": "bad"}, regenerated, "commit", False, False), + ("head mutation", regenerated, regenerated, "other-commit", False, False), + ("output changed after bootstrap", regenerated, before, "commit", True, False), + ("native prerequisite changed during gate", regenerated, regenerated, "commit", True, False), + ("solver companion changed during gate", regenerated, regenerated, "commit", True, False)) + for name, built, final, built_head, executed, passed in cases: + with self.subTest(case=name), tempfile.TemporaryDirectory() as td: + root = Path(td) / "checkout" + (root / ".venv/bin").mkdir(parents=True) + (root / ".venv/bin/python").symlink_to(sys.executable) + output = Path(td) / "evidence" + extension = ".dylib" if sys.platform == "darwin" else ".so" + artifacts = (f"openpilot/common/libparams_c{extension}", "msgq_repo/msgq/ipc_pyx.so", + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so") + artifacts = qualification.prerequisite_artifact_paths(artifacts, "coverage") + for path in artifacts: + artifact = root / path + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(b"built-native") + + def execute_gate(case_name=name, case_root=root, case_artifacts=artifacts): + if case_name == "native prerequisite changed during gate": + (case_root / case_artifacts[2]).write_bytes(b"unexpected replacement") + if case_name == "solver companion changed during gate": + (case_root / case_artifacts[3]).write_bytes(b"unexpected solver replacement") + return True + + with patch.object(sys, "argv", ["qualification", "--gate", "coverage", "--output", str(output)]), \ + patch.object(qualification, "ROOT", root), \ + patch.object(qualification, "source_identity", side_effect=(before, built, final)), \ + patch.object(qualification, "source_head", side_effect=("commit", built_head, built_head)), \ + patch.object(qualification, "generated_identity", return_value={}), \ + patch.object(qualification.Gate, "run", return_value=(0, "")), \ + patch.object(qualification.Gate, "hash_artifact"), \ + patch.object(qualification, "check_import_origins", return_value=True), \ + patch.object(qualification, "imported_module_hashes", return_value={}), \ + patch.object(qualification.Gate, "execute", side_effect=execute_gate) as execute: + self.assertEqual(qualification.main(), 0 if passed else 1) + self.assertEqual(execute.called, executed) + report = json.loads((output / "results.json").read_text()) + self.assertEqual(report["passed"], passed) + self.assertEqual(report["source_unchanged"], executed and built == final) + self.assertEqual(report["prerequisite_artifacts_unchanged"], + executed and name not in ("native prerequisite changed during gate", "solver companion changed during gate")) + if name == "fresh bootstrap": + self.assertEqual(json.loads((output / "source-before.json").read_text()), before) + self.assertEqual(json.loads((output / "source-after-prerequisites.json").read_text()), built) + self.assertEqual(set(report["prerequisite_outputs"]), set(qualification.MPC_PREREQUISITE_OUTPUTS)) + + def test_unittest_summary_preserves_skip_count_and_failed_state(self): + log = "Safety qualification collected 3590 tests\nRan 3590 tests in 33.0s\n\nOK (skipped=425)\nSafety qualification suppressed 0 methods" + self.assertEqual(qualification.parse_unittest_summary(log), + {"ran": 3590, "skipped": 425, "status": "OK", "collected": 3590, "suppressed": 0}) + self.assertEqual(qualification.parse_unittest_summary("Ran 1 test in 0.1s\n\nFAILED (failures=1)"), + {"ran": 1, "skipped": 0, "status": "FAILED", "collected": None, "suppressed": None}) + self.assertEqual(qualification.parse_unittest_summary(""), {"ran": None, "skipped": 0, "status": "missing", "collected": None, "suppressed": None}) + + def test_partial_or_all_skipped_unittest_is_not_qualification(self): + self.assertFalse(qualification.complete_unittest({"status": "OK", "collected": 2, "ran": 1, "skipped": 0, "suppressed": 0})) + self.assertFalse(qualification.complete_unittest({"status": "OK", "collected": 2, "ran": 2, "skipped": 2, "suppressed": 0})) + self.assertTrue(qualification.complete_unittest({"status": "OK", "collected": 2, "ran": 1, "skipped": 0, "suppressed": 1})) + + def test_mutation_summary_exposes_survivors_even_if_upstream_exempts_them(self): + output = "Found 4461 unique candidates\n pruned_build_incompatible: 3\n killed: 4455\n survived: 2\n infra_error: 1\n" + self.assertEqual(qualification.parse_mutation_summary(output), + {"candidates": 4461, "total": None, "killed": 4455, "survived": 2, "infra_error": 1, "pruned_build_incompatible": 3, + "proven_equivalent": 0}) + + def test_full_mutation_requires_matching_artifact_and_subprocess_success(self): + log = "\n".join(("Found 1 unique candidates", " pruned_build_incompatible: 0", " total: 1", + " killed: 1", " survived: 0", " infra_error: 0", "")) + record = self.mutation_artifact() + for code, write_artifact, expected in ((0, True, True), (1, True, False), (0, False, False)): + with self.subTest(code=code, artifact=write_artifact), tempfile.TemporaryDirectory() as td: + gate = qualification.Gate("mutation-full", Path(td), sys.executable) + + def fake_run(label, argv, *, active_gate=gate, active_code=code, should_write=write_artifact, **kwargs): + self.assertEqual(label, "mutation") + self.assertEqual(kwargs["cwd"], qualification.ROOT / "opendbc_repo") + self.assertEqual(argv[-2], "--results-json") + self.assertEqual(Path(argv[-1]), active_gate.output / "mutation-results.json") + if should_write: + Path(argv[-1]).write_text(json.dumps(record)) + return active_code, log + + with patch.object(gate, "run", side_effect=fake_run): + self.assertEqual(gate.execute(), expected) + self.assertEqual(gate.summary["artifact_valid"], write_artifact) + self.assertEqual(bool(gate.artifacts), write_artifact) + + def test_mutation_artifact_rejects_unaccounted_or_unselected_candidate(self): + summary = qualification.parse_mutation_summary("\n".join( + ("Found 1 unique candidates", " pruned_build_incompatible: 0", " total: 1", + " killed: 1", " survived: 0", " infra_error: 0", ""))) + record = self.mutation_artifact() + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "mutation-results.json" + path.write_text(json.dumps(record)) + self.assertTrue(qualification.complete_mutation_artifact(path, summary)) + record["results"][0]["selected_test_count"] = 0 + path.write_text(json.dumps(record)) + self.assertFalse(qualification.complete_mutation_artifact(path, summary)) + record["results"] = [] + path.write_text(json.dumps(record)) + self.assertFalse(qualification.complete_mutation_artifact(path, summary)) + + def test_mutation_artifact_rejects_malformed_ids_hashes_and_signal_as_kill(self): + summary = {"candidates": 1, "pruned_build_incompatible": 0, "total": 1, + "killed": 1, "survived": 0, "infra_error": 0} + original = self.mutation_artifact() + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "mutation-results.json" + for label, change in ( + ("bool id", lambda item: item["results"][0].update(site_id=False)), + ("bool version", lambda item: item.update(schema_version=True)), + ("forged source", lambda item: item["source_sha256"].update({"opendbc/safety/helpers.h": "f" * 64})), + ("forged runner", lambda item: item.update(mutation_runner_sha256="f" * 64)), + ("forged safety input", lambda item: item.update(safety_input_sha256="f" * 64)), + ("signal labeled kill", lambda item: item["results"][0].update(failure_kind="signal", exit_code=-11)), + ): + with self.subTest(label=label): + record = json.loads(json.dumps(original)) + change(record) + path.write_text(json.dumps(record)) + self.assertFalse(qualification.complete_mutation_artifact(path, summary)) + + def test_mutation_artifact_rejects_special_and_oversized_files(self): + summary = {"candidates": 1, "pruned_build_incompatible": 0, "total": 1, + "killed": 1, "survived": 0, "infra_error": 0} + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "mutation-results.json" + os.mkfifo(path) + self.assertFalse(qualification.complete_mutation_artifact(path, summary)) + path.unlink() + target = Path(td) / "target.json" + target.write_text(json.dumps(self.mutation_artifact())) + path.symlink_to(target) + self.assertFalse(qualification.complete_mutation_artifact(path, summary)) + path.unlink() + with path.open("wb") as stream: + stream.truncate(qualification.MAX_MUTATION_ARTIFACT_BYTES + 1) + self.assertFalse(qualification.complete_mutation_artifact(path, summary)) + + def test_mutation_full_rejects_preexisting_artifact_before_launch(self): + with tempfile.TemporaryDirectory() as td: + gate = qualification.Gate("mutation-full", Path(td), sys.executable) + artifact = Path(td) / "mutation-results.json" + artifact.symlink_to(Path(td) / "missing.json") + with patch.object(gate, "run") as run: + self.assertFalse(gate.execute()) + run.assert_not_called() + self.assertFalse(gate.summary["artifact_valid"]) + + def test_mutation_list_does_not_request_result_artifact(self): + with tempfile.TemporaryDirectory() as td: + gate = qualification.Gate("mutation-list", Path(td), sys.executable) + + def fake_run(label, argv, **_kwargs): + self.assertEqual(label, "mutation") + self.assertIn("--list-only", argv) + self.assertNotIn("--results-json", argv) + return 0, "Found 1 unique candidates\n" + + with patch.object(gate, "run", side_effect=fake_run): + self.assertTrue(gate.execute()) + self.assertFalse((Path(td) / "mutation-results.json").exists()) + + def test_origin_check_rejects_second_checkout(self): + good = {"opendbc": str(qualification.ROOT / "opendbc_repo/opendbc/__init__.py"), + "panda": str(qualification.ROOT / "panda/__init__.py"), + "msgq": str(qualification.ROOT / "msgq_repo/msgq/__init__.py"), + "openpilot": str(qualification.ROOT / "openpilot/__init__.py"), + "libsafety_py": str(qualification.ROOT / "opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py"), + "can_parser": str(qualification.ROOT / "opendbc_repo/opendbc/can/parser.py"), + "can_packer": str(qualification.ROOT / "opendbc_repo/opendbc/can/packer.py"), + "can_dbc": str(qualification.ROOT / "opendbc_repo/opendbc/can/dbc.py"), + "cereal_log": str(qualification.ROOT / "openpilot/cereal/log.capnp"), + "cereal_messaging": str(qualification.ROOT / "openpilot/cereal/messaging/__init__.py"), + "msgq_ipc": str(qualification.ROOT / "msgq_repo/msgq/ipc_pyx.so")} + with tempfile.TemporaryDirectory() as td: + output = Path(td) / "imports.log" + with patch.object(qualification.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, json.dumps(good), "")): + self.assertTrue(qualification.check_import_origins("python", {}, output)) + good["opendbc"] = "/data/openpilot/opendbc_repo/opendbc/__init__.py" + with patch.object(qualification.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, json.dumps(good), "")): + self.assertFalse(qualification.check_import_origins("python", {}, output)) + + @skipUnless((qualification.ROOT / ".venv/bin/python").is_file() and + (qualification.ROOT / "msgq_repo/msgq/ipc_pyx.so").is_file(), + "requires the built locked environment; actual origin checks also run in every safety gate") + def test_actual_second_checkout_import_shadow_is_rejected(self): + with tempfile.TemporaryDirectory() as td: + fake = Path(td) / "foreign" + (fake / "panda").mkdir(parents=True) + (fake / "panda/__init__.py").write_text("# second checkout shadow\n") + gate = qualification.Gate("mutation-list", Path(td), qualification.ROOT / ".venv/bin/python") + self.assertTrue(qualification.check_import_origins(gate.python, gate.env, Path(td) / "imports-baseline.log")) + gate.env["PYTHONPATH"] = str(fake) + os.pathsep + gate.env["PYTHONPATH"] + self.assertFalse(qualification.check_import_origins(gate.python, gate.env, Path(td) / "imports.log")) + + def test_mutation_stream_matches_captured_text_exit_and_summary(self): + script = ("import os,sys,time; " + + "os.write(1,b'out\\r'); time.sleep(0.02); os.write(1,b'\\n'); " + + "os.write(2,b'err\\r\\n'); os.write(1,b'\\xe2'); time.sleep(0.02); " + + "os.write(1,b'\\x82\\xac\\nFound 2 unique candidates\\n total: 2\\n killed: 2\\n'); " + + "sys.exit(int(sys.argv[1]))") + for name, exit_code in (("mutation-full", 0), ("mutation-list", 7), ("coverage", 0)): + with self.subTest(gate=name), tempfile.TemporaryDirectory() as td: + gate = qualification.Gate(name, Path(td), sys.executable) + argv = [sys.executable, "-c", script, str(exit_code)] + baseline = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, env=gate.env, cwd=td, check=False) + visible = io.StringIO() + with redirect_stdout(visible): + code, text = gate.run("mutation", argv, cwd=td) + self.assertEqual((code, text), (baseline.returncode, baseline.stdout)) + self.assertEqual((Path(td) / "mutation.log").read_text(), text) + self.assertEqual(visible.getvalue(), text if name.startswith("mutation-") else "") + self.assertEqual(qualification.parse_mutation_summary(text), qualification.parse_mutation_summary(baseline.stdout)) + self.assertEqual(gate.commands[0]["exit_code"], exit_code) + + def test_mutation_progress_is_visible_and_logged_before_child_exits(self): + ready = threading.Event() + + class Visible(io.StringIO): + def write(self, text): + count = super().write(text) + if "ready" in self.getvalue(): + ready.set() + return count + + with tempfile.TemporaryDirectory() as td: + output = Path(td) + release = output / "release" + script = ("import pathlib,sys,time; p=pathlib.Path(sys.argv[1]); " + + "sys.stdout.write('ready'); sys.stdout.flush(); " + + "exec('while not p.exists(): time.sleep(0.01)'); print(' done',flush=True)") + gate = qualification.Gate("mutation-full", output, sys.executable) + results, errors = [], [] + + def invoke(): + try: + results.append(gate.run("mutation", [sys.executable, "-c", script, str(release)], cwd=td, timeout=5)) + except Exception as error: + errors.append(error) + + worker = threading.Thread(target=invoke) + with redirect_stdout(Visible()): + worker.start() + try: + self.assertTrue(ready.wait(3), "No progress until child exit") + self.assertTrue(worker.is_alive()) + self.assertEqual((output / "mutation.log").read_text(), "ready") + finally: + release.touch() + worker.join(6) + self.assertFalse(worker.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(results, [(0, "ready done\n")]) + + def test_mutation_timeout_kills_descendant_group_and_keeps_partial_text(self): + with tempfile.TemporaryDirectory() as td: + output = Path(td) + marker = output / "descendant-survived" + descendant = ("import pathlib,sys,time; p=pathlib.Path(sys.argv[1]); " + + "p.with_suffix('.ready').write_text('ready'); time.sleep(3); p.write_text('alive')") + script = ("import pathlib,subprocess,sys,time; " + + "subprocess.Popen([sys.executable,'-c',sys.argv[1],sys.argv[2]]); " + + "p=pathlib.Path(sys.argv[2]).with_suffix('.ready'); " + + "exec('while not p.exists(): time.sleep(0.01)'); print('partial',flush=True); time.sleep(10)") + gate = qualification.Gate("mutation-full", output, sys.executable) + original_popen = subprocess.Popen + processes = [] + + def spawn(*args, **kwargs): + process = original_popen(*args, **kwargs) + processes.append(process) + return process + + visible = io.StringIO() + start = time.monotonic() + with redirect_stdout(visible), patch.object(qualification.subprocess, "Popen", side_effect=spawn): + code, text = gate.run("mutation", [sys.executable, "-c", script, descendant, str(marker)], cwd=td, timeout=1) + self.assertLess(time.monotonic() - start, 2.5) + self.assertTrue(marker.with_suffix(".ready").exists()) + self.assertEqual(code, 124) + self.assertEqual(processes[0].returncode, -signal.SIGKILL) + self.assertEqual(text, "partial\n\nTimed out; process group killed\n") + self.assertEqual(visible.getvalue(), text) + self.assertEqual((output / "mutation.log").read_text(), text) + self.assertEqual(gate.commands[0]["exit_code"], 124) + time.sleep(3.2) + self.assertFalse(marker.exists(), "Descendant survived timeout group cleanup") + + def test_mutation_sink_failure_cleans_up_child_before_returning_error(self): + class Broken(io.StringIO): + def write(self, text): + raise BrokenPipeError("sink closed") + + with tempfile.TemporaryDirectory() as td: + gate = qualification.Gate("mutation-full", Path(td), sys.executable) + original_popen = subprocess.Popen + processes = [] + + def spawn(*args, **kwargs): + process = original_popen(*args, **kwargs) + processes.append(process) + return process + + script = "import time; print('partial',flush=True); time.sleep(10)" + with redirect_stdout(Broken()), patch.object(qualification.subprocess, "Popen", side_effect=spawn): + code, text = gate.run("mutation", [sys.executable, "-c", script], cwd=td, timeout=2) + self.assertEqual((code, text), (127, "BrokenPipeError: sink closed\n")) + self.assertEqual(processes[0].returncode, -signal.SIGKILL) + self.assertTrue(processes[0].stdout.closed) + self.assertEqual((Path(td) / "mutation.log").read_text(), text) + + def test_mutation_log_open_failure_does_not_launch_child(self): + with tempfile.TemporaryDirectory() as td: + output = Path(td) + gate = qualification.Gate("mutation-full", output, sys.executable) + (output / "mutation.log").mkdir() + marker = output / "child-started" + script = "import pathlib,sys; pathlib.Path(sys.argv[1]).write_text('started')" + with patch.object(qualification.subprocess, "Popen", wraps=subprocess.Popen) as spawn: + with self.assertRaises(IsADirectoryError): + gate.run("mutation", [sys.executable, "-c", script, str(marker)], cwd=td) + spawn.assert_not_called() + self.assertFalse(marker.exists()) + + def test_timeout_fails_and_records_bounded_log(self): + with tempfile.TemporaryDirectory() as td: + output = Path(td) + gate = qualification.Gate("coverage", output, "/usr/bin/python3") + code, text = gate.run("timeout", [sys.executable, "-c", "import time; print('partial', flush=True); time.sleep(10)"], timeout=0.05) + self.assertEqual(code, 124) + self.assertIn("Timed out", text) + self.assertIn("partial", (output / "timeout.log").read_text()) + self.assertEqual(gate.commands[0]["exit_code"], 124) + + def test_cppcheck_wrong_version_is_not_accepted(self): + with tempfile.TemporaryDirectory() as td: + gate = qualification.Gate("opendbc-misra", Path(td), "/usr/bin/python3") + with patch.object(gate, "run", side_effect=[(0, "/tmp/cppcheck\n"), (0, "Cppcheck 2.20\n")]): + self.assertIsNone(gate.cppcheck_dir()) + + def test_misra_runs_both_debug_and_release(self): + with tempfile.TemporaryDirectory() as td: + gate = qualification.Gate("opendbc-misra", Path(td), "/usr/bin/python3") + table = (qualification.ROOT / "opendbc_repo/opendbc/safety/tests/misra/coverage_table").read_text() + calls = [] + + def fake_run(label, argv, **_kwargs): + calls.append((label, argv)) + if label == "misra-table": + return 0, table + if label == "compiler-include": + return 0, "/usr/include\n" + return 0, "Checking...\n" + + with patch.object(gate, "cppcheck_dir", return_value=Path("/tmp/cppcheck")), patch.object(gate, "run", side_effect=fake_run): + self.assertTrue(gate.misra(False)) + variants = {label: argv for label, argv in calls if label.startswith("misra-") and label != "misra-table"} + self.assertEqual(set(variants), {"misra-debug", "misra-release"}) + self.assertIn("-DALLOW_DEBUG", variants["misra-debug"]) + self.assertNotIn("-DALLOW_DEBUG", variants["misra-release"]) + + def test_exception_still_writes_failed_results(self): + with tempfile.TemporaryDirectory() as td: + output = Path(td) / "evidence" + root = Path(td) / "checkout" + (root / ".venv/bin").mkdir(parents=True) + (root / ".venv/bin/python").symlink_to(sys.executable) + source = {"source.py": "hash", **dict.fromkeys(qualification.MPC_PREREQUISITE_OUTPUTS, "generated-hash")} + with patch.object(sys, "argv", ["run_safety_qualification.py", "--gate", "coverage", "--output", str(output)]), \ + patch.object(qualification, "ROOT", root), \ + patch.object(qualification, "source_identity", return_value=source), \ + patch.object(qualification, "source_head", return_value="commit"), \ + patch.object(qualification, "generated_identity", return_value={}), \ + patch.object(qualification, "prerequisite_artifact_identity", return_value={"fixture.so": "hash"}), \ + patch.object(qualification.Gate, "run", return_value=(0, "")) as invoke, \ + patch.object(qualification, "check_import_origins", return_value=True), \ + patch.object(qualification, "imported_module_hashes", return_value={}), \ + patch.object(qualification.Gate, "execute", side_effect=RuntimeError("fixture crash")): + self.assertEqual(qualification.main(), 1) + self.assertIn("openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so", + invoke.call_args_list[0].args[1]) + report = json.loads((output / "results.json").read_text()) + self.assertFalse(report["passed"]) + self.assertIn("fixture crash", report["error"]) diff --git a/tools/ci/tests/test_shebang_format.py b/tools/ci/tests/test_shebang_format.py new file mode 100644 index 0000000000..0c52bad7f5 --- /dev/null +++ b/tools/ci/tests/test_shebang_format.py @@ -0,0 +1,29 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + + +CHECKER = Path(__file__).resolve().parents[3] / 'scripts/lint/check_shebang_format.sh' + + +class ShebangFormatTest(unittest.TestCase): + def check(self, content: bytes) -> subprocess.CompletedProcess: + with tempfile.TemporaryDirectory() as directory: + script = Path(directory) / 'script with spaces' + script.write_bytes(content) + return subprocess.run(['bash', str(CHECKER), str(script)], capture_output=True, text=True) + + def test_binary_payload_with_embedded_shebang_is_not_a_script(self): + result = self.check(b'\x7fELF\x00\n#!/usr/bin/python\n') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_text_script_format_is_still_enforced(self): + for language in ('python3', 'bash'): + with self.subTest(language=language): + self.assertEqual(self.check(f'#!/usr/bin/env {language}\n'.encode()).returncode, 0) + self.assertNotEqual(self.check(f'#!/usr/bin/{language}\n'.encode()).returncode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/tests/test_vehicle_ci.py b/tools/ci/tests/test_vehicle_ci.py new file mode 100644 index 0000000000..5d961f1d52 --- /dev/null +++ b/tools/ci/tests/test_vehicle_ci.py @@ -0,0 +1,205 @@ +from contextlib import ExitStack, redirect_stdout +import io +import json +from pathlib import Path +import sys +import subprocess +import tempfile +from types import ModuleType +import unittest +from unittest.mock import patch + +from tools import test_runner +from tools.ci import run_vehicle_tests +from tools.ci.run_vehicle_tests import audit_interface_modules, interface_coverage, interface_execution_errors, targets_for +from tools.ci.record_firmware import compile_commands + + +class TestVehicleCollection(unittest.TestCase): + def test_source_snapshot_catches_uncommitted_and_untracked_edits(self): + with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, 'ROOT', Path(directory)): + root = Path(directory) + source = root / 'opendbc_repo/car.py' + source.parent.mkdir() + source.write_text('first\n') + + def git(*args): + subprocess.run(['git', *args], cwd=root, check=True, capture_output=True) + + git('init', '-q') + git('add', 'opendbc_repo/car.py') + git('-c', 'user.name=CI Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'fixture') + original = run_vehicle_tests.source_input_snapshot() + source.write_text('second\n') + modified = run_vehicle_tests.source_input_snapshot() + self.assertEqual(original['revision'], modified['revision']) + self.assertNotEqual(original['tracked_patch_sha256'], modified['tracked_patch_sha256']) + source.write_text('first\n') + self.assertEqual(run_vehicle_tests.source_input_snapshot(), original) + added = source.with_name('new_test.py') + added.write_text('first\n') + untracked = run_vehicle_tests.source_input_snapshot() + added.write_text('second\n') + self.assertNotEqual(untracked['untracked_sha256'], run_vehicle_tests.source_input_snapshot()['untracked_sha256']) + + def test_passed_tests_with_changed_source_fail_the_gate_and_keep_outcomes(self): + test_id = 'fixture.TestCarInterfaces.test_car_interfaces_A' + test = unittest.FunctionTestCase(lambda: None) + with tempfile.TemporaryDirectory() as directory, \ + patch.object(test, 'id', return_value=test_id), \ + patch.object(run_vehicle_tests, 'source_provenance', side_effect=[{'inputs': 'before'}, {'inputs': 'after'}]), \ + patch.object(run_vehicle_tests, 'vehicle_inventory', return_value={'platform_ids': ['A'], 'platform_count': 1}), \ + patch.object(run_vehicle_tests, 'native_provenance', return_value={}), \ + patch.object(run_vehicle_tests, 'audit_interface_modules', return_value=([], [])), \ + patch.object(run_vehicle_tests.vehicle_pytest, "split_targets", return_value=(["fixture.py"], [])), \ + patch.object(run_vehicle_tests.vehicle_pytest, "collect", return_value=([], [])), \ + patch.object(test_runner, 'collect', return_value=([test], [])), \ + patch.object(test_runner, 'run_batch', return_value=[test_runner.make_record(test_id)]), redirect_stdout(io.StringIO()): + self.assertEqual(run_vehicle_tests.run('interfaces', directory), 1) + result = json.loads((Path(directory) / 'results.json').read_text()) + coverage = json.loads((Path(directory) / 'coverage.json').read_text()) + self.assertEqual(result['results'][0]['status'], 'passed') + self.assertEqual(result['status'], 'failed') + self.assertIn('source inputs changed', result['gate_errors'][0]) + self.assertEqual(coverage['source_after']['inputs'], 'after') + + def test_exact_platform_collection(self): + ids = [f"fixture.TestCarInterfaces.test_car_interfaces_{platform}" for platform in ("A", "B")] + self.assertEqual(interface_coverage(ids, {"A", "B"}), dict(zip(("A", "B"), ids, strict=True))) + + def test_missing_extra_duplicate_and_abstract_cannot_satisfy_platforms(self): + good = "fixture.TestCarInterfaces.test_car_interfaces_A" + for ids in ([], [good, good], [good, good.replace("_A", "_B")], ["fixture.TestCarModelBase.test_car_params"]): + with self.subTest(ids=ids), self.assertRaises(ValueError): + interface_coverage(ids, {"A"}) + + def test_concrete_platforms_require_passed_outcomes(self): + mapping = {"A": "fixture.A", "B": "fixture.B"} + for status in ("skipped", "xfailed", "failed", "error"): + with self.subTest(status=status): + records = [test_runner.make_record("fixture.A"), test_runner.make_record("fixture.B", status)] + self.assertEqual(len(interface_execution_errors(mapping, records)), 1) + self.assertEqual(len(interface_execution_errors(mapping, [])), 2) + self.assertEqual(interface_execution_errors(mapping, [test_runner.make_record(test_id) for test_id in mapping.values()]), []) + + def test_class_skip_preserved_but_cannot_pass_interface_gate(self): + module = ModuleType("vehicle_skip_fixture") + + @classmethod + def set_up_class(cls): + raise unittest.SkipTest("fixture class unavailable") + + module.TestCarInterfaces = type("TestCarInterfaces", (unittest.TestCase,), { + "__module__": module.__name__, "setUpClass": set_up_class, "test_car_interfaces_A": lambda self: self.fail("unreachable")}) + tests = list(test_runner.flatten(unittest.defaultTestLoader.loadTestsFromModule(module))) + with tempfile.TemporaryDirectory() as directory, patch.dict(sys.modules, {module.__name__: module}), \ + patch.object(run_vehicle_tests, "source_provenance", return_value={}), \ + patch.object(run_vehicle_tests, "vehicle_inventory", return_value={"platform_ids": ["A"], "platform_count": 1}), \ + patch.object(run_vehicle_tests, "native_provenance", return_value={}), \ + patch.object(run_vehicle_tests, "audit_interface_modules", return_value=([], [])), \ + patch.object(run_vehicle_tests.vehicle_pytest, "split_targets", return_value=(["fixture.py"], [])), \ + patch.object(run_vehicle_tests.vehicle_pytest, "collect", return_value=([], [])), \ + patch.object(test_runner, "collect", return_value=(tests, [])), redirect_stdout(io.StringIO()): + self.assertEqual(run_vehicle_tests.run("interfaces", directory), 1) + result = json.loads((Path(directory) / "results.json").read_text()) + coverage = json.loads((Path(directory) / "coverage.json").read_text()) + self.assertEqual(result["results"][0]["status"], "skipped") + self.assertIn("fixture class unavailable", result["results"][0]["detail"]) + self.assertEqual(result["status"], "failed") + self.assertEqual(len(result["gate_errors"]), 1) + self.assertEqual(coverage["interface_results"]["A"]["status"], "skipped") + + def test_initialization_and_collection_failures_leave_error_artifacts(self): + for failing in ("source_provenance", "vehicle_inventory", "native_provenance", "collect"): + for collect_only in (False, True): + with self.subTest(failing=failing, collect_only=collect_only), tempfile.TemporaryDirectory() as directory, ExitStack() as stack: + output = Path(directory) + + def fail(output=output): + self.assertTrue((output / "plan.json").is_file()) + raise ImportError("missing native fixture dependency") + + for name in ("source_provenance", "vehicle_inventory", "native_provenance"): + value = {"platform_ids": ["A"], "platform_count": 1} if name == "vehicle_inventory" else {} + stack.enter_context(patch.object(run_vehicle_tests, name, side_effect=fail if name == failing else None, return_value=value)) + stack.enter_context(patch.object(run_vehicle_tests.vehicle_pytest, "split_targets", return_value=(["fixture.py"], []))) + stack.enter_context(patch.object(run_vehicle_tests.vehicle_pytest, "collect", return_value=([], []))) + if failing == "collect": + stack.enter_context(patch.object(test_runner, "collect", side_effect=ImportError("fixture collection failed"))) + stack.enter_context(redirect_stdout(io.StringIO())) + self.assertEqual(run_vehicle_tests.run("interfaces", output, collect_only), 1) + for name in ("plan.json", "results.json", "coverage.json"): + artifact = json.loads((output / name).read_text()) + self.assertEqual(artifact["status"], "error", name) + self.assertEqual(artifact["exit_code"], 1, name) + self.assertTrue(artifact["collection_errors"], name) + self.assertEqual(artifact["error_phase"], "collection" if failing == "collect" else "initialization", name) + + def test_native_safety_targets_are_top_level_and_complete(self): + root = Path(__file__).resolve().parents[3] / "opendbc_repo/opendbc/safety/tests" + self.assertEqual({Path(target).name for target in targets_for("safety-debug")}, {path.name for path in root.glob("test_*.py")}) + self.assertTrue(all(Path(target).parent.name == "tests" for target in targets_for("safety-debug"))) + + def test_interfaces_do_not_mislabel_historical_route_test_base(self): + root = Path(__file__).resolve().parents[3] + car = root / "opendbc_repo/opendbc/car" + expected = {str(path.relative_to(root)) for path in car.rglob("test_*.py") if path != car / "tests/test_models.py"} + self.assertEqual(set(targets_for("interfaces")), expected) + self.assertIn("opendbc_repo/opendbc/car/tesla/tests/test_tesla.py", expected) + self.assertIn("opendbc_repo/opendbc/car/tests/test_car_interfaces.py", expected) + + def test_recursive_collection_excludes_only_the_known_route_module(self): + with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)): + for name in ("tests/test_models.py", "brand/tests/test_models.py", "brand/nested/test_feature.py"): + path = Path(directory) / "opendbc_repo/opendbc/car" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + self.assertEqual(targets_for("interfaces"), [ + "opendbc_repo/opendbc/car/brand/nested/test_feature.py", "opendbc_repo/opendbc/car/brand/tests/test_models.py"]) + + def test_module_audit_accepts_collected_generated_methods(self): + with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)): + path = Path(directory) / "test_fixture.py" + path.write_text("class TestCar:\n def test_parameters(self): pass\n") + modules, errors = audit_interface_modules([path.name], ["test_fixture.TestCar.test_parameters_A", "test_fixture.TestCar.test_parameters_B"]) + self.assertEqual(errors, []) + self.assertEqual(modules[0]["collected_count"], 2) + self.assertEqual(modules[0]["uncollected_test_methods"], []) + + def test_module_audit_rejects_uncollected_pytest_functions_and_methods(self): + with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)): + path = Path(directory) / "test_fixture.py" + path.write_text("def test_function(): pass\nclass TestPytest:\n def test_method(self): pass\n") + modules, errors = audit_interface_modules([path.name], []) + self.assertEqual(len(errors), 3) + self.assertEqual(modules[0]["module_level_test_functions"], ["test_function"]) + self.assertEqual(modules[0]["uncollected_test_methods"], ["TestPytest.test_method"]) + # A supported class elsewhere in the same module cannot hide these omissions. + _, errors = audit_interface_modules([path.name], ["test_fixture.TestUnit.test_present"]) + self.assertEqual(len(errors), 2) + + def test_runner_reloads_dynamic_class_ids_in_serial_execution(self): + module = ModuleType("vehicle_collection_fixture") + + def test_generated(self): + self.assertTrue(True) + + module.Generated = type("Generated", (unittest.TestCase,), {"__module__": module.__name__, "test_generated": test_generated}) + with patch.dict(sys.modules, {module.__name__: module}): + selected = [test.id() for test in test_runner.flatten(unittest.defaultTestLoader.loadTestsFromModule(module))] + records = test_runner.run_batch(selected, capture_output=True) + self.assertEqual([record["id"] for record in records], selected) + self.assertEqual([record["status"] for record in records], ["passed"]) + + def test_firmware_requires_actual_matching_compile_flags(self): + release = "arm-none-eabi-gcc -o panda/board/obj/panda_h7/main.o -c panda/board/main.c" + debug = release + " -DALLOW_DEBUG" + self.assertEqual(compile_commands(release, "release"), [release.split()]) + self.assertEqual(compile_commands(debug, "debug"), [debug.split()]) + for log, variant in (("scons: up to date", "release"), (release, "debug"), (debug, "release")): + with self.subTest(log=log, variant=variant), self.assertRaises(ValueError): + compile_commands(log, variant) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci/tests/test_vehicle_pytest.py b/tools/ci/tests/test_vehicle_pytest.py new file mode 100644 index 0000000000..7b8b3a70ef --- /dev/null +++ b/tools/ci/tests/test_vehicle_pytest.py @@ -0,0 +1,124 @@ +import tempfile +import textwrap +import os +from unittest.mock import patch +from pathlib import Path +from types import SimpleNamespace +import unittest + +from tools.ci.vehicle_pytest import Recorder, collect, run, split_targets, test_id as normalize_test_id + + +class TestVehiclePytest(unittest.TestCase): + def test_function_and_mixed_files_selected_once(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = {"test_class.py": "class TestOrdinary(unittest.TestCase):\n def test_one(self): pass\n", + "test_function.py": "def test_one(): pass\n", + "test_mixed.py": "class TestOrdinary(unittest.TestCase):\n def test_one(self): pass\ndef test_two(): pass\n", + "test_plain.py": "class TestPlain:\n def test_one(self): pass\n"} + for name, source in fixtures.items(): + (root / name).write_text(source) + unit, functions = split_targets(root, [*fixtures, "test_mixed.py"]) + self.assertEqual(unit, ["test_class.py"]) + self.assertEqual(functions, ["test_function.py", "test_mixed.py", "test_plain.py"]) + + def test_parameterized_id_retains_case_identity(self): + self.assertEqual(normalize_test_id("opendbc_repo/opendbc/car/honda/tests/test_stock.py::test_cp[HONDA-A]"), + "opendbc_repo.opendbc.car.honda.tests.test_stock.test_cp[HONDA-A]") + + def test_records_all_outcomes_and_fixture_failure(self): + for expected, outcome, phase, xfail, detail in ( + ("passed", "passed", "call", None, None), ("failed", "failed", "call", None, "assertion"), + ("error", "failed", "setup", None, "fixture"), ("skipped", "skipped", "setup", None, "skip"), + ("xfailed", "skipped", "call", "known", "xfail"), ("xpassed", "passed", "call", "known", None), + ("xpassed", "failed", "call", None, "[XPASS(strict)] known")): + with self.subTest(expected=expected): + recorder = Recorder() + report = SimpleNamespace(nodeid="test_example.py::test_one", duration=0.25, when=phase, + failed=outcome == "failed", skipped=outcome == "skipped", longrepr=detail, + sections=[("Captured stdout call", "output"), ("Captured stderr call", "error")]) + if xfail: + report.wasxfail = xfail + recorder.pytest_runtest_logreport(report) + record = recorder.records["test_example.test_one"] + self.assertEqual(record["status"], expected) + self.assertEqual(record["time"], 0.25) + self.assertEqual(record["stdout"], "output") + self.assertEqual(record["stderr"], "error") + recorder.pytest_runtest_logreport(SimpleNamespace(nodeid=report.nodeid, duration=0.1, when="teardown", + failed=False, skipped=False, longrepr=None, sections=[])) + self.assertEqual(record["status"], expected) + + def test_imported_classes_deselected_but_inherited_methods_remain(self): + module = SimpleNamespace(__name__="local") + local = SimpleNamespace(module=module, cls=SimpleNamespace(__module__="local")) + imported = SimpleNamespace(module=module, cls=SimpleNamespace(__module__="other")) + deselected = [] + config = SimpleNamespace(hook=SimpleNamespace(pytest_deselected=lambda items: deselected.extend(items))) + items = [local, imported] + Recorder().pytest_collection_modifyitems(None, config, items) + self.assertEqual(items, [local]) + self.assertEqual(deselected, [imported]) + + def test_teardown_error_overrides_pass(self): + recorder = Recorder() + for phase, failed in (("call", False), ("teardown", True)): + recorder.pytest_runtest_logreport(SimpleNamespace(nodeid="test_file.py::test_case", duration=0, when=phase, + failed=failed, skipped=False, longrepr="cleanup" if failed else None, sections=[])) + self.assertEqual(recorder.records["test_file.test_case"]["status"], "error") + + def test_real_mixed_collection_and_execution_stays_serial(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "pytest.ini").write_text("[pytest]\naddopts = -n 4\n") + (root / "test_mixed_vehicle_fixture.py").write_text(textwrap.dedent(f"""\ + import os, unittest, pytest + EXPECTED_PID = {os.getpid()} + class TestUnit(unittest.TestCase): + def test_once(self): + assert os.getpid() == EXPECTED_PID + print('unit-once') + @pytest.mark.parametrize('value', [1, 2]) + def test_parameter(value): + assert os.getpid() == EXPECTED_PID + @pytest.mark.skip(reason='fixture skip') + def test_skip(): pass + @pytest.mark.xfail(reason='known fixture') + def test_xfail(): assert False + @pytest.fixture + def broken(): raise RuntimeError('fixture failed') + def test_error(broken): pass + def test_failure(): assert False + """)) + previous = Path.cwd() + try: + os.chdir(root) + with patch.dict(os.environ, {"PYTEST_ADDOPTS": "-n 8", "PYTEST_PLUGINS": "nonexistent_vehicle_plugin"}): + unit, functions = split_targets(root, ["test_mixed_vehicle_fixture.py"] * 2) + self.assertEqual(unit, []) + self.assertEqual(functions, ["test_mixed_vehicle_fixture.py"]) + nodeids, errors = collect(functions) + self.assertEqual(errors, []) + self.assertEqual(len(nodeids), 7) + records, errors = run(nodeids) + self.assertEqual(errors, []) + self.assertEqual(os.environ["PYTEST_ADDOPTS"], "-n 8") + self.assertEqual(os.environ["PYTEST_PLUGINS"], "nonexistent_vehicle_plugin") + self.assertEqual(len(records), 7) + self.assertEqual(len({record["id"] for record in records}), 7) + self.assertCountEqual([record["status"] for record in records], + ["passed", "passed", "passed", "skipped", "xfailed", "error", "failed"]) + unit_record = next(record for record in records if record["id"].endswith("TestUnit.test_once")) + self.assertEqual(unit_record["stdout"].count("unit-once"), 1) + self.assertTrue(any("fixture failed" in record["detail"] for record in records)) + finally: + os.chdir(previous) + + def test_worker_plugin_configuration_fails_closed(self): + import pytest + for config in (SimpleNamespace(workerinput={}, option=SimpleNamespace()), + SimpleNamespace(option=SimpleNamespace(numprocesses=2)), + SimpleNamespace(option=SimpleNamespace(dist="load"))): + with self.subTest(config=config), self.assertRaises(pytest.UsageError): + Recorder().pytest_configure(config) diff --git a/tools/ci/vehicle_pytest.py b/tools/ci/vehicle_pytest.py new file mode 100644 index 0000000000..40981704eb --- /dev/null +++ b/tools/ci/vehicle_pytest.py @@ -0,0 +1,122 @@ +import ast +from contextlib import contextmanager +import os +from pathlib import Path + + +def split_targets(root, targets): + unittest_targets, pytest_targets = [], [] + for target in dict.fromkeys(targets): + path = root / target.split("::", 1)[0] + tree = ast.parse(path.read_text(), filename=str(path)) + requires_pytest = any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_") or + isinstance(node, ast.ClassDef) and node.name.startswith("Test") and not node.bases + for node in tree.body) + (pytest_targets if requires_pytest else unittest_targets).append(target) + return unittest_targets, pytest_targets + + +def test_id(nodeid): + path, *nodes = nodeid.split("::") + return ".".join((Path(path).with_suffix("").as_posix().replace("/", "."), *nodes)) + + +class Recorder: + def __init__(self): + self.nodeids = [] + self.errors = [] + self.records = {} + + def pytest_configure(self, config): + if hasattr(config, "workerinput") or getattr(config.option, "numprocesses", None) not in (None, 0) or getattr(config.option, "dist", "no") != "no": + import pytest + raise pytest.UsageError("vehicle tests require serial execution in the pinned native process") + + def pytest_collection_modifyitems(self, session, config, items): + selected, imported = [], [] + for item in items: + owner = item.cls if item.cls is not None else item.obj + (selected if getattr(owner, "__module__", None) == item.module.__name__ else imported).append(item) + if imported: + config.hook.pytest_deselected(items=imported) + items[:] = selected + + def pytest_collection_finish(self, session): + self.nodeids = [item.nodeid for item in session.items] + + def pytest_collectreport(self, report): + if report.failed: + self.errors.append(str(report.longrepr)) + + def pytest_runtest_logreport(self, report): + identity = test_id(report.nodeid) + record = self.records.setdefault(identity, {"id": identity, "status": "passed", "detail": "", + "time": 0.0, "stdout": "", "stderr": ""}) + record["time"] += report.duration + xfail = getattr(report, "wasxfail", None) + if report.failed: + status = "xpassed" if report.when == "call" and str(report.longrepr).startswith("[XPASS(strict)]") else "failed" if report.when == "call" else "error" + elif report.skipped: + status = "xfailed" if xfail else "skipped" + elif report.when == "call" and xfail: + status = "xpassed" + else: + status = "passed" + priorities = {"passed": 0, "skipped": 1, "xfailed": 1, "xpassed": 2, "failed": 3, "error": 4} + if priorities[status] >= priorities[record["status"]]: + record["status"] = status + if status != "passed": + record["detail"] = str(report.longrepr) if report.longrepr else str(xfail or "") + for name, content in report.sections: + key = "stderr" if "stderr" in name else "stdout" if "stdout" in name else None + if key and content not in record[key]: + record[key] += content + + +@contextmanager +def serial_environment(): + values = {"PYTEST_ADDOPTS": "", "PYTEST_PLUGINS": "", "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"} + saved = {key: os.environ.get(key) for key in values} + os.environ.update(values) + try: + yield + finally: + for key, value in saved.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def invoke(arguments, recorder): + import pytest + with serial_environment(): + return pytest.main(["-q", "--rootdir=.", "-o", "addopts=", "-p", "no:xdist", "-p", "no:xdist.plugin", *arguments], plugins=[recorder]) + + +def collect(targets): + if not targets: + return [], [] + recorder = Recorder() + code = invoke(["--collect-only", *targets], recorder) + errors = list(recorder.errors) + if code != 0: + errors.append(f"pytest collection exited {code}") + ids = [test_id(nodeid) for nodeid in recorder.nodeids] + if len(set(ids)) != len(ids): + errors.append("pytest collected duplicate test IDs") + return recorder.nodeids, errors + + +def run(nodeids): + if not nodeids: + return [], [] + recorder = Recorder() + code = invoke(nodeids, recorder) + errors = list(recorder.errors) + if recorder.nodeids != nodeids: + errors.append("pytest execution collection changed from the selected IDs") + if code not in (0, 1): + errors.append(f"pytest execution exited {code}") + return list(recorder.records.values()), errors diff --git a/tools/laptop_device_build/jetlink_artifacts.py b/tools/laptop_device_build/jetlink_artifacts.py new file mode 100644 index 0000000000..02706d3361 --- /dev/null +++ b/tools/laptop_device_build/jetlink_artifacts.py @@ -0,0 +1,110 @@ +"""Separate, source-bound QCOM warp package for external Jetlink inference.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import stat + +from tools.laptop_device_build.jetlink_pickle import require_jetlink_qcom_pickle + +CAMERAS = ((1344, 760), (1928, 1208)) +MODEL_SIZES = ((512, 256), (1024, 512)) +MODEL_DIR = Path('openpilot/selfdrive/modeld/models') + + +def warp_names() -> tuple[str, ...]: + return tuple(f'jetlink_warp_{cw}x{ch}_{mw}x{mh}.pkl' for cw, ch in CAMERAS for mw, mh in MODEL_SIZES) + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def sources(root: Path) -> dict[str, str]: + files = [root / 'openpilot/starpilot/models/jetlink_warp.py', + root / 'jetlink_repo/jetlink/openpilot/warp.py', + root / 'jetlink_repo/jetlink/openpilot/interface.py', + root / 'openpilot/common/transformations/camera.py', + root / 'openpilot/common/transformations/model.py', + root / 'openpilot/system/camerad/cameras/nv12_info.py'] + for folder in ('tinygrad_repo/tinygrad', 'tinygrad_repo/extra', 'tinygrad_repo/examples/openpilot'): + files.extend(p for p in (root / folder).rglob('*') if p.is_file() and p.suffix in ('.py', '.c', '.h', '.cl') and '__pycache__' not in p.parts) + if not (root / 'jetlink_repo/jetlink/openpilot/warp.py').is_file(): + raise ValueError('Pinned Jetlink warp sources are required') + return {p.relative_to(root).as_posix(): sha(p) for p in sorted(files)} + + +def create(root: Path, package: Path) -> None: + package.mkdir(parents=True, exist_ok=True) + rows = {} + for name in warp_names(): + path = root / MODEL_DIR / name + require_jetlink_qcom_pickle(path) + rows[name] = {'sha256': sha(path), 'size': path.stat().st_size} + shutil.copyfile(path, package / name) + (package / 'manifest.json').write_text(json.dumps({'version': 1, 'sources': sources(root), 'artifacts': rows}, indent=2) + '\n') + + + +RUNTIME_HELPER = 'tinygrad_repo/examples/openpilot/helpers.py' +RUNTIME_HELPER_BEFORE = '7849829700c575339de7d95ffa869507ad2d65143af4c23a4411ad4d95ba3ae2' +RUNTIME_HELPER_AFTER = '6494f23757a1058d8c90db8434004e8f703bb61d55903ac4500c04a7d511c8a3' +RUNTIME_PROFILE = 'unused-empty-arena-loader-v1' + + +def source_signature(source: dict[str, str]) -> str: + return hashlib.sha256(json.dumps(source, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def _compatible_sources(package: Path, manifest: dict, current: dict[str, str]) -> bool: + saved = manifest.get('sources') + if saved == current: + return True + if not isinstance(saved, dict) or saved.get(RUNTIME_HELPER) != RUNTIME_HELPER_BEFORE or current.get(RUNTIME_HELPER) != RUNTIME_HELPER_AFTER: + return False + if {**current, RUNTIME_HELPER: RUNTIME_HELPER_BEFORE} != saved: + return False + sidecar = package / 'runtime-compatibility.json' + try: + info = sidecar.lstat() + if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= 4096: + return False + expected = {'version': 1, 'profile': RUNTIME_PROFILE, + 'build_manifest_sha256': sha(package / 'manifest.json'), + 'compatible_source_sha256': source_signature(current)} + extension = json.loads(sidecar.read_text()) + return type(extension) is dict and type(extension.get('version')) is int and extension == expected + except (OSError, ValueError): + return False + + +def import_artifact(package: Path, root: Path, target: Path) -> None: + manifest = json.loads((package / 'manifest.json').read_text()) + if manifest.get('version') != 1 or not _compatible_sources(package, manifest, sources(root)): + raise ValueError('Jetlink QCOM warp source closure differs; compile a current package on supported QCOM hardware') + if set(manifest.get('artifacts', {})) != set(warp_names()): + raise ValueError('Jetlink QCOM warp package has an incomplete geometry set') + if target.parent.resolve() != (root / MODEL_DIR).resolve() or target.name not in warp_names(): + raise ValueError('Unexpected Jetlink warp target') + row = manifest['artifacts'][target.name] + artifact = package / target.name + if row != {'sha256': sha(artifact), 'size': artifact.stat().st_size}: + raise ValueError('Jetlink QCOM warp artifact digest or size differs') + require_jetlink_qcom_pickle(artifact) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(artifact, target) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=Path.cwd()) + parser.add_argument('--package', type=Path, required=True) + args = parser.parse_args() + create(args.root.resolve(), args.package.resolve()) + + +if __name__ == '__main__': + main() diff --git a/tools/laptop_device_build/jetlink_pickle.py b/tools/laptop_device_build/jetlink_pickle.py new file mode 100644 index 0000000000..2b6673b0eb --- /dev/null +++ b/tools/laptop_device_build/jetlink_pickle.py @@ -0,0 +1,158 @@ +"""Inspect bare Jetlink captures with inert pickle globals; never run their code.""" +import ast +import io +import pickle +from pathlib import Path +import re + +GLOBALS = { + 'tinygrad.engine.jit._TinyJit', 'tinygrad.engine.jit.CapturedJit', 'tinygrad.tensor.Tensor', + 'tinygrad.uop.ops.UOp', 'tinygrad.uop.Ops', 'tinygrad.uop.ops.ParamArg', 'tinygrad.dtype.DType', + 'tinygrad.dtype.AddrSpace', 'tinygrad.device.Buffer', 'tinygrad.device.BufferSpec', 'builtins.bytearray', + 'tinygrad.uop.ops.CallInfo', 'tinygrad.uop.ops.AxisType', 'tinygrad.uop.ops.KernelInfo', + 'tinygrad.renderer.Estimates', 'tinygrad.uop.ops.ProgramInfo', 'tinygrad.helpers.Target', + 'tinygrad.runtime.support.hcq2.HCQInfo', +} + + +class Inert: + def __new__(cls, *args): + value = object.__new__(cls) + value.args = args + return value + + def __init__(self, *args): + pass + + def __setstate__(self, state): + self.state = state + + +class Inspector(pickle.Unpickler): + def find_class(self, module, name): + tag = f'{module}.{name}' + if tag not in GLOBALS: + raise ValueError(f'Unexpected Jetlink pickle global: {tag}') + return type(name, (Inert,), {'tag': tag}) + + def persistent_load(self, pid): + raise ValueError('Persistent pickle references are not Jetlink captures') + + +def tagged(value, tag): + return isinstance(value, Inert) and value.tag == tag + + +def walk(root): + stack, seen = [root], set() + while stack: + value = stack.pop() + if id(value) in seen: + continue + seen.add(id(value)) + if len(seen) > 200_000: + raise ValueError('Jetlink capture exceeds inspection object bound') + yield value + if isinstance(value, Inert): + stack.extend((value.args, getattr(value, 'state', None))) + elif isinstance(value, (tuple, list)): + stack.extend(value) + elif isinstance(value, dict): + stack.extend(value.values()) + + +def op_codes(): + path = Path(__file__).resolve().parents[2] / 'tinygrad_repo/tinygrad/uop/__init__.py' + body = next(node.body for node in ast.parse(path.read_text()).body if isinstance(node, ast.ClassDef) and node.name == 'Ops') + names = [node.targets[0].id for node in body if isinstance(node, ast.Assign)] + return {name: index + 1 for index, name in enumerate(names)} + + +def shape(view, codes): + if not tagged(view, 'tinygrad.uop.ops.UOp') or len(view.args) != 5: + raise ValueError('Missing Jetlink input/output shape UOp') + op, sources = view.args[:2] + if not tagged(op, 'tinygrad.uop.Ops') or op.args != (codes['RESHAPE'],): + raise ValueError('Jetlink input/output is not explicitly shaped') + dimensions = sources[1] + if dimensions.args[0].args != (codes['STACK'],): + raise ValueError('Missing Jetlink shape dimensions') + values = [] + for dimension in dimensions.args[1]: + if dimension.args[0].args != (codes['CONST'],) or type(dimension.args[2]) is not int: + raise ValueError('Jetlink shape is not fixed') + values.append(dimension.args[2]) + return tuple(values) + + +def dtype(value, name): + return tagged(value, 'tinygrad.dtype.DType') and len(value.args) == 4 and value.args[2:] == name + + +def require_jetlink_qcom_pickle(path: Path) -> None: + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + match = re.fullmatch(r'jetlink_warp_(1344x760|1928x1208)_(512x256|1024x512)\.pkl', path.name) + if not match: + raise ValueError('Unsupported Jetlink warp geometry name') + if not 0 < path.stat().st_size <= 100_000_000: + raise ValueError('Jetlink pickle exceeds inspection byte bound') + stream = io.BytesIO(path.read_bytes()) + try: + root = Inspector(stream).load() + if stream.read(1): + raise ValueError('Trailing bytes after Jetlink capture') + if not tagged(root, 'tinygrad.engine.jit._TinyJit') or len(root.args) != 2 or root.args[0] is not None: + raise ValueError('Jetlink warp is not a bare captured TinyJit') + capture = root.args[1] + if not tagged(capture, 'tinygrad.engine.jit.CapturedJit') or len(capture.args) != 4: + raise ValueError('Missing Jetlink CapturedJit') + ret, linear, names, inputs = capture.args + if names != ['big_frame', 'big_tfm', 'frame', 'tfm'] or len(inputs) != 4: + raise ValueError('Jetlink warp input names/count differ') + codes = op_codes() + for index, (view, variables, kind, device) in enumerate(inputs): + if variables or device != ('QCOM' if index in (0, 2) else 'NPY'): + raise ValueError('Jetlink warp captured input backend differs') + if not dtype(kind, ('unsigned char', 'B') if index in (0, 2) else ('float', 'f')): + raise ValueError('Jetlink warp captured input dtype differs') + if index in (1, 3) and shape(view, codes) != (3, 3): + raise ValueError('Jetlink transform input shape differs') + if not tagged(ret, 'tinygrad.tensor.Tensor') or not tagged(linear, 'tinygrad.uop.ops.UOp'): + raise ValueError('Missing Jetlink tensor return or executable graph') + mw, mh = map(int, match[2].split('x')) + if shape(ret.state[1]['uop'], codes) != (2, 6, mh // 2, mw // 2): + raise ValueError('Jetlink output shape differs') + output = ret.state[1]['uop'] + for _ in range(100): + parameter = output.args[2] + if tagged(parameter, 'tinygrad.uop.ops.ParamArg'): + state = parameter.state + if state['device'] != 'QCOM' or state['size'] != 2 * 6 * (mh // 2) * (mw // 2) or not dtype(state['dtype'], ('unsigned char', 'B')): + raise ValueError('Jetlink output buffer signature differs') + break + output = output.args[1][0] + else: + raise ValueError('Missing Jetlink output buffer') + frame_size = get_nv12_info(*map(int, match[1].split('x')))[3] + slots, compiled = set(), set() + for node in walk(root): + if tagged(node, 'tinygrad.uop.ops.UOp'): + operation = node.args[0].args[0] + if operation == codes['PROGRAM']: + compiled.add('program') + if operation == codes['BINARY'] and isinstance(node.args[2], bytes) and node.args[2]: + compiled.add('binary') + if isinstance(node, str) and (node.startswith(('METAL', 'AMD', 'USB+AMD'))): + raise ValueError('Foreign Jetlink captured backend') + if tagged(node, 'tinygrad.uop.ops.ParamArg'): + state = node.state + if state['device'] == 'QCOM' and state['slot'] in (0, 2): + if state['size'] != frame_size or not dtype(state['dtype'], ('unsigned char', 'B')) or state['buffer'] is not None: + raise ValueError('Jetlink frame input signature differs') + slots.add(state['slot']) + if slots != {0, 2}: + raise ValueError('Missing paired QCOM frame inputs') + if compiled != {'program', 'binary'}: + raise ValueError('Missing compiled Jetlink execution graph') + except (AttributeError, IndexError, KeyError, TypeError, pickle.UnpicklingError, EOFError) as error: + raise ValueError('Malformed Jetlink capture') from error diff --git a/tools/laptop_device_build/tests/test_params_registry_validation.py b/tools/laptop_device_build/tests/test_params_registry_validation.py new file mode 100644 index 0000000000..3a0e2e302d --- /dev/null +++ b/tools/laptop_device_build/tests/test_params_registry_validation.py @@ -0,0 +1,45 @@ +import unittest + +from tools.laptop_device_build.validate_params_registry import compare, expected_registry + + +class TestParamsRegistryValidation(unittest.TestCase): + def registry(self): + return expected_registry('''inline static auto keys = { + {"Plain", {PERSISTENT, STRING}}, + {"Empty", {PERSISTENT, STRING, ""}}, + {"Escaped", {PERSISTENT, STRING, "a\\n\\\"b"}}, + {"Raw", {PERSISTENT, JSON, R"tag({"a":[1,2]})tag"}}, + {"Enum", {PERSISTENT, INT, std::to_string(static_cast(cereal::Mode::STANDARD))}}, + };''', 'enum ParamKeyType { STRING = 0, INT = 2, JSON = 5 };', 'enum Mode { standard @1; }') + + def test_cpp_literals_and_absent_default_are_distinct(self): + self.assertEqual(self.registry(), {'Plain': (0, None), 'Empty': (0, b''), + 'Escaped': (0, b'a\n"b'), 'Raw': (5, b'{"a":[1,2]}'), 'Enum': (2, b'1')}) + + def test_stale_missing_extra_type_and_default_rejected(self): + expected = self.registry() + compare(expected, dict(expected)) + for change in ('missing', 'extra', 'type', 'default'): + with self.subTest(change=change): + actual = dict(expected) + if change == 'missing': + del actual['Raw'] + elif change == 'extra': + actual['OldKey'] = (0, None) + elif change == 'type': + actual['Enum'] = (0, b'1') + else: + actual['Empty'] = (0, None) + with self.assertRaises(ValueError): + compare(expected, actual) + + def test_unknown_expression_and_duplicate_fail_closed(self): + for header in ('auto keys={{"A",{PERSISTENT,STRING,make_default()}}};', + 'auto keys={{"A",{PERSISTENT,STRING}},{"A",{PERSISTENT,STRING}}};'): + with self.assertRaises(ValueError): + expected_registry(header, 'enum ParamKeyType { STRING = 0 };', '') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/laptop_device_build/validate_params_registry.py b/tools/laptop_device_build/validate_params_registry.py new file mode 100644 index 0000000000..cb1896f489 --- /dev/null +++ b/tools/laptop_device_build/validate_params_registry.py @@ -0,0 +1,209 @@ +import argparse +import ctypes +import hashlib +import json +from pathlib import Path +import re +import tempfile + + +def tokens(text): + pattern = re.compile(r'\s+|//[^\n]*|/\*.*?\*/|R"([^ ()\\\t\r\n]{0,16})\(.*?\)\1"|"(?:\\.|[^"\\])*"|[A-Za-z_]\w*|0x[0-9a-fA-F]+|\d+|.', re.S) + return [match.group() for match in pattern.finditer(text) if not match.group().isspace() and not match.group().startswith(('//', '/*'))] + + +def string_value(value): + if value.startswith('R"'): + start = value.index('(') + delimiter = value[2:start] + return value[start + 1:-(len(delimiter) + 2)].encode() + if not value.startswith('"'): + raise ValueError(f'Unsupported C++ string: {value}') + text = value[1:-1] + output = bytearray() + escapes = {'n': 10, 'r': 13, 't': 9, 'b': 8, 'f': 12, 'v': 11, 'a': 7, '\\': 92, '"': 34, "'": 39, '?': 63} + index = 0 + while index < len(text): + if text[index] != '\\': + output.extend(text[index].encode()) + index += 1 + continue + index += 1 + char = text[index] + if char in escapes: + output.append(escapes[char]) + index += 1 + elif char in '01234567': + match = re.match(r'[0-7]{1,3}', text[index:]) + output.append(int(match.group(), 8)) + index += len(match.group()) + elif char == 'x': + match = re.match(r'[0-9a-fA-F]+', text[index + 1:]) + if match is None: + raise ValueError('Invalid hex escape') + output.append(int(match.group(), 16)) + index += 1 + len(match.group()) + else: + raise ValueError(f'Unsupported C++ escape: {char}') + return bytes(output) + + +def expected_registry(header, params_header, schema): + types_match = re.search(r'enum ParamKeyType\s*\{(.*?)\};', params_header, re.S) + if types_match is None: + raise ValueError('Missing ParamKeyType') + types = {name: int(value) for name, value in re.findall(r'(\w+)\s*=\s*(\d+)', types_match.group(1))} + body = tokens(header) + start = body.index('keys') + start = body.index('{', start) + entries = [] + depth = 0 + begin = None + for index in range(start + 1, len(body)): + token = body[index] + if token == '{': + if depth == 0: + begin = index + depth += 1 + elif token == '}': + if depth == 0: + break + depth -= 1 + if depth == 0: + entries.append(body[begin:index + 1]) + result = {} + for entry in entries: + if len(entry) < 8 or entry[2:4] != [',', '{'] or entry[-2:] != ['}', '}']: + raise ValueError(f'Unsupported registry entry: {entry}') + key = string_value(entry[1]).decode() + fields = [] + field = [] + nesting = 0 + for token in entry[4:-2]: + if token in ('(', '{'): + nesting += 1 + if token in (')', '}'): + nesting -= 1 + if token == ',' and nesting == 0: + fields.append(field) + field = [] + else: + field.append(token) + fields.append(field) + if len(fields) not in (2, 3) or len(fields[1]) != 1 or fields[1][0] not in types: + raise ValueError(f'Unsupported attributes for {key}') + default = None + if len(fields) == 3: + expression = fields[2] + if all(token.startswith(('"', 'R"')) for token in expression): + default = b''.join(string_value(token) for token in expression) + else: + value = ''.join(expression) + match = re.fullmatch(r'std::to_string\(static_cast\(cereal::(\w+)::(\w+)\)\)', value) + if match is None: + raise ValueError(f'Unsupported default for {key}: {value}') + enum = re.search(r'enum\s+' + re.escape(match[1]) + r'\s*\{(.*?)\}', schema, re.S) + ordinal = re.search(r'\b' + re.escape(match[2].lower()) + r'\s*@\s*(\d+)\s*;', enum.group(1)) if enum else None + if ordinal is None: + raise ValueError(f'Unresolved enum default for {key}') + default = ordinal[1].encode() + if key in result: + raise ValueError(f'Duplicate registry key: {key}') + result[key] = (types[fields[1][0]], default) + if not result: + raise ValueError('Empty registry') + return result + + +class Buffer(ctypes.Structure): + _fields_ = [('data', ctypes.c_void_p), ('size', ctypes.c_size_t)] + + +class NativeRegistry: + def __init__(self, library, directory): + self.library = ctypes.CDLL(str(Path(library).resolve())) + for name, arguments, returns in ( + ('params_create', [ctypes.c_char_p, ctypes.c_size_t], ctypes.c_void_p), + ('params_destroy', [ctypes.c_void_p], None), + ('params_last_error', [], ctypes.c_char_p), + ('params_check_key', [ctypes.c_void_p, ctypes.c_char_p], ctypes.c_bool), + ('params_get_key_type', [ctypes.c_void_p, ctypes.c_char_p], ctypes.c_int), + ('params_get_default', [ctypes.c_void_p, ctypes.c_char_p], Buffer), + ('params_keys_size', [ctypes.c_void_p], ctypes.c_size_t), + ('params_key_at', [ctypes.c_void_p, ctypes.c_size_t], Buffer), + ): + function = getattr(self.library, name) + function.argtypes = arguments + function.restype = returns + path = str(directory).encode() + self.handle = self.library.params_create(path, len(path)) + self.check_error() + if not self.handle: + raise ValueError('Params constructor returned null') + + def check_error(self): + error = self.library.params_last_error() + if error: + raise ValueError(error.decode(errors='replace')) + + def call(self, name, *args): + value = getattr(self.library, name)(self.handle, *args) + self.check_error() + return value + + def read(self): + result = {} + for index in range(self.call('params_keys_size')): + buffer = self.call('params_key_at', index) + if not buffer.data: + raise ValueError('Null key in native registry') + key = ctypes.string_at(buffer.data, buffer.size) + if not self.call('params_check_key', key): + raise ValueError('Enumerated native key rejected') + kind = self.call('params_get_key_type', key) + buffer = self.call('params_get_default', key) + default = ctypes.string_at(buffer.data, buffer.size) if buffer.data else None + decoded = key.decode() + if decoded in result: + raise ValueError('Duplicate native key') + result[decoded] = (kind, default) + return result + + def close(self): + self.library.params_destroy(self.handle) + self.handle = None + self.check_error() + + +def compare(expected, actual): + missing = sorted(expected.keys() - actual.keys()) + extra = sorted(actual.keys() - expected.keys()) + wrong_type = sorted(key for key in expected.keys() & actual.keys() if expected[key][0] != actual[key][0]) + wrong_default = sorted(key for key in expected.keys() & actual.keys() if expected[key][1] != actual[key][1]) + if missing or extra or wrong_type or wrong_default: + raise ValueError(json.dumps({'missing': missing, 'extra': extra, 'wrongType': wrong_type, 'wrongDefault': wrong_default})) + + +def validate(root): + root = Path(root).resolve() + paths = ['openpilot/common/params_keys.h', 'openpilot/common/params.h', 'openpilot/cereal/log.capnp', 'openpilot/common/libparams_c.so'] + expected = expected_registry(*((root / path).read_text() for path in paths[:3])) + with tempfile.TemporaryDirectory(prefix='params-registry-build-') as directory: + native = NativeRegistry(root / paths[3], directory) + try: + compare(expected, native.read()) + finally: + native.close() + return {'status': 'PASS', 'keys': len(expected), 'inputs': {path: hashlib.sha256((root / path).read_bytes()).hexdigest() for path in paths}} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('root', type=Path) + parser.add_argument('--receipt', type=Path) + arguments = parser.parse_args() + receipt = validate(arguments.root) + output = json.dumps(receipt, sort_keys=True) + if arguments.receipt: + arguments.receipt.write_text(output + '\n') + print(output) diff --git a/tools/release/build_manifest.py b/tools/release/build_manifest.py new file mode 100644 index 0000000000..e8d5d0fde4 --- /dev/null +++ b/tools/release/build_manifest.py @@ -0,0 +1,95 @@ +import argparse +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess + + +def git(root, *args): + return subprocess.check_output(['git', '-C', str(root), *args]) + + +def tracked(root): + return [os.fsdecode(path) for path in git(root, 'ls-files', '-z').split(b'\0') if path] + + +def fingerprint(path): + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + return {'kind': 'symlink', 'mode': stat.S_IMODE(mode), 'sha256': hashlib.sha256(os.fsencode(os.readlink(path))).hexdigest()} + if not stat.S_ISREG(mode): + raise ValueError(f'Not a regular file: {path}') + return {'kind': 'file', 'mode': stat.S_IMODE(mode), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} + + +def read_manifest(path): + path = Path(path) + metadata = path.lstat() + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600: + raise ValueError('Unsafe build manifest') + if metadata.st_size > 16 * 1024 * 1024: + raise ValueError('Oversized build manifest') + value = json.loads(path.read_text()) + if value.get('format') != 'starpilot-ci-build' or value.get('version') != 1: + raise ValueError('Unsupported build manifest') + return value + + +def capture(root, destination, outputs, config): + root = Path(root).resolve() + destination = Path(destination).absolute() + if destination.resolve().is_relative_to(root): + raise ValueError('Build manifest must be outside checkout') + revision = git(root, 'rev-parse', 'HEAD').decode().strip() + if destination.exists(): + existing = read_manifest(destination) + if existing['root'] != str(root) or existing['revision'] != revision or existing['config'] != config: + raise ValueError('Stale build manifest') + return + if git(root, 'status', '--porcelain', '--untracked-files=all'): + raise ValueError('Checkout must be clean before build') + files = {name: fingerprint(root / name) for name in tracked(root)} + selected = sorted(set(outputs) & files.keys()) + if any(files[name]['kind'] != 'file' for name in selected): + raise ValueError('Derived output is not a regular file') + value = {'format': 'starpilot-ci-build', 'version': 1, 'root': str(root), 'revision': revision, + 'config': config, 'files': files, 'outputs': selected} + with open(destination, 'x', opener=lambda path, flags: os.open(path, flags, 0o600)) as stream: + json.dump(value, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + + +def verify(root, manifest): + root = Path(root).resolve() + value = read_manifest(manifest) + if value['root'] != str(root) or value['revision'] != git(root, 'rev-parse', 'HEAD').decode().strip(): + raise ValueError('Build manifest belongs to a different checkout or revision') + files = value['files'] + outputs = set(value['outputs']) + if not outputs <= files.keys() or set(tracked(root)) != files.keys(): + raise ValueError('Tracked inventory changed') + if git(root, 'diff', '--cached', '--name-only'): + raise ValueError('Index changed during build') + if git(root, 'ls-files', '--others', '--exclude-standard', '-z'): + raise ValueError('Untracked files created during build') + for name, before in files.items(): + if Path(name).is_absolute() or '..' in Path(name).parts: + raise ValueError('Unsafe manifest path') + after = fingerprint(root / name) + if after['kind'] != before['kind'] or after['mode'] != before['mode']: + raise ValueError(f'File type or mode changed: {name}') + if name in outputs: + if after['kind'] != 'file' or (root / name).stat().st_size == 0: + raise ValueError(f'Missing or empty derived output: {name}') + elif after != before: + raise ValueError(f'Source or undeclared output changed: {name}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--manifest', required=True) + args = parser.parse_args() + verify(git(Path.cwd(), 'rev-parse', '--show-toplevel').decode().strip(), args.manifest) diff --git a/tools/release/check-dirty.sh b/tools/release/check-dirty.sh index 73d393f3c2..aca469f7b9 100755 --- a/tools/release/check-dirty.sh +++ b/tools/release/check-dirty.sh @@ -4,6 +4,11 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd "$DIR" +if [ -n "${STARPILOT_CI_BUILD_MANIFEST:-}" ]; then + python3 "$DIR/build_manifest.py" --manifest "$STARPILOT_CI_BUILD_MANIFEST" + exit 0 +fi + if [ ! -z "$(git status --porcelain)" ]; then echo "Dirty working tree after build:" git status --porcelain diff --git a/tools/release/release_files.py b/tools/release/release_files.py index 55ffcd4b3b..8313932353 100755 --- a/tools/release/release_files.py +++ b/tools/release/release_files.py @@ -10,7 +10,7 @@ ROOT = os.path.abspath(os.path.join(HERE, "../..")) blacklist = [ ".git/", ".venv/", - ".github/workflows/", + r"^\.github/workflows/", "matlab.*.md", @@ -22,9 +22,10 @@ blacklist = [ # Dependency sources are ordinary tracked files. A gitlink cannot be packaged # without a separate checkout, so fail rather than silently omitting its source. -def release_files(root: str, include_big_model: bool = False) -> list[bytes]: +def release_files(root: str, include_big_model: bool = False, *, require_provider: bool = False) -> list[bytes]: entries = subprocess.check_output(["git", "ls-files", "--stage", "-z"], cwd=root).split(b"\0") files = [] + modes = {} for entry in entries: if not entry: continue @@ -44,7 +45,18 @@ def release_files(root: str, include_big_model: bool = False) -> list[bytes]: if blacklisted: continue + modes[tracked_file] = mode files.append(tracked_file) + if b"prebuilt" in files and b"openpilot/starpilot/replay/owner.py" in files: + for name, allowed_modes in ((b"openpilot/tools/replay/replay", (b"100755",)), + (b"openpilot/tools/replay/libmedia_mux.so", (b"100644", b"100755"))): + if modes.get(name) not in allowed_modes: + raise ValueError(f"Prebuilt Replay Git package missing or invalid mode: {os.fsdecode(name)}; track both Replay runtime outputs") + if require_provider or b"openpilot/starpilot/maps/shadow_lifecycle.py" in files: + for name, mode in ((b"openpilot/starpilot/maps/provider/mapd", b"100755"), + (b"openpilot/starpilot/maps/provider/manifest.json", b"100644")): + if modes.get(name) != mode: + raise ValueError(f"Mapd Git package missing or invalid mode: {os.fsdecode(name)}; track the verified provider package") return files diff --git a/tools/release/tests/test_build_manifest.py b/tools/release/tests/test_build_manifest.py new file mode 100644 index 0000000000..caf0a7bf6b --- /dev/null +++ b/tools/release/tests/test_build_manifest.py @@ -0,0 +1,91 @@ +import subprocess + +import pytest + +from tools.release.build_manifest import capture, verify + + +@pytest.fixture +def checkout(tmp_path): + root = tmp_path / 'repo' + root.mkdir() + subprocess.run(['git', 'init', '-q', str(root)], check=True) + for name in ('source.py', 'output.bin', 'undeclared.bin'): + (root / name).write_text('original\n') + subprocess.run(['git', '-C', str(root), 'add', '.'], check=True) + subprocess.run(['git', '-C', str(root), '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', + 'commit', '-qm', 'fixture'], check=True) + return root, tmp_path / 'manifest.json' + + +def test_declared_output_rebuilt(checkout): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {'arch': 'test'}) + (root / 'output.bin').write_bytes(b'host output') + capture(root, manifest, ['output.bin'], {'arch': 'test'}) + verify(root, manifest) + + +@pytest.mark.parametrize('change', ['source', 'undeclared', 'missing', 'empty', 'mode', 'symlink', 'untracked', 'index']) +def test_build_mutations_rejected(checkout, change): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {'arch': 'test'}) + target = root / 'output.bin' + if change in ('source', 'undeclared'): + (root / ('source.py' if change == 'source' else 'undeclared.bin')).write_text('changed') + elif change == 'missing': + target.unlink() + elif change == 'empty': + target.write_bytes(b'') + elif change == 'mode': + target.chmod(0o755) + elif change == 'symlink': + target.unlink() + target.symlink_to('source.py') + elif change == 'untracked': + (root / 'unexpected.py').write_text('created') + else: + target.write_text('changed') + subprocess.run(['git', '-C', str(root), 'add', 'output.bin'], check=True) + with pytest.raises((ValueError, FileNotFoundError)): + verify(root, manifest) + + +def test_dirty_capture_rejected(checkout): + root, manifest = checkout + (root / 'source.py').write_text('changed') + with pytest.raises(ValueError, match='clean'): + capture(root, manifest, ['output.bin'], {}) + + +def test_stale_config_and_root_rejected(checkout, tmp_path): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {'arch': 'one'}) + with pytest.raises(ValueError, match='Stale'): + capture(root, manifest, ['output.bin'], {'arch': 'two'}) + other = tmp_path / 'other' + subprocess.run(['git', 'clone', '-q', str(root), str(other)], check=True) + with pytest.raises(ValueError, match='different checkout'): + verify(other, manifest) + + +def test_unsafe_manifest_rejected(checkout): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {}) + manifest.chmod(0o644) + with pytest.raises(ValueError, match='Unsafe'): + verify(root, manifest) + + +def test_manifest_inside_checkout_rejected(checkout): + root, _ = checkout + with pytest.raises(ValueError, match='outside'): + capture(root, root / 'manifest.json', ['output.bin'], {}) + + +def test_manifest_parent_symlink_inside_checkout_rejected(checkout, tmp_path): + root, _ = checkout + alias = tmp_path / 'alias' + alias.symlink_to(root, target_is_directory=True) + with pytest.raises(ValueError, match='outside'): + capture(root, alias / 'manifest.json', ['output.bin'], {}) diff --git a/tools/release/tests/test_release_files.py b/tools/release/tests/test_release_files.py index dd63e960bb..3eb36d7949 100644 --- a/tools/release/tests/test_release_files.py +++ b/tools/release/tests/test_release_files.py @@ -58,6 +58,29 @@ class TestReleaseFiles(unittest.TestCase): with self.assertRaisesRegex(ValueError, "Cannot package gitlink: panda"): release_files(str(self.root)) + def test_prebuilt_replay_requires_tracked_runtime_outputs(self): + source = Path(__file__).resolve().parents[3] + for name in (".gitignore", "openpilot/tools/replay/.gitignore"): + self.track(name, (source / name).read_text()) + self.track("openpilot/starpilot/replay/owner.py") + self.assertIn(b"openpilot/starpilot/replay/owner.py", release_files(str(self.root))) + self.track("prebuilt", "") + for name in ("openpilot/tools/replay/replay", "openpilot/tools/replay/libmedia_mux.so"): + with self.assertRaisesRegex(ValueError, "Prebuilt Replay Git package missing"): + release_files(str(self.root)) + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("compiled runtime fixture\n") + path.chmod(0o755) + self.git("add", ".") + selected = release_files(str(self.root)) + self.assertIn(b"openpilot/tools/replay/replay", selected) + self.assertIn(b"openpilot/tools/replay/libmedia_mux.so", selected) + (self.root / "openpilot/tools/replay/replay").chmod(0o644) + self.git("add", ".") + with self.assertRaisesRegex(ValueError, "invalid mode"): + release_files(str(self.root)) + def test_unresolved_merge_cannot_be_packaged(self): self.track("main.py") blob = self.git("hash-object", "-w", "--stdin", data=b"conflict\n").strip().decode() diff --git a/tools/scripts/test_ssh.py b/tools/scripts/test_ssh.py new file mode 100644 index 0000000000..bfb5ef3a0b --- /dev/null +++ b/tools/scripts/test_ssh.py @@ -0,0 +1,78 @@ +import importlib.util +from io import BytesIO +from pathlib import Path +from unittest.mock import Mock, patch +import queue + +import pytest + + +@pytest.fixture +def device_ssh(): + spec = importlib.util.spec_from_file_location("device_ssh", Path(__file__).with_name("ssh.py")) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_read_banner_accepts_ssh_and_rejects_other_services(device_ssh): + device_ssh.read_banner(BytesIO(b"SSH-2.0-OpenSSH\r\n")) + with pytest.raises(ConnectionError, match="no ssh banner"): + device_ssh.read_banner(BytesIO(b"HTTP/1.1 200 OK\r\n")) + + +def test_athena_probe_kills_its_proxy_after_reading_banner(device_ssh): + process = Mock(stdout=BytesIO(b"SSH-2.0-OpenSSH\r\n")) + processes = [] + command = ["ssh", "-W", "comma-device:22", "proxy"] + with patch.object(device_ssh.subprocess, "Popen", return_value=process) as spawn: + device_ssh.probe_athena(command, processes) + assert processes == [process] + assert spawn.call_args.args == (command,) + process.kill.assert_called_once_with() + + +@pytest.mark.parametrize("routes,results,expected", [ + ({"local": "local", "athena": "proxy"}, [("local", 1.0, None)], "local"), + ({"local": "local", "athena": "proxy"}, [("athena", 2.0, None), ("local", None, "refused")], "athena"), + ({"local": "local", "athena": "proxy"}, [("athena", 2.0, None), ("local", 1.0, None)], "local"), + ({"local": "local", "athena": "proxy"}, [("local", None, "not found"), ("athena", 2.0, None)], "athena"), + ({"local": "local", "athena": "proxy"}, [("local", None, "refused"), ("athena", None, "refused")], None), + ({"athena": "proxy"}, [("athena", 2.0, None)], "athena"), +]) +def test_probe_routes_selects_only_a_successful_available_route(device_ssh, routes, results, expected): + completed = queue.Queue() + for result in results: + completed.put(result) + with patch.object(device_ssh.queue, "Queue", return_value=completed), \ + patch.object(device_ssh.threading, "Thread") as threads, \ + patch.object(device_ssh.time, "monotonic", return_value=0.0): + choice, status = device_ssh.probe_routes(routes, ["ssh", "proxy"]) + assert choice == expected + assert status == {name: (ms, error) for name, ms, error in results} + assert threads.call_count == len(routes) + assert all(call.kwargs["daemon"] for call in threads.call_args_list) + + +def test_probe_routes_deadline_without_a_banner_returns_no_route(device_ssh): + completed = Mock(get=Mock(side_effect=queue.Empty)) + with patch.object(device_ssh.queue, "Queue", return_value=completed), \ + patch.object(device_ssh.threading, "Thread"), \ + patch.object(device_ssh.time, "monotonic", return_value=0.0): + choice, status = device_ssh.probe_routes({"local": "local", "athena": "proxy"}, ["ssh", "proxy"]) + assert choice is None + assert status == {} + completed.get.assert_called_once_with(timeout=device_ssh.ATHENA_TIMEOUT) + + +def test_probe_routes_deadline_retains_successful_proxy_while_local_is_pending(device_ssh): + completed = Mock(get=Mock(side_effect=[("athena", 2.0, None), queue.Empty])) + with patch.object(device_ssh.queue, "Queue", return_value=completed), \ + patch.object(device_ssh.threading, "Thread"), \ + patch.object(device_ssh.time, "monotonic", return_value=0.0): + choice, status = device_ssh.probe_routes({"local": "local", "athena": "proxy"}, ["ssh", "proxy"]) + assert choice == "athena" + assert status == {"athena": (2.0, None)} + assert completed.get.call_count == 2 + assert all(call.kwargs["timeout"] == device_ssh.ATHENA_TIMEOUT for call in completed.get.call_args_list) diff --git a/tools/setup_dependencies.sh b/tools/setup_dependencies.sh index 6f023f7b53..e6af9cda79 100755 --- a/tools/setup_dependencies.sh +++ b/tools/setup_dependencies.sh @@ -125,6 +125,7 @@ function install_python_deps() { echo "installing python packages..." uv sync --frozen --all-extras source .venv/bin/activate + python3 scripts/lint/check_workflows.py --install } # --- Main --- diff --git a/tools/test_runner.py b/tools/test_runner.py index 2121907e33..0003b1202a 100755 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -3,6 +3,7 @@ import argparse from collections import Counter from concurrent.futures import as_completed, ProcessPoolExecutor from itertools import batched +import json import math import os from pathlib import Path @@ -108,7 +109,7 @@ class Result(unittest.TestResult): self.mark(test, "error", self._exc_info_to_string(err, test)) def addSkip(self, test, reason): - self.mark(test, "skipped") + self.mark(test, "skipped", reason or "No skip reason provided") def addExpectedFailure(self, test, err): self.mark(test, "xfailed") @@ -185,7 +186,8 @@ def make_batches(tests, workers): module = sys.modules[cls.__module__] if hasattr(module, "setUpModule") or hasattr(module, "tearDownModule"): key = cls.__module__ - elif "setUpClass" in cls.__dict__ or "tearDownClass" in cls.__dict__: + elif any(method in base.__dict__ for base in cls.__mro__ if base is not unittest.TestCase + for method in ("setUpClass", "tearDownClass")): key = f"{cls.__module__}.{cls.__qualname__}" else: parallel.append(test.id()) @@ -197,15 +199,71 @@ def make_batches(tests, workers): return sorted(batches, key=len, reverse=True) +def account_for_tests(test_ids, records, tests=()): + """Return one outcome per selected test, expanding unittest fixture events.""" + selected = set(test_ids) + fixtures = {} + for test in tests: + if test.id() not in selected: + continue + cls = type(test) + scopes = (("Class", f"{cls.__module__}.{cls.__qualname__}"), ("Module", cls.__module__)) + for scope, name in scopes: + for phase in ("setUp", "tearDown"): + fixtures.setdefault(f"{phase}{scope} ({name})", []).append(test.id()) + + accounted = {} + events = [] + for record in records: + if record["id"] not in selected: + events.append(record) + elif record["id"] in accounted: + accounted[record["id"]] = make_record(record["id"], "error", "Runner received duplicate outcomes for this selected test") + else: + accounted[record["id"]] = record.copy() + + for event in events: + affected = fixtures.get(event["id"]) + status = event["status"] + detail = f"{event['id']}: {event['detail']}" + if affected is None: + affected = [test_id for test_id in test_ids if test_id not in accounted] or test_ids + status = "error" + detail = f"Unexpected runner event {detail}" + for test_id in affected: + if test_id not in accounted: + accounted[test_id] = make_record(test_id, status, detail) + else: + record = accounted[test_id] + if record["status"] not in FAILURES or status == "error": + record["status"] = status + record["detail"] += ("\n\n" if record["detail"] else "") + detail + + for test_id in test_ids: + if test_id not in accounted: + accounted[test_id] = make_record(test_id, "error", "Runner did not receive an outcome for this selected test") + return [accounted[test_id] for test_id in test_ids] + + def run_batch(test_ids, capture_output): result = Result(capture_output) outside = Capture(capture_output) + tests = [] os.chdir(ROOT) outside.start() try: - unittest.TestLoader().loadTestsFromNames(test_ids).run(result) + suite = unittest.TestLoader().loadTestsFromNames(test_ids) + tests = list(flatten(suite)) + suite.run(result) + except Exception: + detail = traceback.format_exc() + if result.current is not None: + result.current.update(status="error", detail=detail) + result.stopTest(None) + result.records.append(make_record("batch execution", "error", detail)) finally: stdout, stderr = outside.stop() + result.records = account_for_tests(test_ids, result.records, tests) failures = [item for item in result.records if item["status"] in FAILURES] if failures: # attach class/module fixture output to the first related failure failures[0]["stdout"] = stdout + failures[0]["stdout"] @@ -220,7 +278,8 @@ def run_parallel(batches, workers, warning_action, capture_output): try: yield future.result() except Exception: - yield [make_record(futures[future][0], "error", traceback.format_exc())] + detail = traceback.format_exc() + yield [make_record(test_id, "error", detail) for test_id in futures[future]] def report(records, errors, duration_count, elapsed): @@ -237,6 +296,11 @@ def report(records, errors, duration_count, elapsed): if item[stream]: print(paint(f"\n--- captured {stream} ---", 33)) print(item[stream].rstrip()) + skipped = Counter(r["detail"] or "No skip reason provided" for r in records if r["status"] == "skipped") + if skipped: + print(paint("\nskipped tests", 33)) + for reason, count in sorted(skipped.items()): + print(f"{count:8d} {reason}") timed = sorted((r for r in records if r["time"]), key=lambda r: r["time"], reverse=True) if duration_count: timed = timed[:duration_count] @@ -257,6 +321,29 @@ def report(records, errors, duration_count, elapsed): return 5 +def write_json_report(path, test_ids, records, errors, elapsed, exit_code): + payload = { + "schema_version": 1, + "collected": len(test_ids), + "selected_test_ids": test_ids, + "collection_errors": errors, + "results": records, + "duration_seconds": elapsed, + "exit_code": exit_code, + } + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as output: + temporary = Path(output.name) + json.dump(payload, output, indent=2) + output.write("\n") + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("targets", nargs="*", help="files, directories, dotted IDs, or path.py::Class::test") @@ -266,6 +353,7 @@ def main(): parser.add_argument("-v", "--verbose", action="store_true", help="show every test") parser.add_argument("--durations", type=int, default=10, metavar="N", help="show N slowest tests; 0 shows all") parser.add_argument("-W", "--warnings", choices=("error", "default", "always", "ignore"), default="error") + parser.add_argument("--json-output", type=Path, help="write every selected test's outcome and the exit status to JSON") args = parser.parse_args() capture_output = not args.no_capture @@ -280,6 +368,7 @@ def main(): print(summary) records = [] column = 0 + interrupted = False try: if workers < 2: streams = (run_batch(batch, capture_output) for batch in batches) @@ -299,10 +388,21 @@ def main(): column = 0 except KeyboardInterrupt: print(paint("\ninterrupted", 31)) - return 2 + interrupted = True if column: print() - return report(records, errors, args.durations, time.monotonic() - started) + records = account_for_tests([test.id() for test in tests], records) + elapsed = time.monotonic() - started + exit_code = report(records, errors, args.durations, elapsed) + if interrupted: + exit_code = 2 + if args.json_output: + try: + write_json_report(args.json_output, [test.id() for test in tests], records, errors, elapsed, exit_code) + except OSError as e: + print(f"Unable to write JSON report: {e}", file=sys.stderr) + return 1 + return exit_code if __name__ == "__main__": diff --git a/tools/vendor/tests/test_runner_accounting.py b/tools/vendor/tests/test_runner_accounting.py new file mode 100644 index 0000000000..01e2e6be73 --- /dev/null +++ b/tools/vendor/tests/test_runner_accounting.py @@ -0,0 +1,178 @@ +import contextlib +from concurrent.futures import Future +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +from tools import test_runner + + +class TestRunnerAccounting(unittest.TestCase): + def setUp(self): + self.module = types.ModuleType("runner_accounting_fixture") + sys.modules[self.module.__name__] = self.module + self.addCleanup(sys.modules.pop, self.module.__name__) + + def fixture(self, name="Example", base=unittest.TestCase, **methods): + cls = type(name, (base,), { + "__module__": self.module.__name__, + "test_first": lambda self: None, + "test_second": lambda self: None, + **methods, + }) + setattr(self.module, name, cls) + return list(test_runner.flatten(unittest.TestLoader().loadTestsFromTestCase(cls))) + + def run_fixture(self, tests): + ids = [test.id() for test in tests] + records = test_runner.run_batch(ids, capture_output=False) + self.assertEqual([record["id"] for record in records], ids) + return records + + def test_inherited_class_skip_accounts_for_each_selected_test(self): + def skip(cls): + raise unittest.SkipTest("abstract safety fixture") + + base = type("FixtureBase", (unittest.TestCase,), {"setUpClass": classmethod(skip)}) + tests = self.fixture(base=base) + batches = test_runner.make_batches(tests, workers=8) + self.assertEqual(batches, [[test.id() for test in tests]]) + records = self.run_fixture(tests) + self.assertEqual([r["status"] for r in records], ["skipped", "skipped"]) + for record in records: + self.assertIn("setUpClass (runner_accounting_fixture.Example)", record["detail"]) + self.assertIn("abstract safety fixture", record["detail"]) + + def test_module_skip_and_missing_reason_are_explicit(self): + def skip(): + raise unittest.SkipTest + + self.module.setUpModule = skip + tests = self.fixture() + self.fixture("Another") + self.assertEqual(len(test_runner.make_batches(tests, workers=8)), 1) + records = self.run_fixture(tests) + self.assertTrue(all(r["status"] == "skipped" for r in records)) + self.assertTrue(all("setUpModule" in r["detail"] and "No skip reason provided" in r["detail"] for r in records)) + + def test_regular_skip_preserves_reason(self): + tests = self.fixture(test_first=unittest.skip("not applicable")(lambda self: None)) + records = self.run_fixture(tests) + self.assertEqual(records[0]["status"], "skipped") + self.assertEqual(records[0]["detail"], "not applicable") + self.assertEqual(records[1]["status"], "passed") + + def test_class_setup_and_teardown_errors_account_for_every_test(self): + def fail(cls): + raise RuntimeError("fixture unavailable") + + for method in ("setUpClass", "tearDownClass"): + with self.subTest(method=method): + tests = self.fixture(**{method: classmethod(fail)}) + records = self.run_fixture(tests) + self.assertTrue(all(r["status"] == "error" for r in records)) + self.assertTrue(all(method in r["detail"] and "fixture unavailable" in r["detail"] for r in records)) + + def test_module_setup_and_teardown_errors_account_for_every_test(self): + def fail(): + raise RuntimeError("module unavailable") + + for method in ("setUpModule", "tearDownModule"): + with self.subTest(method=method): + setattr(self.module, method, fail) + tests = self.fixture() + self.fixture("Another") + records = self.run_fixture(tests) + self.assertTrue(all(r["status"] == "error" for r in records)) + self.assertTrue(all(method in r["detail"] and "module unavailable" in r["detail"] for r in records)) + delattr(self.module, method) + + def test_missing_and_duplicate_outcomes_fail(self): + records = test_runner.account_for_tests(["first", "missing"], [test_runner.make_record("first")]) + self.assertEqual([r["status"] for r in records], ["passed", "error"]) + self.assertIn("did not receive an outcome", records[1]["detail"]) + duplicate = test_runner.account_for_tests(["first"], [test_runner.make_record("first"), test_runner.make_record("first")]) + self.assertEqual(duplicate[0]["status"], "error") + + def test_loader_failure_is_reported_for_selected_id(self): + ids = ["runner_accounting_fixture.Missing.test_missing"] + records = test_runner.run_batch(ids, capture_output=False) + self.assertEqual([r["id"] for r in records], ids) + self.assertEqual(records[0]["status"], "error") + self.assertIn("Missing", records[0]["detail"]) + + def test_uncaught_batch_failure_accounts_for_all_selected_tests(self): + def fail(self, result=None): + raise RuntimeError("runner failure") + + records = self.run_fixture(self.fixture(run=fail)) + self.assertTrue(all(r["status"] == "error" for r in records)) + self.assertTrue(all("runner failure" in r["detail"] for r in records)) + + def test_worker_failure_accounts_for_whole_batch(self): + future = Future() + future.set_exception(RuntimeError("worker exited")) + with patch.object(test_runner, "ProcessPoolExecutor") as executor: + executor.return_value.__enter__.return_value.submit.return_value = future + records = list(test_runner.run_parallel([["first", "second"]], 2, "error", False))[0] + self.assertEqual([r["id"] for r in records], ["first", "second"]) + self.assertTrue(all(r["status"] == "error" and "worker exited" in r["detail"] for r in records)) + + def test_report_includes_skip_reasons_and_missing_result_failure(self): + records = test_runner.account_for_tests(["skipped", "missing"], [test_runner.make_record("skipped", "skipped", "fixture reason")]) + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = test_runner.report(records, [], 10, 0) + self.assertEqual(status, 1) + self.assertIn("fixture reason", output.getvalue()) + self.assertIn("1 skipped, 1 error", output.getvalue()) + + def test_json_report_preserves_accounting_and_errors(self): + records = test_runner.account_for_tests(["skipped", "missing"], [test_runner.make_record("skipped", "skipped", "fixture reason")]) + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "nested/results.json" + test_runner.write_json_report(report, ["skipped", "missing"], records, ["collection failed"], 1.25, 1) + saved = json.loads(report.read_text()) + self.assertEqual(saved["collected"], 2) + self.assertEqual(saved["selected_test_ids"], ["skipped", "missing"]) + self.assertEqual(saved["results"], records) + self.assertEqual(saved["collection_errors"], ["collection failed"]) + self.assertEqual(saved["duration_seconds"], 1.25) + self.assertEqual(saved["exit_code"], 1) + + def test_cli_json_report_smoke(self): + with tempfile.TemporaryDirectory() as directory: + temporary = Path(directory) + (temporary / "runner_json_fixture.py").write_text("\n".join(( + "import unittest", + "class Example(unittest.TestCase):", + " def test_pass(self): pass", + " @unittest.skip('fixture not applicable')", + " def test_skip(self): pass", + "", + ))) + report = temporary / "results.json" + result = subprocess.run( + [sys.executable, str(test_runner.ROOT / "tools/test_runner.py"), "-j1", "runner_json_fixture", "--json-output", str(report)], + env={**os.environ, "PYTHONPATH": os.pathsep.join((str(temporary), str(test_runner.ROOT)))}, + capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + saved = json.loads(report.read_text()) + self.assertIn("collected 2 tests", result.stdout) + self.assertIn("1 passed, 1 skipped", result.stdout) + self.assertEqual(saved["collected"], len(saved["results"])) + self.assertEqual(saved["exit_code"], result.returncode) + self.assertEqual(saved["collection_errors"], []) + self.assertEqual({r["id"] for r in saved["results"]}, set(saved["selected_test_ids"])) + skipped = next(r for r in saved["results"] if r["status"] == "skipped") + self.assertEqual(skipped["detail"], "fixture not applicable") + + +if __name__ == "__main__": + unittest.main()