From 806655b052372afe830f6b0d2ba6dc73a222fcaf Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 20 Feb 2026 15:48:09 -0800 Subject: [PATCH] CI: replace docker with `op setup` (#37282) --- .github/workflows/auto-cache/action.yaml | 5 +- .github/workflows/badges.yaml | 6 +- .../workflows/compile-openpilot/action.yaml | 10 +-- .github/workflows/repo-maintenance.yaml | 17 +--- .../workflows/setup-with-retry/action.yaml | 4 - .github/workflows/setup/action.yaml | 20 ++--- .github/workflows/tests.yaml | 78 +++++++------------ system/webrtc/tests/test_webrtcd.py | 65 ---------------- 8 files changed, 49 insertions(+), 156 deletions(-) delete mode 100644 system/webrtc/tests/test_webrtcd.py diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml index 377b1eedc..42c8f8fd2 100644 --- a/.github/workflows/auto-cache/action.yaml +++ b/.github/workflows/auto-cache/action.yaml @@ -52,7 +52,4 @@ runs: # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps - id: scons-cache-setup shell: bash - run: | - mkdir -p ${{ inputs.path }} - sudo chmod -R 777 ${{ inputs.path }} - sudo chown -R $USER ${{ inputs.path }} + run: mkdir -p ${{ inputs.path }} diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml index 3f9c9c1c5..23f2c135d 100644 --- a/.github/workflows/badges.yaml +++ b/.github/workflows/badges.yaml @@ -5,9 +5,7 @@ on: workflow_dispatch: env: - BASE_IMAGE: openpilot-base - DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c + PYTHONPATH: ${{ github.workspace }} jobs: badges: @@ -23,7 +21,7 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Push badges run: | - ${{ env.RUN }} "python3 selfdrive/ui/translations/create_badges.py" + python3 selfdrive/ui/translations/create_badges.py rm .gitattributes diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml index 4015746c0..627b4845a 100644 --- a/.github/workflows/compile-openpilot/action.yaml +++ b/.github/workflows/compile-openpilot/action.yaml @@ -6,16 +6,16 @@ runs: - shell: bash name: Build openpilot with all flags run: | - ${{ env.RUN }} "scons -j$(nproc)" - ${{ env.RUN }} "release/check-dirty.sh" + scons -j$(nproc) + release/check-dirty.sh - shell: bash name: Cleanup scons cache and rebuild run: | - ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ - scons -j$(nproc) --cache-populate" + rm -rf /tmp/scons_cache/* + scons -j$(nproc) --cache-populate - name: Save scons cache uses: actions/cache/save@v4 if: github.ref == 'refs/heads/master' with: - path: .ci_cache/scons_cache + path: /tmp/scons_cache key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml index 55d1c2052..10b2e8ab2 100644 --- a/.github/workflows/repo-maintenance.yaml +++ b/.github/workflows/repo-maintenance.yaml @@ -6,9 +6,7 @@ on: workflow_dispatch: env: - BASE_IMAGE: openpilot-base - BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + PYTHONPATH: ${{ github.workspace }} jobs: update_translations: @@ -20,8 +18,7 @@ jobs: submodules: true - uses: ./.github/workflows/setup-with-retry - name: Update translations - run: | - ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish" + run: python3 selfdrive/ui/update_translations.py --vanish - name: Create Pull Request uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 with: @@ -37,18 +34,14 @@ jobs: package_updates: name: package_updates runs-on: ubuntu-latest - container: - image: ghcr.io/commaai/openpilot-base:latest if: github.repository == 'commaai/openpilot' steps: - uses: actions/checkout@v6 with: submodules: true + - uses: ./.github/workflows/setup-with-retry - name: uv lock - run: | - python3 -m ensurepip --upgrade - pip3 install uv - uv lock --upgrade + run: uv lock --upgrade - name: uv pip tree id: pip_tree run: | @@ -57,12 +50,10 @@ jobs: echo 'EOF' >> $GITHUB_OUTPUT - name: bump submodules run: | - git config --global --add safe.directory '*' git submodule update --remote git add . - name: update car docs run: | - export PYTHONPATH="$PWD" scons -j$(nproc) --minimal opendbc_repo python selfdrive/car/docs.py git add docs/CARS.md diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml index 98a391360..923cc3aad 100644 --- a/.github/workflows/setup-with-retry/action.yaml +++ b/.github/workflows/setup-with-retry/action.yaml @@ -1,10 +1,6 @@ name: 'openpilot env setup, with retry on failure' inputs: - docker_hub_pat: - description: 'Auth token for Docker Hub, required for BuildJet jobs' - required: false - default: '' sleep_time: description: 'Time to sleep between retries' required: false diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml index 818060c3b..56f009645 100644 --- a/.github/workflows/setup/action.yaml +++ b/.github/workflows/setup/action.yaml @@ -34,23 +34,19 @@ runs: - id: date shell: bash run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV - - shell: bash - run: echo "$CACHE_COMMIT_DATE" - id: scons-cache uses: ./.github/workflows/auto-cache with: - path: .ci_cache/scons_cache + path: /tmp/scons_cache key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} restore-keys: | scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }} scons-${{ runner.arch }} - # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001 - - id: normalize-file-permissions - shell: bash - name: Normalize file permissions to ensure a consistent docker build cache - run: | - find . -type f -executable -not -perm 755 -exec chmod 755 {} \; - find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \; - # build our docker image - shell: bash - run: eval ${{ env.BUILD }} + name: Run setup + run: ./tools/op.sh setup + - shell: bash + name: Setup cache dirs + run: | + mkdir -p /tmp/comma_download_cache + echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d89250731..e1765eb56 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,13 +18,8 @@ concurrency: cancel-in-progress: true env: - PYTHONWARNINGS: error - BASE_IMAGE: openpilot-base - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: selfdrive/test/docker_build.sh base - - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - + CI: 1 + PYTHONPATH: ${{ github.workspace }} PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical jobs: @@ -38,6 +33,7 @@ jobs: || fromJSON('["ubuntu-24.04"]') }} env: STRIPPED_DIR: /tmp/releasepilot + PYTHONPATH: /tmp/releasepilot steps: - uses: actions/checkout@v6 with: @@ -53,15 +49,13 @@ jobs: run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh - uses: ./.github/workflows/setup-with-retry - name: Build openpilot and run checks - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "python3 system/manager/build.py" + timeout-minutes: 30 + working-directory: ${{ env.STRIPPED_DIR }} + run: python3 system/manager/build.py - name: Run tests timeout-minutes: 1 - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "release/check-dirty.sh" + working-directory: ${{ env.STRIPPED_DIR }} + run: release/check-dirty.sh - name: Check submodules if: github.repository == 'commaai/openpilot' timeout-minutes: 3 @@ -78,11 +72,6 @@ jobs: - uses: actions/checkout@v6 with: submodules: true - - name: Setup docker push - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' - run: | - echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - $DOCKER_LOGIN - uses: ./.github/workflows/setup-with-retry - uses: ./.github/workflows/compile-openpilot timeout-minutes: 30 @@ -106,7 +95,6 @@ jobs: - name: Install dependencies run: ./tools/mac_setup.sh env: - PYTHONWARNINGS: default # package install has DeprecationWarnings HOMEBREW_DISPLAY_INSTALL_TIMES: 1 - run: git lfs pull - name: Getting scons cache @@ -128,8 +116,6 @@ jobs: (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') || fromJSON('["ubuntu-24.04"]') }} - env: - PYTHONWARNINGS: default steps: - uses: actions/checkout@v6 with: @@ -137,7 +123,7 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Static analysis timeout-minutes: 1 - run: ${{ env.RUN }} "scripts/lint/lint.sh" + run: scripts/lint/lint.sh unit_tests: name: unit tests @@ -154,15 +140,14 @@ jobs: - uses: ./.github/workflows/setup-with-retry id: setup-step - name: Build openpilot - run: ${{ env.RUN }} "scons -j$(nproc)" + run: scons -j$(nproc) - name: Run unit tests timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} run: | - ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ - # Pre-compile Python bytecode so each pytest worker doesn't need to - $PYTEST --collect-only -m 'not slow' -qq && \ - MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \ - chmod -R 777 /tmp/comma_download_cache" + source selfdrive/test/setup_xvfb.sh + # Pre-compile Python bytecode so each pytest worker doesn't need to + $PYTEST --collect-only -m 'not slow' -qq + MAX_EXAMPLES=1 $PYTEST -m 'not slow' process_replay: name: process replay @@ -182,17 +167,14 @@ jobs: id: dependency-cache uses: actions/cache@v5 with: - path: .ci_cache/comma_download_cache + path: /tmp/comma_download_cache key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/test_processes.py') }} - name: Build openpilot - run: | - ${{ env.RUN }} "scons -j$(nproc)" + run: scons -j$(nproc) - name: Run replay timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} continue-on-error: ${{ github.ref == 'refs/heads/master' }} - run: | - ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ - chmod -R 777 /tmp/comma_download_cache" + run: selfdrive/test/process_replay/test_processes.py -j$(nproc) - name: Print diff id: print-diff if: always() @@ -226,9 +208,9 @@ jobs: - name: Run regen if: false timeout-minutes: 4 - run: | - ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ - chmod -R 777 /tmp/comma_download_cache" + env: + ONNXCPU: 1 + run: $PYTEST selfdrive/test/process_replay/test_regen.py simulator_driving: name: simulator driving @@ -246,14 +228,13 @@ jobs: - uses: ./.github/workflows/setup-with-retry id: setup-step - name: Build openpilot - run: | - ${{ env.RUN }} "scons -j$(nproc)" + run: scons -j$(nproc) - name: Driving test timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }} run: | - ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ - source selfdrive/test/setup_vsound.sh && \ - CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py" + source selfdrive/test/setup_xvfb.sh + source selfdrive/test/setup_vsound.sh + pytest -s tools/sim/tests/test_metadrive_bridge.py create_ui_report: name: Create UI Report @@ -269,13 +250,12 @@ jobs: submodules: true - uses: ./.github/workflows/setup-with-retry - name: Build openpilot - run: ${{ env.RUN }} "scons -j$(nproc)" + run: scons -j$(nproc) - name: Create UI Report - run: > - ${{ env.RUN }} "PYTHONWARNINGS=ignore && - source selfdrive/test/setup_xvfb.sh && - python3 selfdrive/ui/tests/diff/replay.py && - python3 selfdrive/ui/tests/diff/replay.py --big" + run: | + source selfdrive/test/setup_xvfb.sh + python3 selfdrive/ui/tests/diff/replay.py + python3 selfdrive/ui/tests/diff/replay.py --big - name: Upload UI Report uses: actions/upload-artifact@v6 with: diff --git a/system/webrtc/tests/test_webrtcd.py b/system/webrtc/tests/test_webrtcd.py deleted file mode 100644 index e5f6ceaa3..000000000 --- a/system/webrtc/tests/test_webrtcd.py +++ /dev/null @@ -1,65 +0,0 @@ -import pytest -import asyncio -import json -# for aiortc and its dependencies -import warnings -warnings.filterwarnings("ignore", category=DeprecationWarning) -warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel - -from openpilot.system.webrtc.webrtcd import get_stream - -import aiortc -from teleoprtc import WebRTCOfferBuilder -from openpilot.common.parameterized import parameterized_class - - -@parameterized_class(("in_services", "out_services"), [ - (["testJoystick"], ["carState"]), - ([], ["carState"]), - (["testJoystick"], []), - ([], []), -]) -@pytest.mark.asyncio -class TestWebrtcdProc: - async def assertCompletesWithTimeout(self, awaitable, timeout=1): - try: - async with asyncio.timeout(timeout): - await awaitable - except TimeoutError: - pytest.fail("Timeout while waiting for awaitable to complete") - - async def test_webrtcd(self, mocker): - mock_request = mocker.MagicMock() - async def connect(offer): - body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services} - mock_request.json.side_effect = mocker.AsyncMock(return_value=body) - response = await get_stream(mock_request) - response_json = json.loads(response.text) - return aiortc.RTCSessionDescription(**response_json) - - builder = WebRTCOfferBuilder(connect) - builder.offer_to_receive_video_stream("road") - builder.offer_to_receive_audio_stream() - if len(self.in_services) > 0 or len(self.out_services) > 0: - builder.add_messaging() - - stream = builder.stream() - - await self.assertCompletesWithTimeout(stream.start()) - await self.assertCompletesWithTimeout(stream.wait_for_connection()) - - assert stream.has_incoming_video_track("road") - assert stream.has_incoming_audio_track() - assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0) - - video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track() - await self.assertCompletesWithTimeout(video_track.recv()) - await self.assertCompletesWithTimeout(audio_track.recv()) - - await self.assertCompletesWithTimeout(stream.stop()) - - # cleanup, very implementation specific, test may break if it changes - assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test" - _, session = mock_request.app["streams"].__setitem__.call_args.args - await self.assertCompletesWithTimeout(session.post_run_cleanup()) -