diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a24c524047..2bd1ccfd62 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,6 @@ FROM ghcr.io/commaai/openpilot-base:latest -RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa libegl1-mesa +RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa libegl1-mesa gdb RUN pip install ipython jupyter jupyterlab RUN cd /tmp && \ @@ -8,6 +8,8 @@ RUN cd /tmp && \ curl -L -o virtualgl.deb "https://downloads.sourceforge.net/project/virtualgl/3.1/virtualgl_3.1_$ARCH.deb" && \ dpkg -i virtualgl.deb +RUN usermod -aG video batman + USER batman RUN cd $HOME && \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a7a63658ed..7174ee2800 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,8 +17,8 @@ "--volume=/tmp/.X11-unix:/tmp/.X11-unix", "--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority", "--volume=${localEnv:HOME}/.comma:/home/batman/.comma", + "--volume=${localEnv:HOME}/.azure:/home/batman/.azure", "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache", - "--volume=/tmp/devcontainer_scons_cache:/tmp/scons_cache", "--shm-size=1G", "--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux "--publish=0.0.0.0:8070-8079:8070-8079" // body ZMQ services @@ -29,7 +29,9 @@ "installOhMyZsh": false, "upgradePackages": false, "username": "batman" - } + }, + "ghcr.io/devcontainers-contrib/features/gh-cli:1": {}, + "ghcr.io/devcontainers/features/azure-cli:1": {} }, "containerUser": "batman", "remoteUser": "batman", @@ -43,5 +45,8 @@ "lharri73.dbc" ] } - } + }, + "mounts": [ + "type=volume,source=scons_cache,target=/tmp/scons_cache" + ] } \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/pc_bug_report.yml b/.github/ISSUE_TEMPLATE/pc_bug_report.yml index db3eb22e5c..761c8b1a0a 100644 --- a/.github/ISSUE_TEMPLATE/pc_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/pc_bug_report.yml @@ -24,7 +24,7 @@ body: id: os-version attributes: label: OS Version - placeholder: Ubuntu 20.04 + placeholder: Ubuntu 24.04 validations: required: true diff --git a/.github/PULL_REQUEST_TEMPLATE/car_port.md b/.github/PULL_REQUEST_TEMPLATE/car_port.md index 690c24c9b0..c7aa2b96c2 100644 --- a/.github/PULL_REQUEST_TEMPLATE/car_port.md +++ b/.github/PULL_REQUEST_TEMPLATE/car_port.md @@ -8,7 +8,7 @@ assignees: '' **Checklist** -- [ ] added entry to CarInfo in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs +- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs - [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py) - [ ] route with openpilot: - [ ] route with stock system: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3e3f42dcbc..2b4a5ed48f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -44,7 +44,7 @@ Explain how you tested this bug fix. **Checklist** -- [ ] added entry to CarInfo in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs +- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs - [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py) - [ ] route with openpilot: - [ ] route with stock system: diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml index 173803f7f0..e99c46e3c3 100644 --- a/.github/workflows/auto-cache/action.yaml +++ b/.github/workflows/auto-cache/action.yaml @@ -19,14 +19,14 @@ runs: using: "composite" steps: - name: setup namespace cache - if: ${{ contains(runner.name, 'nsc') }} + if: ${{ contains(runner.name, 'nsc') }} uses: namespacelabs/nscloud-cache-action@v1 with: path: ${{ inputs.path }} - name: setup github cache if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }} - uses: 'actions/cache@v3' + uses: 'actions/cache@v4' with: path: ${{ inputs.path }} key: ${{ inputs.key }} @@ -34,7 +34,7 @@ runs: - name: setup github cache if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }} - uses: 'actions/cache/restore@v3' + uses: 'actions/cache/restore@v4' with: path: ${{ inputs.path }} key: ${{ inputs.key }} @@ -46,4 +46,4 @@ runs: run: | mkdir -p ${{ inputs.path }} sudo chmod -R 777 ${{ inputs.path }} - sudo chown -R $USER ${{ inputs.path }} \ No newline at end of file + sudo chown -R $USER ${{ inputs.path }} diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml index abb6c38d6b..8316fedda0 100644 --- a/.github/workflows/auto_pr_review.yaml +++ b/.github/workflows/auto_pr_review.yaml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'commaai/openpilot' steps: - - uses: Vankka/pr-target-branch-action@69ab6dd5c221de3548b3b6c4d102c1f4913d3baa + - uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -34,6 +34,26 @@ jobs: already-exists-action: close_this already-exists-comment: "Your PR should be made against the `master` branch" + comment: + runs-on: ubuntu-latest + steps: + - name: comment + uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 + if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' + with: + message: | + + Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following: + * Convert your PR to a draft unless it's ready to review + * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md) + * Before marking as "ready for review", ensure: + * the goal is clearly stated in the description + * all the tests are passing + * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged) + * include a route or your device' dongle ID if relevant + comment_tag: run_id + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + check-pr-template: runs-on: ubuntu-latest permissions: @@ -41,7 +61,7 @@ jobs: issues: write pull-requests: write actions: read - if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' + if: false && github.event.pull_request.head.repo.full_name != 'commaai/openpilot' steps: - uses: actions/github-script@v7 with: @@ -91,7 +111,7 @@ jobs: // Utility function to check if a list of checkboxes is a subset of another list of checkboxes isCheckboxSubset = (templateCheckBoxTexts, prTextCheckBoxTexts) => { - // Check if each template checkbox text is a substring of at least one PR checkbox text + // Check if each template checkbox text is a substring of at least one PR checkbox text // (user should be allowed to add additional text) return templateCheckBoxTexts.every((item) => prTextCheckBoxTexts.some((element) => element.includes(item))) } diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml index 2f1a7d67c4..cf8bdc7109 100644 --- a/.github/workflows/badges.yaml +++ b/.github/workflows/badges.yaml @@ -12,7 +12,7 @@ env: jobs: badges: name: create badges - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'commaai/openpilot' permissions: contents: write diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml index 2945b67d2e..4015746c0e 100644 --- a/.github/workflows/compile-openpilot/action.yaml +++ b/.github/workflows/compile-openpilot/action.yaml @@ -14,7 +14,7 @@ runs: ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ scons -j$(nproc) --cache-populate" - name: Save scons cache - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 if: github.ref == 'refs/heads/master' with: path: .ci_cache/scons_cache diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c555eafc73..a32989f7d0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,7 +20,7 @@ env: jobs: docs: name: build docs - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 45 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml index 990be73f72..d8963ec89f 100644 --- a/.github/workflows/prebuilt.yaml +++ b/.github/workflows/prebuilt.yaml @@ -11,7 +11,7 @@ env: jobs: build_prebuilt: name: build prebuilt - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'commaai/openpilot' env: PUSH_IMAGE: true @@ -22,7 +22,7 @@ jobs: steps: - name: Wait for green check mark if: ${{ github.event_name != 'workflow_dispatch' }} - uses: lewagon/wait-on-check-action@595dabb3acf442d47e29c9ec9ba44db0c6bdd18f + uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc with: ref: master wait-interval: 30 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 38dbf07a47..34e333bb59 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,7 +12,7 @@ jobs: ImageOS: ubuntu20 container: image: ghcr.io/commaai/openpilot-base:latest - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.repository == 'commaai/openpilot' permissions: checks: read @@ -24,7 +24,7 @@ jobs: sudo apt-get install -y libyaml-dev - name: Wait for green check mark if: ${{ github.event_name != 'workflow_dispatch' }} - uses: lewagon/wait-on-check-action@595dabb3acf442d47e29c9ec9ba44db0c6bdd18f + uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc with: ref: master wait-interval: 30 @@ -47,7 +47,7 @@ jobs: export PYTHONPATH=$TARGET_DIR cd $TARGET_DIR scons -j$(nproc) - selfdrive/car/tests/test_car_interfaces.py + pytest -n logical selfdrive/car/tests/test_car_interfaces.py - name: Push master-ci run: | unset TARGET_DIR diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml index bd882210fa..0e6606112b 100644 --- a/.github/workflows/repo-maintenance.yaml +++ b/.github/workflows/repo-maintenance.yaml @@ -2,15 +2,16 @@ name: repo maintenance on: schedule: - - cron: "0 12 * * 1" # every Monday at 12am UTC (4am PST) + - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST) workflow_dispatch: jobs: bump_submodules: name: bump_submodules - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest container: image: ghcr.io/commaai/openpilot-base:latest + if: github.repository == 'commaai/openpilot' steps: - uses: actions/checkout@v4 with: @@ -21,8 +22,9 @@ jobs: git -c submodule."tinygrad".update=none submodule update --remote git add . - name: Create Pull Request - uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5 + uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 with: + author: Vehicle Researcher token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} commit-message: bump submodules title: '[bot] Bump submodules' @@ -33,9 +35,10 @@ jobs: labels: bot package_updates: name: package_updates - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest container: image: ghcr.io/commaai/openpilot-base:latest + if: github.repository == 'commaai/openpilot' steps: - uses: actions/checkout@v4 - name: poetry lock @@ -47,8 +50,9 @@ jobs: git config --global --add safe.directory '*' pre-commit autoupdate - name: Create Pull Request - uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5 + uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 with: + author: Vehicle Researcher token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} commit-message: Update Python packages and pre-commit hooks title: '[bot] Update Python packages and pre-commit hooks' diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 964c36776e..2be83b1654 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -14,7 +14,6 @@ concurrency: env: PYTHONWARNINGS: error BASE_IMAGE: openpilot-base - CL_BASE_IMAGE: openpilot-base-cl AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} @@ -22,16 +21,12 @@ env: RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - BUILD_CL: selfdrive/test/docker_build.sh cl - - RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c - - PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 + PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical jobs: build_release: name: build release - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest env: STRIPPED_DIR: /tmp/releasepilot steps: @@ -74,9 +69,9 @@ jobs: matrix: arch: ${{ fromJson( ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || + ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} - runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }} + runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 with: @@ -92,7 +87,7 @@ jobs: strategy: matrix: arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} - runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }} + runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }} if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' steps: - uses: actions/checkout@v4 @@ -106,15 +101,10 @@ jobs: - uses: ./.github/workflows/setup-with-retry with: docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - - name: Build and push CL Docker image - if: matrix.arch == 'x86_64' - run: | - unset TARGET_ARCHITECTURE - eval "$BUILD_CL" docker_push_multiarch: name: docker push multiarch tag - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' needs: [docker_push] steps: @@ -127,13 +117,13 @@ jobs: - name: Merge x64 and arm64 tags run: | export PUSH_IMAGE=true - selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64 + scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64 static_analysis: name: static analysis runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 with: @@ -146,7 +136,7 @@ jobs: valgrind: name: valgrind - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -165,8 +155,8 @@ jobs: unit_tests: name: unit tests runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 with: @@ -182,12 +172,12 @@ jobs: run: | ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ - $PYTEST --timeout 60 -m 'not slow' -n $(nproc) && \ + $PYTEST --timeout 60 -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ ./selfdrive/ui/tests/test_translations.py" - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.job }} env: @@ -196,8 +186,8 @@ jobs: process_replay: name: process replay runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 with: @@ -207,7 +197,7 @@ jobs: docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - name: Cache test routes id: dependency-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: .ci_cache/comma_download_cache key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }} @@ -225,7 +215,7 @@ jobs: id: print-diff if: always() run: cat selfdrive/test/process_replay/diff.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() continue-on-error: true with: @@ -236,7 +226,7 @@ jobs: run: | ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only" - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.job }} env: @@ -244,34 +234,32 @@ jobs: regen: name: regen - runs-on: 'ubuntu-20.04' - steps: + runs-on: 'ubuntu-latest' + steps: - uses: actions/checkout@v4 with: submodules: true - uses: ./.github/workflows/setup-with-retry - name: Cache test routes id: dependency-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: .ci_cache/comma_download_cache key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }} - name: Build base Docker image run: eval "$BUILD" - - name: Build Docker image - run: eval "$BUILD_CL" - name: Build openpilot run: | ${{ env.RUN }} "scons -j$(nproc)" - name: Run regen timeout-minutes: 30 run: | - ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ + ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ chmod -R 777 /tmp/comma_download_cache" test_modeld: name: model tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -279,9 +267,6 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Build base Docker image run: eval "$BUILD" - - name: Build Docker image - # Sim docker is needed to get the OpenCL drivers - run: eval "$BUILD_CL" - name: Build openpilot run: | ${{ env.RUN }} "scons -j$(nproc)" @@ -289,17 +274,17 @@ jobs: - name: Run model replay with ONNX timeout-minutes: 4 run: | - ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \ + ${{ env.RUN }} "unset PYTHONWARNINGS && \ ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \ coverage combine && \ coverage xml" - name: Run unit tests timeout-minutes: 4 run: | - ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \ + ${{ env.RUN }} "unset PYTHONWARNINGS && \ $PYTEST selfdrive/modeld" - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.job }} env: @@ -308,8 +293,8 @@ jobs: test_cars: name: cars runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} strategy: fail-fast: false matrix: @@ -328,7 +313,7 @@ jobs: - name: Build openpilot run: ${{ env.RUN }} "scons -j$(nproc)" - name: Test car models - timeout-minutes: 25 + timeout-minutes: 10 run: | ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \ chmod -R 777 /tmp/comma_download_cache" @@ -336,7 +321,7 @@ jobs: NUM_JOBS: 5 JOB_ID: ${{ matrix.job }} - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: name: ${{ github.job }}-${{ matrix.job }} env: @@ -344,7 +329,7 @@ jobs: car_docs_diff: name: PR comments - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 @@ -355,7 +340,7 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Get base car info run: | - ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info" + ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" sudo chown -R $USER:$USER ${{ github.workspace }} - uses: actions/checkout@v4 with: @@ -367,19 +352,19 @@ jobs: run: | cd current ${{ env.RUN }} "scons -j$(nproc)" - output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info") + output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") output="${output//$'\n'/'%0A'}" echo "::set-output name=diff::$output" - name: Find comment if: ${{ env.AZURE_TOKEN != '' }} - uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d + uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e id: fc with: issue-number: ${{ github.event.pull_request.number }} body-includes: This PR makes changes to - name: Update comment if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }} - uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4 + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} @@ -387,7 +372,7 @@ jobs: edit-mode: replace - name: Delete comment if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }} - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | github.rest.issues.deleteComment({ @@ -398,7 +383,7 @@ jobs: create_ui_report: name: Create UI Report - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -410,10 +395,10 @@ jobs: run: > ${{ env.RUN }} "PYTHONWARNINGS=ignore && source selfdrive/test/setup_xvfb.sh && - export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && + export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && python selfdrive/ui/tests/test_ui/run.py" - name: Upload Test Report - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: report - path: selfdrive/ui/tests/test_ui/report \ No newline at end of file + path: selfdrive/ui/tests/test_ui/report diff --git a/.github/workflows/setup-pre-commit/action.yaml b/.github/workflows/setup-pre-commit/action.yaml index 1b3e16e73f..f07a106861 100644 --- a/.github/workflows/setup-pre-commit/action.yaml +++ b/.github/workflows/setup-pre-commit/action.yaml @@ -4,7 +4,7 @@ runs: using: "composite" steps: - uses: ./.github/workflows/auto-cache - with: + with: path: .ci_cache/pre-commit key: pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }} restore-keys: | diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 2d60d2f0e9..b33945cae2 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -12,15 +12,16 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: exempt-milestones: true # pull request config stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.' close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.' + stale-pr-label: stale delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo - exempt-pr-labels: "ignore stale,needs testing,car port" # if wip or it needs testing from the community, don't mark as stale + exempt-pr-labels: "ignore stale,needs testing,car port,car" # if wip or it needs testing from the community, don't mark as stale days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }} days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index c185fd1d3c..ccd3368cb0 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -12,22 +12,17 @@ concurrency: env: BASE_IMAGE: openpilot-base - CL_BASE_IMAGE: openpilot-base-cl DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} BUILD: selfdrive/test/docker_build.sh base RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - BUILD_CL: selfdrive/test/docker_build.sh cl - - RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c - jobs: plotjuggler: name: plotjuggler - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -36,15 +31,15 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Build openpilot timeout-minutes: 5 - run: ${{ env.RUN }} "scons -j$(nproc) cereal/ common/ --minimal" + run: ${{ env.RUN }} "scons -j$(nproc) cereal/ common/ opendbc/ --minimal" - name: Test PlotJuggler timeout-minutes: 2 run: | ${{ env.RUN }} "pytest tools/plotjuggler/" - simulator: - name: simulator - runs-on: ubuntu-20.04 + simulator_build: + name: simulator docker build + runs-on: ubuntu-latest if: github.repository == 'commaai/openpilot' timeout-minutes: 45 steps: @@ -52,8 +47,6 @@ jobs: with: submodules: true - uses: ./.github/workflows/setup-with-retry - - name: Build base cl image - run: eval "$BUILD_CL" - name: Setup to push to repo if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' run: | @@ -63,6 +56,28 @@ jobs: run: | selfdrive/test/docker_build.sh sim + simulator_driving: + name: simulator driving + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: git lfs pull + - uses: ./.github/workflows/setup-with-retry + - name: Build base docker image + run: eval "$BUILD" + - name: Build openpilot + run: | + ${{ env.RUN }} "scons -j$(nproc)" + - name: Run bridge test + run: | + ${{ env.RUN }} "export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ + source selfdrive/test/setup_xvfb.sh && \ + source selfdrive/test/setup_vsound.sh && \ + CI=1 tools/sim/tests/test_metadrive_bridge.py" + devcontainer: name: devcontainer runs-on: ubuntu-latest @@ -92,8 +107,8 @@ jobs: notebooks: name: notebooks - runs-on: ubuntu-20.04 - if: github.repository == 'commaai/openpilot' + runs-on: ubuntu-latest + if: false && github.repository == 'commaai/openpilot' timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -104,6 +119,6 @@ jobs: timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache run: ${{ env.RUN }} "scons -j$(nproc)" - name: Test notebooks - timeout-minutes: 2 + timeout-minutes: 3 run: | - ${{ env.RUN }} "pip install nbmake && pytest --nbmake tools/car_porting/examples/" \ No newline at end of file + ${{ env.RUN }} "pip install nbmake && pytest --nbmake tools/car_porting/examples/" diff --git a/.gitignore b/.gitignore index 3e91531d08..3da75aaea4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ venv/ .overlay_init .overlay_consistent .sconsign.dblite -.vscode* model2.png a.out .hypothesis diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 02fbd9c892..77e0f81a75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,6 +145,7 @@ build: when: always check no source code sent: + image: alpine stage: sanity variables: FORBIDDEN_FILE_EXTENSIONS: "*.a,*.o,*.os,*.pyc,moc_*,*.cc,Jenkinsfile,supercombo.onnx,.sconsign.dblite" @@ -166,6 +167,7 @@ check no source code sent: - when: never .publish_base: &publish_base + image: alpine variables: GIT_SUBMODULE_STRATEGY: normal stage: publish @@ -214,6 +216,7 @@ publish to public github prebuilt: - when: never .notify_discord: ¬ify_discord + image: alpine stage: notify needs: ["build"] variables: diff --git a/.idea/tools/External Tools.xml b/.idea/tools/External Tools.xml index 75b33a6fd7..92f206447d 100644 --- a/.idea/tools/External Tools.xml +++ b/.idea/tools/External Tools.xml @@ -20,4 +20,18 @@