Merge branch 'upstream/openpilot/master'

# Conflicts:
#	.pre-commit-config.yaml
#	README.md
#	cereal
#	opendbc
#	panda
#	selfdrive/car/torque_data/override.yaml

.github/labeler.yaml

@@ -0,0 +1,61 @@
+CI / testing:
+  - all:
+    - changed-files: ['.github/**']
+  - all:
+    - changed-files: ['**/test_*']
+
+car: 
+  - all:
+    - changed-files: ['selfdrive/car/**']
+
+body:
+  - all:
+    - changed-files: ['selfdrive/car/body/*']
+chrysler:
+  - all:
+    - changed-files: ['selfdrive/car/chrysler/*']
+ford: 
+  - all:
+    - changed-files: ['selfdrive/car/ford/*']
+gm: 
+  - all:
+    - changed-files: ['selfdrive/car/gm/*']
+honda: 
+  - all:
+    - changed-files: ['selfdrive/car/honda/*']
+hyundai: 
+  - all:
+    - changed-files: ['selfdrive/car/hyundai/*']
+mazda: 
+  - all:
+    - changed-files: ['selfdrive/car/mazda/*']
+nissan: 
+  - all:
+    - changed-files: ['selfdrive/car/nissan/*']
+subaru: 
+  - all:
+    - changed-files: ['selfdrive/car/subaru/*']
+tesla: 
+  - all:
+    - changed-files: ['selfdrive/car/tesla/*']
+toyota: 
+  - all:
+    - changed-files: ['selfdrive/car/toyota/*']
+volkswagen: 
+  - all:
+    - changed-files: ['selfdrive/car/volkswagen/*']
+
+simulation:
+  - all:
+    - changed-files: ['tools/sim/**']
+ui:
+  - all:
+    - changed-files: ['selfdrive/ui/**']
+tools: 
+  - all:
+    - changed-files: ['tools/**']
+
+multilanguage:
+  - all:
+    - changed-files: ['selfdrive/ui/translations/**']
+

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,27 @@
+name: 'compile openpilot'
+
+inputs:
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/labeler.yaml

@@ -0,0 +1,18 @@
+name: "Pull Request Labeler"
+on:
+  pull_request_target:
+
+jobs:
+  labeler:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - uses: actions/labeler@v5.0.0-alpha.1
+      with:
+        dot: true
+        configuration-path: .github/labeler.yaml

.github/workflows/prebuilt.yaml

@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
 
 env:
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_GHCR_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
@@ -29,5 +29,5 @@ jobs:
         submodules: true
     - name: Build and Push docker image
       run: |
-        $DOCKER_LOGIN
+        $DOCKER_GHCR_LOGIN
         eval "$BUILD"

.github/workflows/selfdrive_tests.yaml

@@ -16,7 +16,8 @@ env:
   CL_BASE_IMAGE: openpilot-base-cl
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
 
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_GHCR_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_HUB_LOGIN: docker login -u adeebshihadeh -p ${{ secrets.DOCKER_HUB_PAT }}
   BUILD: selfdrive/test/docker_build.sh base
 
   RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
@@ -67,30 +68,27 @@ jobs:
         cd $STRIPPED_DIR
         ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
 
-  build_all:
-    name: build all
-    runs-on: ubuntu-20.04
+  build:
+    strategy:
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
+    # login only on arm machines, due to buildjet rate limits
+    - name: Setup docker
+      if: contains(runner.name, 'buildjet')
+      run: |
+        $DOCKER_HUB_LOGIN
     - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot with all flags
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - name: Cleanup scons cache and rebuild
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-    - name: Save scons cache
-      uses: actions/cache/save@v3
-      if: github.ref == 'refs/heads/master'
       with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        cache_key_prefix: scons_${{ matrix.arch }}
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
+      with:
+        cache_key_prefix: scons_${{ matrix.arch }}
 
   build_mac:
     name: build macos
@@ -185,8 +183,10 @@ jobs:
         sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd
 
   docker_push:
-    name: docker push
-    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
     steps:
     - uses: actions/checkout@v3
@@ -195,14 +195,39 @@ jobs:
     - name: Setup to push to repo
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
+        $DOCKER_GHCR_LOGIN
+    # login only on arm machines, due to buildjet rate limits
+    - name: Additional setup for buildjet
+      if: contains(runner.name, 'buildjet')
+      run: |
+        $DOCKER_HUB_LOGIN
     - uses: ./.github/workflows/setup-with-retry
       with:
         git-lfs: false
     - name: Build and push CL Docker image
+      if: matrix.arch == 'x86_64'
       run: |
+        unset TARGET_ARCHITECTURE
         eval "$BUILD_CL"
 
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-20.04
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [docker_push]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - name: Setup docker
+      run: |
+        $DOCKER_GHCR_LOGIN
+    - name: Merge x64 and arm64 tags
+      run: |
+        export PUSH_IMAGE=true
+        selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-20.04
@@ -249,18 +274,10 @@ jobs:
       timeout-minutes: 40
       run: |
         ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
-                        $PYTEST && \
-                        selfdrive/locationd/test/_test_locationd_lib.py && \
-                        ./system/ubloxd/tests/test_glonass_runner && \
+                        $PYTEST -n auto --dist=loadscope --timeout 30 -o cpp_files=test_* && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \
-                        ./common/tests/test_common && \
-                        ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
-                        ./system/loggerd/tests/test_logger &&\
-                        ./system/proclogd/tests/test_proclog && \
-                        ./tools/replay/tests/test_replay && \
-                        ./tools/cabana/tests/test_cabana && \
                         ./system/camerad/test/ae_gray_test && \
                         ./selfdrive/test/process_replay/test_fuzzy.py"
     - name: "Upload coverage to Codecov"
@@ -324,7 +341,7 @@ jobs:
         ${{ env.RUN }} "scons -j$(nproc)"
     # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
     - name: Run model replay with ONNX
-      timeout-minutes: 2
+      timeout-minutes: 3
       run: |
         ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \
                            ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \

.github/workflows/setup-with-retry/action.yaml

@@ -5,9 +5,14 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
-
-env:
-  SLEEP_TIME: 30 # Time to sleep between retries
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons_x86_64'
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
 
 runs:
   using: "composite"
@@ -17,23 +22,26 @@ runs:
       continue-on-error: true
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup1.outcome == 'failure'
       shell: bash
-      run: sleep ${{ env.SLEEP_TIME }}
+      run: sleep ${{ inputs.sleep_time }}
     - id: setup2
       if: steps.setup1.outcome == 'failure'
       uses: ./.github/workflows/setup
       continue-on-error: true
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup2.outcome == 'failure'
       shell: bash
-      run: sleep ${{ env.SLEEP_TIME }}
+      run: sleep ${{ inputs.sleep_time }}
     - id: setup3
       if: steps.setup2.outcome == 'failure'
       uses: ./.github/workflows/setup
       with:
         git_lfs: ${{ inputs.git_lfs }}
-        is_retried: true
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
+        is_retried: true

.github/workflows/setup/action.yaml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons_x86_64'
   is_retried:
     description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
     required: false
@@ -35,10 +39,10 @@ runs:
       uses: actions/cache/restore@v3
       with:
         path: .ci_cache/scons_cache
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
-          scons-${{ env.CACHE_COMMIT_DATE }}-
-          scons-
+          ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-
+          ${{ inputs.cache_key_prefix }}-
     # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps
     - id: scons-cache-setup
       shell: bash

.github/workflows/tools_tests.yaml

@@ -13,7 +13,7 @@ concurrency:
 env:
   BASE_IMAGE: openpilot-base
   CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_GHCR_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
 
   BUILD: selfdrive/test/docker_build.sh base
 
@@ -58,7 +58,7 @@ jobs:
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
+        $DOCKER_GHCR_LOGIN
     - name: Build and push sim image
       run: |
         selfdrive/test/docker_build.sh sim
@@ -78,7 +78,7 @@ jobs:
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
+        $DOCKER_GHCR_LOGIN
     - name: Build and push docs image
       run: |
         selfdrive/test/docker_build.sh docs