Merge remote-tracking branch 'commaai/openpilot/master'

# Conflicts:
#	.github/workflows/selfdrive_tests.yaml
#	.pre-commit-config.yaml
#	README.md
#	cereal
#	common/version.h
#	opendbc
#	panda
#	selfdrive/car/chrysler/carcontroller.py
#	selfdrive/car/chrysler/carstate.py
#	selfdrive/car/chrysler/interface.py
#	selfdrive/car/gm/carstate.py
#	selfdrive/car/honda/carstate.py
#	selfdrive/car/hyundai/carcontroller.py
#	selfdrive/car/hyundai/carstate.py
#	selfdrive/car/hyundai/hyundaican.py
#	selfdrive/car/hyundai/interface.py
#	selfdrive/car/hyundai/radar_interface.py
#	selfdrive/car/hyundai/values.py
#	selfdrive/car/interfaces.py
#	selfdrive/car/subaru/carcontroller.py
#	selfdrive/car/subaru/carstate.py
#	selfdrive/car/subaru/interface.py
#	selfdrive/car/subaru/subarucan.py
#	selfdrive/car/torque_data/override.yaml
#	selfdrive/car/toyota/carcontroller.py
#	selfdrive/car/toyota/carstate.py
#	selfdrive/car/toyota/interface.py
#	selfdrive/car/toyota/toyotacan.py
#	selfdrive/car/volkswagen/carstate.py
#	selfdrive/controls/lib/desire_helper.py
#	selfdrive/controls/lib/longitudinal_planner.py
#	selfdrive/monitoring/dmonitoringd.py
#	selfdrive/thermald/power_monitoring.py
#	system/loggerd/uploader.py

.github/workflows/badges.yaml

@@ -7,7 +7,7 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
 
 jobs:
   badges:

.github/workflows/prebuilt.yaml

@@ -5,12 +5,8 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
   build_prebuilt:
@@ -18,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     env:
-      IMAGE_NAME: openpilot-prebuilt
+      PUSH_IMAGE: true
     steps:
     - name: Wait for green check mark
       if: ${{ github.event_name != 'workflow_dispatch' }}
@@ -31,13 +27,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Build Docker image
-      run: |
-        eval "$BUILD"
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
-    - name: Push to container registry
+    - name: Build and Push docker image
       run: |
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
-        docker tag $DOCKER_REGISTRY/$IMAGE_NAME:latest $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA
+        eval "$BUILD"

.github/workflows/repo-maintenance.yaml

@@ -1,4 +1,4 @@
-name: repo
+name: repo maintenance
 
 on:
   schedule:
@@ -6,13 +6,17 @@ on:
   workflow_dispatch:
 
 jobs:
-  pre-commit-autoupdate:
-    name: pre-commit autoupdate
+  updates:
+    name: updates
     runs-on: ubuntu-20.04
     container:
       image: ghcr.io/commaai/openpilot-base:latest
     steps:
     - uses: actions/checkout@v3
+    - name: poetry lock
+      run: |
+        pip install poetry
+        poetry lock
     - name: pre-commit autoupdate
       run: |
         git config --global --add safe.directory '*'
@@ -21,8 +25,8 @@ jobs:
       uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
       with:
         token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
-        commit-message: Update pre-commit hook versions
-        title: 'pre-commit: autoupdate hooks'
-        branch: pre-commit-updates
+        commit-message: Update Python packages and pre-commit hooks
+        title: 'Update Python packages and pre-commit hooks'
+        branch: auto-package-updates
         base: master
         delete-branch: true

.github/workflows/setup/action.yaml

@@ -1,15 +1,17 @@
 name: 'openpilot env setup'
 
 inputs:
-  save-cache:
-    default: false
+  git_lfs:
+    description: 'Whether or not to pull the git lfs'
     required: false
+    default: 'true'
 
 runs:
   using: "composite"
   steps:
     # do this after checkout to ensure our custom LFS config is used to pull from GitLab
     - shell: bash
+      if: ${{ inputs.git_lfs == 'true' }}
       run: git lfs pull
 
     # build cache
@@ -18,20 +20,32 @@ runs:
       run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
     - shell: bash
       run: echo "$CACHE_COMMIT_DATE"
-    - shell: bash
-      run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/master' || inputs.save-cache == 'false'
-    - id: scons-cache
-      # TODO: change the version to the released version
-      # when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+    - id: restore-scons-cache
+      uses: actions/cache/restore@v3
       with:
-        path: /tmp/scons_cache
+        path: .ci_cache/scons_cache
         key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
           scons-${{ env.CACHE_COMMIT_DATE }}-
           scons-
-
+    # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      if: steps.restore-scons-cache.outputs.cache-hit != 'true'
+      run: mkdir -p $GITHUB_WORKSPACE/.ci_cache/scons_cache
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    - id: setup-buildx-action
+      if: contains(runner.name, 'buildjet')
+      name: Set up Docker Buildx on buildjet to ensure a consistent cache
+      uses: docker/setup-buildx-action@v2
+      with:
+        driver: docker-container
     # build our docker image
     - shell: bash
-      run: eval ${{ env.BUILD }}
+      run: eval ${{ env.BUILD }}

.github/workflows/tools_tests.yaml

@@ -2,8 +2,8 @@ name: tools
 
 on:
   push:
-    branches-ignore:
-      - 'testing-closet*'
+    branches:
+      - master
   pull_request:
 
 concurrency:
@@ -13,75 +13,89 @@ concurrency:
 env:
   BASE_IMAGE: openpilot-base
   CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_REGISTRY: ghcr.io/commaai
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
 
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
-  BUILD_CL: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+  BUILD_CL: selfdrive/test/docker_build.sh cl
+
+  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
 
 jobs:
   plotjuggler:
     name: plotjuggler
     runs-on: ubuntu-20.04
+    timeout-minutes: 45
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Build Docker image
       run: eval "$BUILD"
-    - name: Unit test
+    - name: Build openpilot
+      timeout-minutes: 5
+      run: ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal --minimal"
+    - name: Test PlotJuggler
+      timeout-minutes: 2
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal && \
-                        apt-get update && \
-                        apt-get install -y libdw-dev libqt5svg5-dev libqt5x11extras5-dev && \
-                        cd /tmp/openpilot/tools/plotjuggler && \
-                        ./test_plotjuggler.py"
+        ${{ env.RUN }} "pytest tools/plotjuggler/"
 
   simulator:
     name: simulator
     runs-on: ubuntu-20.04
-    env:
-      IMAGE_NAME: openpilot-sim
     if: github.repository == 'commaai/openpilot'
+    timeout-minutes: 45
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Pull LFS
-      run: git lfs pull
-    - name: Build base image
-      run: eval "$BUILD"
+    - uses: ./.github/workflows/setup
     - name: Build base cl image
       run: eval "$BUILD_CL"
-    - name: Build simulator image
-      run: DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
-    - name: Push to container registry
+    - name: Setup to push to repo
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
+    - name: Build and push sim image
+      run: |
+        selfdrive/test/docker_build.sh sim
 
   docs:
     name: build docs
     runs-on: ubuntu-20.04
+    timeout-minutes: 45
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Build docker container
-      run: |
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
-    - name: Push docker container
+    - uses: ./.github/workflows/setup
+      with:
+        git_lfs: false
+    - name: Setup to push to repo
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/openpilot-docs:latest
-
+    - name: Build and push docs image
+      run: |
+        selfdrive/test/docker_build.sh docs
 
+  devcontainer:
+    name: devcontainer
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Setup Dev Container CLI
+      run: npm install -g @devcontainers/cli
+    - name: Build dev container image
+      run: devcontainer build --workspace-folder .
+    - name: Run dev container
+      run: devcontainer up --workspace-folder .
+    - name: Test environment
+      run: devcontainer exec --workspace-folder . scons --dry-run