Merge branch 'upstream/openpilot/master' into sync-20240608

# Conflicts:
#    cereal
#    opendbc
#    panda
#    release/files_common
#    selfdrive/monitoring/dmonitoringd.py
#    selfdrive/ui/translations/main_zh-CHS.ts
#    selfdrive/ui/translations/main_zh-CHT.ts

.github/workflows/auto_pr_review.yaml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   labeler:
-    name: apply labels
+    name: review
     permissions:
       contents: read
       pull-requests: write
@@ -14,17 +14,17 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: false
+
+    # Label PRs
     - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
 
-  pr_branch_check:
-    name: check branch
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    steps:
-    - uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
+    # Check PR target branch
+    - name: check branch
+      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
+      if: github.repository == 'commaai/openpilot'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -34,194 +34,20 @@ jobs:
         already-exists-action: close_this
         already-exists-comment: "Your PR should be made against the `master` branch"
 
-  comment:
-    runs-on: ubuntu-latest
-    steps:
-      - name: comment
-        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6
-        if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
-        with:
-          message: |
-              <!-- _(run_id **${{ github.run_id }}**)_ -->
-              Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
-              * Convert your PR to a draft unless it's ready to review
-              * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
-              * Before marking as "ready for review", ensure:
-                * the goal is clearly stated in the description
-                * all the tests are passing
-                * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
-                * include a route or your device' dongle ID if relevant
-          comment_tag: run_id
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  check-pr-template:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      issues: write
-      pull-requests: write
-      actions: read
-    if: false && github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
-    steps:
-      - uses: actions/github-script@v7
-        with:
-          script: |
-            // Comment to add to the PR if no template has been used
-            const NO_TEMPLATE_MESSAGE =
-              "It looks like you didn't use one of the Pull Request templates. Please check [the contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md). \
-            Also make sure that you didn't modify any of the checkboxes or headings within the template.";
-            // body data for future requests
-            const body_data = {
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-            };
-
-            // Utility function to extract all headings
-            const extractHeadings = (markdown) => {
-              const headingRegex = /^(#{1,6})\s+(.+)$/gm;
-              const boldTextRegex = /^(?:\*\*|__)(.+?)(?:\*\*|__)\s*$/gm;
-              const headings = [];
-              let headingMatch;
-              while ((headingMatch = headingRegex.exec(markdown))) {
-                headings.push(headingMatch[2].trim());
-              }
-              let boldMatch;
-              while ((boldMatch = boldTextRegex.exec(markdown))) {
-                headings.push(boldMatch[1].trim());
-              }
-              return headings;
-            };
-
-            // Utility function to extract all check box descriptions
-            const extractCheckBoxTexts = (markdown) => {
-              const checkboxRegex = /^\s*-\s*\[( |x)\]\s+(.+)$/gm;
-              const checkboxes = [];
-              let match;
-              while ((match = checkboxRegex.exec(markdown))) {
-                checkboxes.push(match[2].trim());
-              }
-              return checkboxes;
-            };
-
-            // Utility function to check if a list is a subset of another list
-            isSubset = (subset, superset) => {
-              return subset.every((item) => superset.includes(item));
-            };
-
-            // Utility function to check if a list of checkboxes is a subset of another list of checkboxes
-            isCheckboxSubset = (templateCheckBoxTexts, prTextCheckBoxTexts) => {
-              // Check if each template checkbox text is a substring of at least one PR checkbox text
-              // (user should be allowed to add additional text)
-              return templateCheckBoxTexts.every((item) => prTextCheckBoxTexts.some((element) => element.includes(item)))
-            }
-
-            // Get filenames of all currently checked-in PR templates
-            const template_contents = await github.rest.repos.getContent({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              path: ".github/PULL_REQUEST_TEMPLATE",
-            });
-            var template_filenames = [];
-            for (const content of template_contents.data) {
-              template_filenames.push(content.path);
-            }
-            console.debug("Received template filenames: " + template_filenames);
-            // Retrieve templates
-            var templates = [];
-            for (const template_filename of template_filenames) {
-              const template_response = await github.rest.repos.getContent({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                path: template_filename,
-              });
-              // Convert Base64 content back
-              const decoded_template = atob(template_response.data.content);
-              const headings = extractHeadings(decoded_template);
-              const checkboxes = extractCheckBoxTexts(decoded_template);
-              if (!headings.length && !checkboxes.length) {
-                console.warn(
-                  "Invalid template! Contains neither headings nor checkboxes, ignoring it: \n" +
-                    decoded_template
-                );
-              } else {
-                templates.push({ headings: headings, checkboxes: checkboxes });
-              }
-            }
-            // Retrieve the PR Body
-            const pull_request = await github.rest.issues.get({
-              ...body_data,
-            });
-            const pull_request_text = pull_request.data.body;
-            console.debug("Received Pull Request body: \n" + pull_request_text);
-
-            /* Check if the PR Body matches one of the templates
-            A template is defined by all headings and checkboxes it contains
-            We extract all Headings and Checkboxes from the PR text and check if any of the templates is a subset of that
-            */
-            const pr_headings = extractHeadings(pull_request_text);
-            const pr_checkboxes = extractCheckBoxTexts(pull_request_text);
-            console.debug("Found Headings in PR body:\n" + pr_headings);
-            console.debug("Found Checkboxes in PR body:\n" + pr_checkboxes);
-            var template_found = false;
-            // Iterate over each template to check if it applies
-            for (const template of templates) {
-              console.log(
-                "Checking for headings: [" +
-                  template.headings +
-                  "] and checkboxes: [" +
-                  template.checkboxes + "]"
-              );
-              if (
-                isCheckboxSubset(template.checkboxes, pr_checkboxes) &&
-                isSubset(template.headings, pr_headings)
-              ) {
-                console.debug("Found matching template!");
-                template_found = true;
-              }
-            }
-
-            // List comments from previous runs
-            var existing_comments = [];
-            const comments = await github.rest.issues.listComments({
-              ...body_data,
-            });
-            for (const comment of comments.data) {
-              if (comment.body === NO_TEMPLATE_MESSAGE) {
-                existing_comments.push(comment);
-              }
-            }
-
-            // Add a comment to the PR that it is not using a the template (but only if this comment does not exist already)
-            if (!template_found) {
-              var comment_already_sent = false;
-
-              // Add an 'in-bot-review' label since this PR doesn't have the template
-              github.rest.issues.addLabels({
-                ...body_data,
-                labels: ["in-bot-review"],
-              });
-
-              if (existing_comments.length < 1) {
-                github.rest.issues.createComment({
-                  ...body_data,
-                  body: NO_TEMPLATE_MESSAGE,
-                });
-              }
-            } else {
-              // If template has been found, delete any old comment about missing template
-              for (const existing_comment of existing_comments) {
-                github.rest.issues.deleteComment({
-                  ...body_data,
-                  comment_id: existing_comment.id,
-                });
-              }
-              // Remove the 'in-bot-review' label after the review is done and the PR has passed
-              github.rest.issues.removeLabel({
-                ...body_data,
-                name: "in-bot-review",
-              }).catch((error) => {
-                console.log("Label 'in-bot-review' not found, ignoring");
-              });
-            }
-
+    # Welcome comment
+    - name: comment
+      uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6
+      if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
+      with:
+        message: |
+            <!-- _(run_id **${{ github.run_id }}**)_ -->
+            Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
+            * Convert your PR to a draft unless it's ready to review
+            * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
+            * Before marking as "ready for review", ensure:
+              * the goal is clearly stated in the description
+              * all the tests are passing
+              * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
+              * include a route or your device' dongle ID if relevant
+        comment_tag: run_id
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/selfdrive_tests.yaml

@@ -47,13 +47,13 @@ jobs:
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: |
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "python selfdrive/manager/build.py"
+        ${{ env.RUN }} "python system/manager/build.py"
     - name: Run tests
       timeout-minutes: 3
       run: |
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
-                        MAX_EXAMPLES=5 $PYTEST selfdrive/car"
+                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
     - name: pre-commit
       timeout-minutes: 3
       run: |
@@ -62,7 +62,7 @@ jobs:
         cp pyproject.toml $STRIPPED_DIR
         cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all && chmod -R 777 /tmp/pre-commit"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
   build:
     strategy:
@@ -76,24 +76,8 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      with:
-        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
-    - uses: ./.github/workflows/compile-openpilot
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
-
-  docker_push:
-    name: docker push
-    strategy:
-      matrix:
-        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
-    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }}
-    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Setup to push to repo
+    - name: Setup docker push
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
@@ -101,12 +85,14 @@ jobs:
     - uses: ./.github/workflows/setup-with-retry
       with:
         docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
 
   docker_push_multiarch:
     name: docker push multiarch tag
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-    needs: [docker_push]
+    needs: [build]
     steps:
     - uses: actions/checkout@v4
       with:
@@ -134,24 +120,6 @@ jobs:
       timeout-minutes: 4
       run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
-  valgrind:
-    name: valgrind
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run valgrind
-      timeout-minutes: 1
-      run: |
-        ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py"
-    - name: Print logs
-      if: always()
-      run: cat selfdrive/test/valgrind_logs.txt
-
   unit_tests:
     name: unit tests
     runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
@@ -200,7 +168,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: .ci_cache/comma_download_cache
-        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
+        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }}
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
@@ -225,64 +193,18 @@ jobs:
       if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
       run: |
         ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  regen:
-    name: regen
-    runs-on: 'ubuntu-latest'
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v4
-      with:
-        path: .ci_cache/comma_download_cache
-        key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }}
-    - name: Build base Docker image
-      run: eval "$BUILD"
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run regen
-      timeout-minutes: 30
-      run: |
-        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-                           chmod -R 777 /tmp/comma_download_cache"
-
-  test_modeld:
-    name: model tests
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build base Docker image
-      run: eval "$BUILD"
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
     # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
     - name: Run model replay with ONNX
       timeout-minutes: 4
       run: |
         ${{ env.RUN }} "unset PYTHONWARNINGS && \
-                           ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
-                           coverage combine && \
-                           coverage xml"
-    - name: Run unit tests
+                        ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+                        coverage combine && coverage xml"
+    - name: Run regen
       timeout-minutes: 4
       run: |
-        ${{ env.RUN }} "unset PYTHONWARNINGS && \
-                           $PYTEST selfdrive/modeld"
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v4
       with:
@@ -298,7 +220,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job: [0, 1, 2, 3, 4]
+        job: [0, 1]
     steps:
     - uses: actions/checkout@v4
       with:
@@ -313,12 +235,12 @@ jobs:
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
-      timeout-minutes: 10
+      timeout-minutes: 20
       run: |
         ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
-        NUM_JOBS: 5
+        NUM_JOBS: 2
         JOB_ID: ${{ matrix.job }}
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v4

.github/workflows/stale.yaml

@@ -5,8 +5,8 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 7
-  DAYS_BEFORE_PR_STALE: 30
+  DAYS_BEFORE_PR_CLOSE: 3
+  DAYS_BEFORE_PR_STALE: 14
 
 jobs:
   stale:

.github/workflows/tools_tests.yaml

@@ -20,23 +20,6 @@ env:
 
 
 jobs:
-  plotjuggler:
-    name: plotjuggler
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      timeout-minutes: 5
-      run: ${{ env.RUN }} "scons -j$(nproc) cereal/ common/ opendbc/ --minimal"
-    - name: Test PlotJuggler
-      timeout-minutes: 2
-      run: |
-        ${{ env.RUN }} "pytest tools/plotjuggler/"
-
   simulator_build:
     name: simulator docker build
     runs-on: ubuntu-latest
@@ -66,8 +49,6 @@ jobs:
         submodules: true
     - run: git lfs pull
     - uses: ./.github/workflows/setup-with-retry
-    - name: Build base docker image
-      run: eval "$BUILD"
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
@@ -76,7 +57,7 @@ jobs:
         ${{ env.RUN }} "export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
                         source selfdrive/test/setup_xvfb.sh && \
                         source selfdrive/test/setup_vsound.sh && \
-                        CI=1 tools/sim/tests/test_metadrive_bridge.py"
+                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
 
   devcontainer:
     name: devcontainer
@@ -92,7 +73,7 @@ jobs:
     - name: Setup Dev Container CLI
       run: npm install -g @devcontainers/cli
     - name: Build dev container image
-      run: devcontainer build --workspace-folder .
+      run: ./scripts/retry.sh devcontainer build --workspace-folder .
     - name: Run dev container
       run: |
         mkdir -p /tmp/devcontainer_scons_cache/
@@ -104,21 +85,3 @@ jobs:
         devcontainer exec --workspace-folder . pip install pip-install-test
         devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
         devcontainer exec --workspace-folder . sudo touch /root/test.txt
-
-  notebooks:
-    name: notebooks
-    runs-on: ubuntu-latest
-    if: false && github.repository == 'commaai/openpilot'
-    timeout-minutes: 45
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Test notebooks
-      timeout-minutes: 3
-      run: |
-        ${{ env.RUN }} "pip install nbmake && pytest --nbmake tools/car_porting/examples/"