test

This reverts commit a0fe916e4b.

.github/labeler.yaml

@@ -44,7 +44,7 @@ subaru:
 
 tesla:
   - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/telsa/*'
+    - any-glob-to-all-files: 'selfdrive/car/tesla/*'
 
 toyota:
   - changed-files:

.github/workflows/badges.yaml

@@ -7,7 +7,7 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
   badges:

.github/workflows/ci_weekly_report.yaml

@@ -10,38 +10,30 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
 jobs:
   setup:
     if: github.repository == 'commaai/openpilot'
     runs-on: ubuntu-latest
     outputs:
-      ci_runs: ${{ steps.ci_runs_setup.outputs.value }}
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
     steps:
       - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
         run: |
-          CI_RUNS=${{ inputs.ci_runs || '50' }}
-          mylist="value=["
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
 
-          for i in $(seq 1 $CI_RUNS);
-          do
-              if [ $i != $CI_RUNS ]; then
-                  mylist+="\"$i\", "
-              else
-                  mylist+="\"$i\"]"
-              fi
-          done
-
-          echo "$mylist" >> $GITHUB_OUTPUT
-          echo "Number of CI runs for report: $CI_RUNS"
   ci_matrix_run:
     needs: [ setup ]
     strategy:
       fail-fast: false
-      matrix:
-        value: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
     uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
     with:
-      run_number: ${{ matrix.value }}
+      run_number: ${{ matrix.run_number }}
 
   report:
     needs: [ci_matrix_run]
@@ -62,62 +54,48 @@ jobs:
               })
             var report = {}
             jobs.slice(1, jobs.length-1).forEach(job => {
-              const jobName = job.name.split('/')[2].trim();
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
               report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
               switch (job.conclusion) {
                 case "success":
-                  report[jobName].successes.push(job.html_url); break;
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
                 case "failure":
-                  report[jobName].failures.push(job.html_url); break;
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
                 case "cancelled":
-                  report[jobName].cancelled.push(job.html_url); break;
+                  report[jobName].cancelled.push({ "run_number": run, "link": job.html_url }); break;
               }
             });
-            return JSON.stringify(report);
+            return JSON.stringify({"jobs": report});
 
       - name: Add job results to summary
         env:
           JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
         run: |
-          echo $JOB_RESULTS > job_results.json
-          generate_html_table() {
-            echo "<table>"
-            echo "<thead>"
-            echo "  <tr>"
-            echo "    <th>Job</th>"
-            echo "    <th>Succeeded ✅</th>"
-            echo "    <th>Failed ❌</th>"
-            echo "    <th>Cancelled (timed out) ⏰</th>"
-            echo "  </tr>"
-            echo "</thead>"
-            jq -r '
-              "<tbody>",
-              keys[] as $job |
-              "<tr>",
-              "  <td>\($job)</td>",
-              "  <td>",
-              "    <details>",
-              "      <summary>(\(.[$job].successes | length))</summary>",
-              "      \(.[$job].successes[])<br>",
-              "    </details>",
-              "  </td>",
-              "  <td>",
-              "    <details>",
-              "      <summary>(\(.[$job].failures | length))</summary>",
-              "      \(.[$job].failures[])<br>",
-              "    </details>",
-              "  </td>",
-              "  <td>",
-              "    <details>",
-              "      <summary>(\(.[$job].cancelled | length))</summary>",
-              "      \(.[$job].cancelled[])<br>",
-              "    </details>",
-              "  </td>",
-              "</tr>"
-            ' job_results.json
-            echo  "</tbody>"
-            echo  "</table>"
-          }
-          echo "# CI Job Summary" >> $GITHUB_STEP_SUMMARY
-          generate_html_table >> $GITHUB_STEP_SUMMARY
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
 
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/codeql.yml

@@ -0,0 +1,97 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master", "*-c3", "master*" ]
+  pull_request:
+    branches: [ "master", "*-c3", "master*" ]
+  schedule:
+    - cron: '39 7 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: c-cpp
+          build-mode: autobuild
+        - language: javascript-typescript
+          build-mode: none
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docs.yaml

@@ -15,31 +15,24 @@ concurrency:
   group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
-env:
-  BASE_IMAGE: openpilot-base
-
-  BUILD: selfdrive/test/docker_build.sh base
-
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
 jobs:
   docs:
     name: build docs
     runs-on: ubuntu-latest
-    timeout-minutes: 45
-    if: false  # TODO: replace this with the new docs
+    timeout-minutes: 1
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
+
+    # Build
     - name: Build docs
       run: |
-        ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make -j$(nproc) html"
+        # TODO: can we install just the "docs" dependency group without the normal deps?
+        pip install mkdocs
+        mkdocs build
 
+    # Push to docs.comma.ai
     - uses: actions/checkout@v4
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       with:
@@ -54,16 +47,17 @@ jobs:
         source release/identity.sh
 
         cd openpilot-docs
-
         git checkout --orphan tmp
         git rm -rf .
 
-        cp -r ../build/docs/html/ docs/
-        cp -r ../docs/README.md .
+        # copy over docs
+        cp -r ../docs_site/ docs/
+
+        # GitHub pages config
         touch docs/.nojekyll
         echo -n docs.comma.ai > docs/CNAME
-        git add -f .
 
+        git add -f .
         git commit -m "build docs"
 
         # docs live in different repo to not bloat openpilot's full clone size

.github/workflows/jenkins-pr-trigger.yaml

@@ -0,0 +1,45 @@
+name: jenkins scan
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+jobs:
+  # TODO: gc old branches in a separate job in this workflow
+  scan-comments:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.issue.pull_request }}
+    steps:
+    - name: Check for trigger phrase
+      id: check_comment
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const triggerPhrase = "trigger-jenkins";
+          const comment = context.payload.comment.body;
+          const commenter = context.payload.comment.user.login;
+
+          const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            username: commenter
+          });
+
+          const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin';
+
+          return (hasWriteAccess && comment.includes(triggerPhrase));
+        result-encoding: json
+
+    - name: Checkout repository
+      if: steps.check_comment.outputs.result == 'true'
+      uses: actions/checkout@v4
+      with:
+        ref: refs/pull/${{ github.event.issue.number }}/head
+
+    - name: Push to tmp-jenkins branch
+      if: steps.check_comment.outputs.result == 'true'
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git checkout -b tmp-jenkins-${{ github.event.issue.number }}
+        GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}

.github/workflows/mirror_to_gitlab.yaml

@@ -0,0 +1,46 @@
+name: Mirror to GitLab
+
+on:
+  push:
+  delete:
+  workflow_dispatch:  # This enables manual triggering
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for other instances of this workflow to conclude
+        uses: softprops/turnstyle@8db075d65b19bf94e6e8687b504db69938dc3c65
+        with:
+          same-branch-only: 'true'
+          abort-after-seconds: 300
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0  # Fetch full history
+
+      - name: Set up Git
+        run: |
+          git config --global user.name 'GitHub Action'
+          git config --global user.email 'action@github.com'
+
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Add GitLab public keys
+        run: |
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      # Note: If you have issues with "push rejected missing LFS" or something make sure you disabled LFS on the GITLAB repo if you intend to use a different LFS repo other than the target repo.
+      - name: Sync and commit changes
+        id: sync-and-commit
+        run: |
+          # Add GitLab remote
+          git remote add gitlab git@gitlab.com:sunnypilot/sunnyhaibin/sunnypilot-github-mirror.git
+          git push -u --force gitlab ${{ github.ref }}

.github/workflows/repo-maintenance.yaml

@@ -46,17 +46,13 @@ jobs:
         python3 -m ensurepip --upgrade
         pip3 install uv
         uv lock --upgrade
-    - name: pre-commit autoupdate
-      run: |
-        git config --global --add safe.directory '*'
-        pre-commit autoupdate
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:
         author: Vehicle Researcher <user@comma.ai>
         token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
-        commit-message: Update Python packages and pre-commit hooks
-        title: '[bot] Update Python packages and pre-commit hooks'
+        commit-message: Update Python packages
+        title: '[bot] Update Python packages'
         branch: auto-package-updates
         base: master
         delete-branch: true

.github/workflows/selfdrive_tests.yaml

@@ -25,7 +25,7 @@ env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
 
@@ -45,7 +45,6 @@ jobs:
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-    - uses: ./.github/workflows/setup-pre-commit
     - uses: ./.github/workflows/setup-with-retry
     - name: Check submodules
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
@@ -62,20 +61,22 @@ jobs:
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
                         MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
-    - name: pre-commit
-      timeout-minutes: 3
+    - name: static analysis
+      timeout-minutes: 1
       run: |
         cd $GITHUB_WORKSPACE
-        cp .pre-commit-config.yaml $STRIPPED_DIR
         cp pyproject.toml $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"
+        ${{ env.RUN }} "scripts/lint.sh"
 
   build:
     strategy:
       matrix:
-        arch: ${{ fromJson('["x86_64"]') }} # TODO: Re-add build test for aarch64 once we switched to ubuntu-2404
-    runs-on: ubuntu-latest
+        arch: ${{ fromJson(
+           ((github.repository == 'commaai/openpilot') &&
+              ((github.event_name != 'pull_request') ||
+               (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }}
     steps:
     - uses: actions/checkout@v4
       with:
@@ -84,7 +85,7 @@ jobs:
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        : # (TODO: Re-add this once we test other archs) echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
     - uses: ./.github/workflows/setup-with-retry
       with:
@@ -93,7 +94,7 @@ jobs:
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
 
   build_mac:
-    name: build macos
+    name: build macOS
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v4
@@ -103,11 +104,36 @@ jobs:
     - name: Install dependencies
       run: ./tools/mac_setup.sh
       env:
-        SKIP_PROMPT: 1
         # package install has DeprecationWarnings
         PYTHONWARNINGS: default
-    - name: Test openpilot environment
-      run: . .venv/bin/activate && scons -h
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - name: Getting scons cache
+      uses: 'actions/cache@v4'
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}-macos
+    - name: Building openpilot
+      run: . .venv/bin/activate && scons -j$(nproc)
+
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [build]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: false
+    - name: Setup docker
+      run: |
+        $DOCKER_LOGIN
+    - name: Merge x64 and arm64 tags
+      run: |
+        export PUSH_IMAGE=true
+        scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
 
   static_analysis:
     name: static analysis
@@ -118,13 +144,12 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-pre-commit
     - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: pre-commit
-      timeout-minutes: 4
-      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit"
+    - name: static analysis
+      timeout-minutes: 1
+      run: ${{ env.RUN }} "scripts/lint.sh"
 
   unit_tests:
     name: unit tests
@@ -145,6 +170,7 @@ jobs:
       timeout-minutes: 15
       run: |
         ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
                         $PYTEST --timeout 60 -m 'not slow' && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
@@ -309,8 +335,10 @@ jobs:
             })
 
   create_ui_report:
+    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
     name: Create UI Report
     runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -322,59 +350,10 @@ jobs:
         run: >
             ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                             source selfdrive/test/setup_xvfb.sh &&
+                            export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' &&
                             python selfdrive/ui/tests/test_ui/run.py"
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:
-          name: report-${{ inputs.run_number }}
-          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}
-      - name: Get changes to selfdrive/ui
-        if: ${{ github.event_name == 'pull_request' }}
-        id: changed-files
-        uses: tj-actions/changed-files@v44
-        with:
-         files: |
-            selfdrive/ui/**
-      - name: Checkout ci-artifacts
-        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
-        uses: actions/checkout@v4
-        with:
-          repository: commaai/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/ci-artifacts
-          ref: master
-      - name: Push Screenshots
-        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
-        working-directory: ${{ github.workspace }}/ci-artifacts
-        run: |
-          git checkout -b openpilot/pr-${{ github.event.pull_request.number }}
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          sudo mv ${{ github.workspace }}/selfdrive/ui/tests/test_ui/report/screenshots/* .
-          git add .
-          git commit -m "screenshots for PR #${{ github.event.pull_request.number }}"
-          git push origin openpilot/pr-${{ github.event.pull_request.number }} --force
-      - name: Comment Screenshots on PR
-        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          message: |
-            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
-            ## UI Screenshots
-            <table>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/homescreen.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_map.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_sidebar.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_network.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_device.png"></td>
-              </tr>
-            </table>
-          comment_tag: run_id_screenshots
-          pr_number: ${{ github.event.pull_request.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          name: report-${{ github.event.number }}
+          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-pre-commit/action.yaml

@@ -1,12 +0,0 @@
-name: 'set up pre-commit environment'
-
-runs:
-  using: "composite"
-  steps:
-    - uses: ./.github/workflows/auto-cache
-      with:
-        path: .ci_cache/pre-commit
-        key: pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
-        restore-keys: |
-          pre-commit-
-        save: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' }}

.github/workflows/stale.yaml

@@ -5,8 +5,8 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 3
-  DAYS_BEFORE_PR_STALE: 14
+  DAYS_BEFORE_PR_CLOSE: 2
+  DAYS_BEFORE_PR_STALE: 9
 
 jobs:
   stale:

.github/workflows/tools_tests.yaml

@@ -21,7 +21,7 @@ env:
 
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: docker run --shm-size 2G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 
 jobs:
@@ -39,7 +39,8 @@ jobs:
         ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run bridge test
       run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+        ${{ env.RUN }} "export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
+                        source selfdrive/test/setup_xvfb.sh && \
                         source selfdrive/test/setup_vsound.sh && \
                         CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
 
@@ -73,12 +74,6 @@ jobs:
           scons-${{ runner.arch }}-ubuntu2004
     - name: Building openpilot
       run: uv run scons -u -j$(nproc)
-    - name: Saving scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   devcontainer:
     name: devcontainer

.github/workflows/ui_preview.yaml

@@ -0,0 +1,91 @@
+name: "ui preview"
+on:
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened, edited]
+    branches:
+      - 'master'
+    paths:
+      - 'selfdrive/ui/**'
+
+env:
+  UI_JOB_NAME: "Create UI Report"
+
+jobs:
+  preview:
+    if: github.repository == 'commaai/openpilot'
+    name: preview
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Waiting for ui test to start
+        run: sleep 30
+
+      - name: Wait for ui report
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: ${{ env.UI_JOB_NAME }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allowed-conclusions: success
+          wait-interval: 20
+
+      - name: Get workflow run ID
+        id: get_run_id
+        run: |
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ github.event.pull_request.head.sha }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+
+      - name: Checkout ci-artifacts
+        uses: actions/checkout@v4
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/ci-artifacts
+          ref: master
+
+      - name: Download artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          run_id: ${{ steps.get_run_id.outputs.run_id }}
+          search_artifacts: true
+          name: report-${{ github.event.number }}
+          path: ${{ github.workspace }}/ci-artifacts
+
+      - name: Push Screenshots
+        working-directory: ${{ github.workspace }}/ci-artifacts
+        run: |
+          git checkout -b openpilot/pr-${{ github.event.number }}
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git add ${{ github.workspace }}/ci-artifacts/*
+          git commit -m "screenshots for PR #${{ github.event.number }}"
+          git push origin openpilot/pr-${{ github.event.number }} --force
+
+      - name: Comment Screenshots on PR
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
+            ## UI Screenshots
+            <table>
+              <tr>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/homescreen.png"></td>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad.png"></td>
+              </tr>
+              <tr>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_map.png"></td>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_sidebar.png"></td>
+              </tr>
+              <tr>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/settings_network.png"></td>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/settings_device.png"></td>
+              </tr>
+            </table>
+          comment_tag: run_id_screenshots
+          pr_number: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -14,6 +14,8 @@ model2.png
 a.out
 .hypothesis
 
+/docs_site/
+
 *.dylib
 *.DSYM
 *.d
@@ -54,6 +56,7 @@ selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 selfdrive/modeld/_modeld
+selfdrive/modeld/_navmodeld
 selfdrive/modeld/_dmonitoringmodeld
 /src/
 
@@ -102,4 +105,4 @@ Pipfile
 ### VisualStudioCode Patch ###
 # Ignore all local history of files
 .history
-.ionide
+.ionide

.gitlab-ci.yml

@@ -0,0 +1,240 @@
+variables:
+  BUILD_DIR: "/data/openpilot"
+  OUTPUT_DIR: "${CI_PROJECT_DIR}/output"
+  CI_DIR: "${CI_PROJECT_DIR}/release/ci"
+  VERSION: 'echo $(date "+%Y.%m.%d")'
+  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
+  DEV_C3_SOURCE_BRANCH: "master-dev-c3"
+  GIT_CONFIG_USER_EMAIL: "gitlab@pipeline.com"
+  GIT_CONFIG_USER_NAME: "Gitlab Pipeline"
+  PUBLIC_REPO_URL: "https://github.com/sunnyhaibin/sunnypilot"
+  BASE_BUILD_NUMER: 3000
+
+  EXTRA_VERSION_IDENTIFIER: "${CI_PIPELINE_IID}"
+  NEW_BRANCH: ${CI_COMMIT_REF_NAME}-prebuilt
+
+stages:
+  - build
+  - sanity
+  - publish
+  - notify
+
+default:
+  retry: 2
+  tags:
+    - sunnypilot
+    - x86
+
+.default_before_script: &default_before_script
+  - 'if [ "$EXTRA_VERSION_IDENTIFIER" = "$CI_PIPELINE_IID" ]; then export EXTRA_VERSION_IDENTIFIER=$((CI_PIPELINE_IID + BASE_BUILD_NUMER)); fi'
+  - 'export VERSION=$(eval $VERSION)-${EXTRA_VERSION_IDENTIFIER}'
+  - 'mkdir -p "${BUILD_DIR}/"'
+  - 'git config --global user.email "${GIT_CONFIG_USER_EMAIL}"'
+  - 'git config --global user.name "${GIT_CONFIG_USER_NAME}"'
+
+
+workflow: # If running on any branch other than main.
+  rules:
+    # We are an MR, but it's a draft, we won't proceed with anything.
+    - if: '$CI_MERGE_REQUEST_TITLE =~ /^wip:/i || $CI_MERGE_REQUEST_TITLE =~ /^draft:/i'
+      when: never
+
+    # Below are the rules when a commit is done (code has been added to the branch)
+    # Commit to master-dev-c3
+    - if: $CI_COMMIT_REF_NAME == $DEV_C3_SOURCE_BRANCH
+      variables:
+        EXTRA_VERSION_IDENTIFIER: "${CI_PIPELINE_IID}"
+        NEW_BRANCH: "dev-c3"
+        AUTO_BUILD: true
+      when: always
+    #commit made to main (master)
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        NEW_BRANCH: "staging-c3"
+        VERSION: 'cat common/version.h | grep COMMA_VERSION | sed -e "s/[^0-9|.]//g"'
+        EXTRA_VERSION_IDENTIFIER: "staging"
+        AUTO_PUBLISH: true
+        AUTO_BUILD: true
+      when: always
+    # if tag
+    - if: $CI_COMMIT_TAG
+      variables:
+        NEW_BRANCH: "release-c3"
+        VERSION: 'cat common/version.h | grep COMMA_VERSION | sed -e "s/[^0-9|.]//g"'
+        EXTRA_VERSION_IDENTIFIER: "release"
+        AUTO_BUILD: true
+    - when: always
+
+
+build:
+  stage: build
+  cache:
+    key: "${CI_COMMIT_REF_SLUG}"
+    paths:
+      - "${CI_DIR}/scons_cache"
+  before_script:
+    - find $BUILD_DIR/ -mindepth 1 -delete
+    - *default_before_script
+    - "echo Starting build stage..."
+    - "echo BUILD_DIR: ${BUILD_DIR}"
+    - "echo CI_DIR: ${CI_DIR}"
+    - "echo VERSION: ${VERSION}"
+    - "echo CI_COMMIT_REF_NAME: ${CI_COMMIT_REF_NAME}"
+    - git config --global --add safe.directory ${CI_PROJECT_DIR}
+  script:
+    - export PYTHONPATH="$BUILD_DIR"
+    - "echo Building Panda..."
+    - scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache ${CI_PROJECT_DIR}/panda
+    - mkdir -p ${BUILD_DIR}
+    - ls -la ${BUILD_DIR}
+    - "echo Building Rest..."
+    - ./release/release_files.py | sort | uniq | rsync -vrRl --files-from=- . $BUILD_DIR/
+    # - cp -pR --parents $(cat release/files_common release/files_tici | sort | uniq) $BUILD_DIR/ 2> >(grep -v 'warning:' >&2)
+    - cd $BUILD_DIR
+    - sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py # comment panda jungle when prebuilt
+    - scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache --minimal
+    - touch ${BUILD_DIR}/prebuilt
+    - sudo rm -rf ${OUTPUT_DIR}
+    - mkdir -p ${OUTPUT_DIR}
+    # We first include the paths we want to keep, even if we later will be excluding the other things on those paths
+    - rsync -avm
+      --include='**/panda/board/'
+      --include='**/panda/board/obj'
+      --include='**/panda/board/obj/panda.bin.signed'
+      --include='**/panda/board/obj/panda_h7.bin.signed'
+      --include='**/panda/board/obj/bootstub.panda.bin'
+      --include='**/panda/board/obj/bootstub.panda_h7.bin'
+      --exclude='.sconsign.dblite'
+      --exclude='*.a'
+      --exclude='*.A'
+      --exclude='*.o'
+      --exclude='*.O'
+      --exclude='*.os'
+      --exclude='*.OS'
+      --exclude='*.pyc'
+      --exclude='*.PYC'
+      --exclude='moc_*'
+      --exclude='MOC_*'
+      --exclude='*.cc'
+      --exclude='*.CC'
+      --exclude='Jenkinsfile'
+      --exclude='supercombo.onnx'
+      --exclude='**/panda/board/*'
+      --exclude='**/panda/board/obj/**'
+      --exclude='**/panda/certs/'
+      --exclude='**/panda/crypto/'
+      --exclude='**/release/'
+      --exclude='**/.github/'
+      --exclude='**/selfdrive/ui/replay/'
+      --exclude='**/__pycache__/'
+      --exclude='**/selfdrive/ui/*.h'
+      --exclude='**/selfdrive/ui/**/*.h'
+      --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/'
+      --exclude='**/.git/'
+      --exclude='**/SConstruct'
+      --exclude='**/SConscript'
+      --delete-excluded
+      --chown=comma:comma
+      ${BUILD_DIR}/ ${OUTPUT_DIR}/
+  after_script:
+    # cleanup build dir after doing work
+    - find $BUILD_DIR/ -mindepth 1 -delete
+  artifacts:
+    paths:
+      - ${OUTPUT_DIR}/
+  tags: [ 'sunnypilot', 'tici' ]
+  rules:
+    - if: $AUTO_BUILD
+      when: always
+    - when: manual
+
+.publish_base: &publish_base
+  image: alpine
+  variables:
+    GIT_SUBMODULE_STRATEGY: normal
+  stage: publish
+  needs:
+    - job: build
+      artifacts: true
+  before_script:
+    - 'apk update && apk upgrade'
+    - 'apk add git bash openssh'
+    - 'eval $(ssh-agent -s)'
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - 'mkdir -p ~/.ssh/'
+    - 'chmod 700 ~/.ssh'
+    - export HOST=$(echo ${GIT_ORIGIN} | cut -d'@' -f2 | cut -d':' -f1)
+    - echo fetching ssh pub keys for ${HOST}
+    - 'ssh-keyscan -H ${HOST} >> ~/.ssh/known_hosts' # Adding gitlab to trusted
+    - 'chmod 644 ~/.ssh/known_hosts'
+    - *default_before_script
+  script:
+    - echo "${GIT_ORIGIN}"
+    - echo "Calling to publish [${CI_DIR}/publish.sh ${CI_PROJECT_DIR} ${OUTPUT_DIR} ${NEW_BRANCH} ${VERSION} ${GIT_ORIGIN}]"
+    - git config --global --add safe.directory ${OUTPUT_DIR}
+    - $CI_DIR/publish.sh "${CI_PROJECT_DIR}" "${OUTPUT_DIR}" "${NEW_BRANCH}" "${VERSION}" "${GIT_ORIGIN}" "-${EXTRA_VERSION_IDENTIFIER}"
+  allow_failure: false
+
+publish to private gitlab prebuilt:
+  extends: ".publish_base"
+  variables:
+    GIT_ORIGIN: git@gitlab.com:sunnypilot/public/sunnypilot-prebuilts.git
+  rules:
+    - if: $AUTO_BUILD
+      when: on_success
+    - if: $CI_MERGE_REQUEST_IID
+      when: on_success
+    - when: manual
+
+publish to public github prebuilt:
+  extends: ".publish_base"
+  variables:
+    GIT_ORIGIN: git@github.com:sunnyhaibin/sunnypilot.git
+    GIT_CONFIG_USER_EMAIL: "jason.wen@sunnypilot.ai"
+    GIT_CONFIG_USER_NAME: "Jason Wen"
+  rules:
+    - if: $AUTO_PUBLISH
+      when: on_success
+    - when: manual
+
+.notify_discord: &notify_discord
+  image: alpine
+  stage: notify
+  needs: ["build"]
+  variables:
+    DISCORD_HOOK: "${DISCORD_MANUAL_BUILD_WEBHOOK_URL}" # Default hook if not overriden by children
+  before_script:
+    - 'apk add curl jq envsubst'
+  script:
+    - echo using [${TEMPLATE}]
+    - cat release/ci/${TEMPLATE} | envsubst | tee payload.json
+    - 'curl -X POST -H "Content-Type: application/json" -d "$(cat payload.json)" ${DISCORD_HOOK} | jq .'
+  rules:
+    - if: $NEW_BRANCH
+      when: on_success
+    - when: never
+  allow_failure: true
+
+notify pending action:
+  extends: ".notify_discord"
+  variables:
+    TEMPLATE: "discord_template_notify_dev_private.json"
+  before_script:
+    - !reference [".notify_discord", "before_script"]
+    - export AVATAR_URL=$(curl -s -X GET "https://gitlab.com/api/v4/avatar?email=${GITLAB_USER_EMAIL}" | jq -r '.avatar_url')
+
+notify new dev build:
+  extends: ".notify_discord"
+  stage: notify
+  needs: ["publish to public github prebuilt"] # This notify shall only happen after a publish to github public
+  variables:
+    TEMPLATE: "discord_template_notify_dev_public.json"
+  before_script:
+    - !reference [".notify_discord", "before_script"]
+    - export EXTRA_VERSION_IDENTIFIER=$((CI_PIPELINE_IID + BASE_BUILD_NUMER))
+  rules:
+    - if: $NEW_BRANCH == "dev-c3"
+      variables:
+        DISCORD_HOOK: "${DISCORD_NEW_BUILD_WEBHOOK_URL}" # Overriding hook because we know we are dev-c3
+    - !reference [".notify_discord", "rules"]

.gitmodules

@@ -1,21 +1,18 @@
 [submodule "panda"]
   path = panda
-  url = ../../sunnyhaibin/panda.git
+  url = https://github.com/sunnyhaibin/panda.git
 [submodule "opendbc"]
-  path = opendbc
-  url = ../../sunnyhaibin/opendbc.git
+  path = opendbc_repo
+  url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = ../../commaai/msgq.git
+  url = https://github.com/sunnypilot/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
-  url = ../../commaai/rednose.git
-[submodule "body"]
-  path = body
-  url = ../../commaai/body.git
+  url = https://github.com/commaai/rednose.git
 [submodule "teleoprtc_repo"]
   path = teleoprtc_repo
-  url = ../../commaai/teleoprtc
+  url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/geohot/tinygrad.git
+  url = https://github.com/tinygrad/tinygrad.git

.idea/customTargets.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CLionExternalBuildManager">
+    <target id="a62f99e8-5ec4-434c-8122-49efed5af108" name="Poetry SCons Build Debug" defaultType="TOOL">
+      <configuration id="b93ec964-16e5-4962-a12e-3ed360ce8f02" name="Poetry SCons Build Debug">
+        <build type="TOOL">
+          <tool actionId="Tool_External Tools_Poetry SCons Build Debug" />
+        </build>
+        <clean type="TOOL">
+          <tool actionId="Tool_External Tools_Poetry SCons Clean" />
+        </clean>
+      </configuration>
+    </target>
+    <target id="edd8ad9d-183b-467c-a355-0d9a0ecab026" name="Poetry SCons Build Release" defaultType="TOOL">
+      <configuration id="09523339-5ce3-4223-ab9e-904f38ad7752" name="Poetry SCons Build Release">
+        <build type="TOOL">
+          <tool actionId="Tool_External Tools_Poetry SCons Build Release" />
+        </build>
+        <clean type="TOOL">
+          <tool actionId="Tool_External Tools_Poetry SCons Clean" />
+        </clean>
+      </configuration>
+    </target>
+  </component>
+</project>

.idea/tools/External Tools.xml

@@ -0,0 +1,23 @@
+<toolSet name="External Tools">
+  <tool name="Poetry SCons Build Debug" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
+    <exec>
+      <option name="COMMAND" value="bash" />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db --ccflags=\&quot;-fno-inline\&quot;&quot;" />
+      <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
+    </exec>
+  </tool>
+  <tool name="Poetry SCons Clean" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
+    <exec>
+      <option name="COMMAND" value="bash" />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -c&quot; " />
+      <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
+    </exec>
+  </tool>
+  <tool name="Poetry SCons Build Release" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
+    <exec>
+      <option name="COMMAND" value="bash" />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db&quot; " />
+      <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
+    </exec>
+  </tool>
+</toolSet>

.importlinter

@@ -0,0 +1,41 @@
+[importlinter]
+root_packages =
+    openpilot
+
+[importlinter:contract:1]
+name = Forbid imports from openpilot.selfdrive.car to openpilot.system
+type = forbidden
+source_modules =
+    openpilot.selfdrive.car
+forbidden_modules =
+    openpilot.system
+    openpilot.body
+    openpilot.docs
+    openpilot.msgq
+    openpilot.panda
+    openpilot.rednose
+    openpilot.release
+    openpilot.teleoprtc
+    openpilot.tinygrad
+ignore_imports =
+    openpilot.selfdrive.car.card -> openpilot.common.realtime
+    openpilot.selfdrive.car.card -> openpilot.selfdrive.controls.lib.events
+    openpilot.selfdrive.car.interfaces -> openpilot.selfdrive.controls.lib.events
+    openpilot.selfdrive.car.tests.test_models -> openpilot.tools.lib.logreader
+    openpilot.selfdrive.car.tests.test_models -> openpilot.selfdrive.car.card
+    openpilot.selfdrive.car.tests.test_models -> openpilot.tools.lib.route
+    openpilot.selfdrive.car.tests.test_models -> openpilot.system.hardware.hw
+    openpilot.selfdrive.car.tests.test_models -> openpilot.selfdrive.test.helpers
+    openpilot.selfdrive.car.isotp_parallel_query -> openpilot.common.swaglog
+    openpilot.selfdrive.car.fw_versions -> openpilot.common.swaglog
+    openpilot.selfdrive.car.disable_ecu -> openpilot.common.swaglog
+    openpilot.selfdrive.car.vin -> openpilot.common.swaglog
+    openpilot.selfdrive.car.ecu_addrs -> openpilot.common.swaglog
+    openpilot.selfdrive.car.car_helpers -> openpilot.common.swaglog
+    openpilot.selfdrive.car.car_helpers -> openpilot.system.version
+    openpilot.selfdrive.car.interfaces -> openpilot.selfdrive.controls.lib.drive_helpers
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.latcontrol_angle
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.longcontrol
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.latcontrol_torque
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.latcontrol_pid
+unmatched_ignore_imports_alerting = warn

.pre-commit-config.yaml

@@ -1,98 +0,0 @@
-exclude: '^(tinygrad_repo)'
-repos:
--   repo: meta
-    hooks:
-    -   id: check-hooks-apply
-    -   id: check-useless-excludes
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-    hooks:
-    -   id: check-ast
-        exclude: '^(third_party)/'
-    -   id: check-json
-        exclude: '.devcontainer/devcontainer.json|.vscode/'  # these support JSON with comments
-    -   id: check-toml
-    -   id: check-xml
-    -   id: check-yaml
-    -   id: check-merge-conflict
-    -   id: check-symlinks
-    -   id: check-executables-have-shebangs
-    -   id: check-shebang-scripts-are-executable
-    -   id: check-added-large-files
-        exclude: '(docs/CARS.md)|(uv.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
-        args:
-        - --maxkb=120
-        - --enforce-all
--   repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
-    hooks:
-    -   id: codespell
-        exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(uv.lock)'
-        args:
-        # if you've got a short variable name that's getting flagged, add it here
-        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
-        - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
--   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.0
-    hooks:
-    -   id: ruff
-        exclude: '^(third_party/)|(msgq/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
--   repo: local
-    hooks:
-    -   id: mypy
-        name: mypy
-        entry: mypy
-        language: system
-        types: [python]
-        args:
-        - --local-partial-types
-        - --explicit-package-bases
-        exclude: '^(third_party/)|(body/)|(msgq/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
--   repo: local
-    hooks:
-    -   id: cppcheck
-        name: cppcheck
-        entry: cppcheck
-        language: system
-        types: [c++]
-        exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)'
-        args:
-        - --error-exitcode=1
-        - --language=c++
-        - --quiet
-        - --force
-        - -j8
-        - --library=qt
-        - --include=third_party/kaitai/kaitaistream.h
--   repo: https://github.com/cpplint/cpplint
-    rev: 1.6.1
-    hooks:
-    -   id: cpplint
-        exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)'
-        args:
-        - --quiet
-        - --counting=total
-        - --linelength=240
-        # https://google.github.io/styleguide/cppguide.html
-        # relevant rules are whitelisted, see all options with: cpplint --filter=
-        - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces
--   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.16.2
-    hooks:
-    -   id: cython-lint
-        exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)'
-        args:
-        - --max-line-length=240
-        - --ignore=E111, E302, E305
--   repo: local
-    hooks:
-    -   id: test_translations
-        name: test translations
-        entry: pytest selfdrive/ui/tests/test_translations.py
-        language: system
-        pass_filenames: false
-        files: '^selfdrive/ui/translations/'
--   repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.28.6
-    hooks:
-    -   id: check-github-workflows

.run/Build Debug.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Build Debug" type="CLionExternalRunConfiguration" factoryName="Application" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://$ProjectFileDir$/selfdrive/ui" PASS_PARENT_ENVS_2="true" PROJECT_NAME="openpilot-special" TARGET_NAME="Poetry SCons Build Debug" CONFIG_NAME="Poetry SCons Build Debug" RUN_PATH="ui">
+    <envs>
+      <env name="QT_DBL_CLICK_DIST" value="150" />
+    </envs>
+    <method v="2">
+      <option name="CLION.EXTERNAL.BUILD" enabled="true" />
+    </method>
+  </configuration>
+</component>

.run/Build Release.run.xml

@@ -0,0 +1,10 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Build Release" type="CLionExternalRunConfiguration" factoryName="Application" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://$ProjectFileDir$/selfdrive/ui" PASS_PARENT_ENVS_2="true" PROJECT_NAME="openpilot-special" TARGET_NAME="Poetry SCons Build Release" CONFIG_NAME="Poetry SCons Build Release" RUN_PATH="ui">
+    <envs>
+      <env name="QT_DBL_CLICK_DIST" value="150" />
+    </envs>
+    <method v="2">
+      <option name="CLION.EXTERNAL.BUILD" enabled="true" />
+    </method>
+  </configuration>
+</component>

.vscode/launch.json

@@ -7,6 +7,7 @@
       "description": "Select the process to debug",
       "options": [
         "selfdrive/controls/controlsd.py",
+        "selfdrive/navd/navd.py",
         "system/timed/timed.py",
         "tools/sim/run_bridge.py"
       ]

CHANGELOGS.md

@@ -1,6 +1,64 @@
 sunnypilot - 0.9.8.0 (2024-xx-xx)
 ========================
 * Always on driver monitoring toggle
+************************
+* UPDATED: Synced with commaai's openpilot
+  * master commit 4ef757c (July 06, 2024)
+* NEW❗: Default Driving Model: Notre Dame (July 01, 2024)
+* NEW❗: Longitudinal: Acceleration Personality thanks to kegman, rav4kumar, and arne1282!
+  * Select from three distinct acceleration personalities: Eco, Normal, and Sport
+  * Acceleration personalities are integrated directly into the model's acceleration matrix and can be activated in real-time!
+* NEW❗: Longitudinal: Dynamic Personality thanks to rav4kumar!
+  * Dynamically adjusts following distance and reaction based on your "Driving Personality" setting
+  * Personalities adapt in real-time to your speed and the distance to the lead car
+  * Provides a more responsive and tailored driving experience compared to predefined settings
+* UPDATED: Driving Personality: Updated mode names
+  * Aggressive, Moderate, Standard, Relaxed
+* NEW❗: Hyundai CAN: Enable Cruise Main by Default
+  * Set CRUISE MAIN to ON by default when the car starts, without engaging MADS
+  * This feature only applies when "openpilot Longitudinal Control (Alpha)" is enabled under the "Toggles" menu
+* NEW❗: Toyota - Enhanced Blind Spot Monitor (BSM) thanks to arne182, rav4kumar, and eFiniLan!
+  * Enables Blind Spot Monitor (BSM) signals parsing in sunnypilot using the factory Blind Spot Monitor (BSM)
+  * sunnypilot will use debugging CAN messages to receive unfiltered BSM signals, allowing detection of more objects
+  * Supported platforms
+    * RAV4 TSS1, equipped with factory Blind Spot Monitoring (BSM)
+    * Lexus LSS1, equipped with factory Blind Spot Monitoring (BSM)
+    * Toyota TSS1/1.5, equipped with factory Blind Spot Monitoring (BSM)
+    * Prius TSS2, equipped with factory Blind Spot Monitoring (BSM)
+  * NOTE: Only enable this feature if your Toyota/Lexus vehicle has factory Blind Spot Monitor equipped, and mentioned in the supported platforms list
+* UPDATED: Toyota: TSS2 longitudinal: Custom Tuning
+  * Re-tuned and tested by the community (July 1, 2024)
+* UPDATED: Driving Model Selector v5
+  * NEW❗: Driving Model additions
+    * Notre Dame (July 01, 2024) - NDv3
+* UPDATED: Neural Network Lateral Control (NNLC)
+  * NEW❗: Remove Lateral Jerk Response (Alpha)
+  * FIXED: Hotfix for "lazy" steering performance in tighter curves thanks to twilsonco!
+* UPDATED: Toyota: Continued support for Smart DSU (SDSU) and Radar CAN Filter
+  * In response to the official deprecation of support for Smart DSU (SDSU) and Radar CAN Filter in the upstream ([commaai/openpilot#32777](https://github.com/commaai/openpilot/pull/32777)), sunnypilot will continue maintaining software support for Smart DSU (SDSU) and Radar CAN Filter
+* UPDATED: Continued support for Mapbox navigation
+  * In response to the official temporary deprecation of support for Mapbox navigation in the upstream ([commaai/openpilot#32773](https://github.com/commaai/openpilot/pull/32773)), sunnypilot will continue maintaining software support for Mapbox navigation
+* NEW❗: Toyota - Automatic Brake Hold (AHB) thanks to AlexandreSato!
+  * When you stop the vehicle completely by depressing the brake pedal, sunnypilot will activate Auto Brake Hold
+  * NOTE: Only for Toyota/Lexus vehicles with TSS2/LSS2
+* NEW❗: Toyota - Automatic Door Locking and Unlocking thanks to AlexandreSato, cydia2020, and dragonpilot-community!
+  * Auto Lock by Speed: All doors are automatically locked when vehicle speed is approximately 6 mph (10 km/h) or higher
+  * Auto Unlock by Shift to P: All doors are automatically unlocked when shifting the shift lever to P
+* FIXED: Driving Personality:
+  * Maniac mode now correctly enforced when selected
+* FIXED: Experimental Model Distance Button Hold
+  * Experimental Model toggle with distance button hold no longer changes Personality
+  * Personality setting remains consistent when switching between Chill and Experimental Mode
+* UI Updates
+  * Display Metrics Below Chevron
+    * NEW❗: Time to Lead Car
+      * Displays the time to reach the position previously occupied by the lead car
+    * NEW❗: Display Distance, Speed, and Time to Lead Car simultaneously
+* Ford F-150 2022-23 support
+* Ford F-150 Lightning 2021-23 support
+* Ford Mustang Mach-E 2021-23 support
+* Hyundai Kona Electric Non-SCC 2019 support thanks to NikitaNekrasov!
+* Kia Ceed Plug-in Hybrid Non-SCC 2022 support thanks to TerminatorNL!
 
 sunnypilot - 0.9.7.1 (2024-06-13)
 ========================
@@ -37,6 +95,8 @@ sunnypilot - 0.9.7.1 (2024-06-13)
   * Force sunnypilot in the offroad state even when the car is on
   * When Forced Offroad mode is on, allows changing offroad-only settings even when the car is turned on
   * To engage/disengage Force Offroad, go to Settings -> Device panel
+* NEW❗: Ford CAN-FD longitudinal
+  * NEW❗: Parse speed limit sign recognition from camera for certain supported platforms
 * UPDATED: Auto Lane Change Timer -> Auto Lane Change by Blinker
   * NEW❗: New "Off" option to disable lane change by blinker
 * UPDATED: Pause Lateral Below Speed with Blinker
@@ -44,6 +104,8 @@ sunnypilot - 0.9.7.1 (2024-06-13)
     * Pause lateral actuation with blinker when traveling below the desired speed selected. Default is 20 MPH or 32 km/h.
 * UPDATED: Hyundai CAN Longitudinal
   * Auto-enable radar tracks on platforms with applicable Mando radar
+* UPDATED: Hyundai CAN-FD Radar-based SCC
+  * Longitudinal support for CAN-FD Radar-based SCC cars
 * UPDATED: Hyundai CAN-FD Camera-based SCC
   * NEW❗: Parse lead info for camera-based SCC platforms with longitudinal support
     * Improve lead tracking when using openpilot longitudinal

HOW-TOS.md

@@ -0,0 +1,77 @@
+# How Tos
+This page is a repository of useful how-tos as a supplement for additional information.
+
+Table of Contents
+=======================
+
+* [Radar Tracks](#Radar-Tracks)
+    * [Enable Radar Tracks](#-Enable-Radar-Tracks)
+* [Enable Mapbox Navigation](#-Enable-Mapbox-Navigation)
+
+---
+
+<details><summary><h3>📡 Radar Tracks</h3></summary>
+
+Radar tracks can now be enabled manually on applicable cars through SSH thanks to [@greghogan](https://github.com/greghogan) and [@pd0wm](https://github.com/pd0wm).
+
+Some Hyundai radars can be reconfigured to output (debug) radar points on bus 1.
+Reconfiguration is done over UDS by reading/writing to 0x0142 using the Read/Write Data By Identifier
+endpoints (0x22 & 0x2E). This script checks your radar firmware version against a list of known
+firmware versions. If you want to try on a new radar, make sure to note the default config value
+in case it is different from the other radars and you need to revert the changes.
+After changing the config the car should not show any faults when openpilot is not running.
+These config changes are persistent across car reboots. You need to run this script again
+to go back to the default values.
+
+**USE AT YOUR OWN RISK!** Stock system safety features, like AEB and FCW, might be affected by these changes.
+
+**How radar points can be used along with vision:**
+* Current OP long policy is identify with vision first, if vision sees a vehicle match it to a radar point. If vision sees nothing you get a false negative and no lead car detection. (Source: [Hubblesphere#7894 from comma.ai community Discord](https://discord.com/channels/469524606043160576/872899198738104330/872913890793635872))
+
+### 🚨 Enable Radar Tracks
+
+***(EXPERIMENTAL, as of January 1st, 2022)***
+
+***(Only applicable to some Hyundai, Kia, and Genesis cars, as of January 1st, 2022)***
+
+*(Base on version 0.8.12 [`devel`](https://github.com/commaai/openpilot/tree/devel))*
+
+**USE AT YOUR OWN RISK!** Stock system safety features, like AEB and FCW, might be affected by these changes.
+
+1. Ensure the car is at the `OFF` ignition position.
+2. Connect your compatible comma device (EON, C2, C3) to the car. comma device power should be ON.
+3. Use a laptop or applicable device to connect to your comma device via SSH. (Tips: Instructions to SSH in [HERE](https://github.com/commaai/openpilot/wiki/SSH))
+4. In the SSH terminal after successfully connected to your comma device, execute the following commands:
+    1. `pkill -f openpilot`
+    2. `python /data/openpilot/selfdrive/debug/hyundai_enable_radar_points.py`
+    3. Follow the instructions in the script:
+        * `Power on the vehicle keeping the engine off (press start button twice) then type OK to continue`.
+            * If successful, the following message should appear: `[DONE]. Restart your vehicle and ensure there are no faults`.
+            * If the script did not run successfully, reach out to the community in [Sunnyhaibin's Openpilot Discord Server](https://discord.gg/wRW3meAgtx) or `#hyundai-kia-genesis channel` on [commaai community Discord Server](https://discord.comma.ai) for assistance.
+    4. Reboot your comma device:
+        1. C3: `sudo reboot`
+        2. C2 or EON: `reboot`
+5. Once your comma device is rebooted, start your car with engine on (with or without comma device connected). Ensure that there are no faults from the car. If there are faults, reach out to the community in [Sunnyhaibin's Openpilot Discord Server](https://discord.gg/wRW3meAgtx) or `#hyundai-kia-genesis channel` on [commaai community Discord Server](https://discord.comma.ai) for assistance.
+6. Go for a quick drive and drive behind a lead car with varied follow distance. Then, come back and allow the drive to upload its `rlogs` in [comma Connect](https://connect.comma.ai).
+7. With all `rlogs` uploaded, open the drive in Cabana from [comma Connect](https://connect.comma.ai). Load DBC -> `hyundai_kia_mando_front_radar.dbc`, then search `RADAR_TRACK_50x` (`x` could be anything), open any of them, and look at `LONG_DIST`.
+8. If the radar tracks data is relevant with the lead car you drove behind, you are done! Your car now have radar tracks enabled.
+</details>
+
+<details><summary><h3>🗺 Enable Mapbox Navigation</h3></summary>
+
+1) Create a free mapbox account. Account will ask for a credit card for verification. You will not be charged for the free tier.
+2) On the Dashboard, you will see a section called Access Tokens. Click `Create a Token`. Name it whatever you like. Set the scopes to allow everything for both Public and Secret. Copy both of these keys. **YOU WON'T BE ABLE TO ACCESS THE SECRET KEY AFTER THIS WINDOW.**
+3) Once rebooted, connect your C3 to a network with internet access and find the C3’s IP address.
+4) In a browser, navigate to that IP with **port 8082** (i.e 192.168.1.69:8082). You should be greeted with the Comma logo and a public key input field.
+5) Paste your Public token (pk.xx), click enter, paste your Secret key (sk.xx), click enter. You can now search for places. This page will be available at your devices’s IP address/port 8082 to search for destinations.
+6) To set Home and Work addresses, search for a place, select Home/Work from the dropdown and click Navigate. For non-Home/Work destinations, select Recent Places.<br>*At this time, it is not possible to search directly on the C3.*
+
+**TIPS:**
+- If your C3 is showing a black screen that says “Map Loading”, performing a reboot via the UI should fix it.
+- If your phone can create a Hotspot, you are able to connect the C3 to your phone hotspot and use your phone browser to search for places.
+- In the Navigation panel on the C3, you can select Home, Work, and from a list of Recent Places you have navigated to without needing a browser (assuming the C3 is connected to the internet.)
+
+**IMPORTANT NOTE:** Your C3 will require an active internet connection to download map data, generate driving directions, and ETA. Once map data and directions are downloaded, it *is* possible to use it offline, however nothing will update (such as new driving direction after a missed turn, updated ETA, map data further into your drive etc.)
+
+***NAVIGATION NOTE:** At this time, mapbox does not support alphanumeric addresses (i.e W123N1234 Main St). There is currently no known workaround for this.*
+</details>

Jenkinsfile

@@ -26,6 +26,7 @@ export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
 export GIT_COMMIT=${env.GIT_COMMIT}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
+export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}'
 # only use 1 thread for tici tests since most require HIL
 export PYTEST_ADDOPTS="-n 0"
 
@@ -82,7 +83,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
 
     lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
       docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
-        timeout(time: 20, unit: 'MINUTES') {
+        timeout(time: 35, unit: 'MINUTES') {
           retry (3) {
             device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }
@@ -134,8 +135,10 @@ def pcStage(String stageName, Closure body) {
 def setupCredentials() {
   withCredentials([
     string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
+    string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN')
   ]) {
     env.AZURE_TOKEN = "${AZURE_TOKEN}"
+    env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}"
   }
 }
 

LICENSE.md

@@ -0,0 +1,21 @@
+# Custom MIT License
+
+Copyright (c) 2024, Haibin Wen, SUNNYPILOT LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to view and modify the Software, subject to the following conditions:
+
+1. **Permission Required**: Permission Required for Commercial, For-Profit, or Closed Source Use: Use of the Software, in whole or in part, for any commercial purposes, for-profit projects, or in closed source projects requires explicit written permission from the original author(s).
+
+2. **Redistribution**: Any redistribution of the Software, modified or unmodified, must retain this license notice and the following acknowledgment:
+   "This software is licensed under a custom license requiring permission for use."
+
+3. **Visibility**: Any project that uses the Software must visibly mention the following acknowledgment:
+   "This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use."
+
+4. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Contact sunnypilot Support <support@sunnypilot.ai> for permission requests.
+
+---
+
+Haibin Wen, SUNNYPILOT LLC

README.md

@@ -170,16 +170,16 @@ After running the command to install the desired branch, your comma device shoul
     * Speed Limit Control (SLC) - Set speed limit based on map data or car interface (if applicable)
     * HKG only: Highway Driving Assist (HDA) status integration - Use cars native speed sign detection to set desired speed (on applicable HKG cars only)
 - [**Gap Adjust Cruise (GAC)**](#gap-adjust-cruise) - Allow `GAP`/`INTERVAL`/`DISTANCE` button on the steering wheel or on-screen button to adjust the follow distance from the lead car. See table below for options
-    - [**Quiet Drive 🤫**](#-quiet-drive) - Toggle to mute all notification sounds (excluding driver safety warnings)
-    - [**Auto Lane Change Timer**](#Auto-Lane-Change-Timer) - Set a timer to delay the auto lane change operation when the blinker is used. No nudge on the steering wheel is required to auto lane change if a timer is set
-    - [**Force Car Recognition (FCR)**](#Force-Car-Recognition-) - Use a selector to force your car to be recognized by sunnypilot
-    - [**Fix sunnypilot No Offroad**](#Fix-sunnypilot-No-Offroad) - Enforce sunnypilot to go offroad and turns off after shutting down the car. This feature fixes non-official devices running sunnypilot without comma power
-    - [**Enable ACC+MADS with RES+/SET-**](#Enable-ACC+MADS-with-RES+/SET-) - Engage both ACC and MADS with a single press of RES+ or SET- button
-    - [**Offline OSM Maps**](#Offline-OSM-Maps) - OSM database can now be downloaded locally for offline use[^2]. This enables offline SLC, V-TSC and M-TSC. Currently available for US South, US West, US Northeast, Florida, Taiwan, South Africa and New Zealand
-    - [**Various Live Tuning**](#Various-Live-Tuning) - Ability to tailor your driving experience on the fly:
-        * Enforce Torque Lateral Control - Use the newest [torque controller](https://blog.comma.ai/0815release/#torque-controller) for all vehicles.
-        * Torque Lateral Control Live Tune - Ability to adjust the torque controller’s `FRICTION` and `LAT_ACCEL_FACTOR` values to suit your vehicle.
-        * Torque Lateral Controller Self-Tune - Enable automatic turning for the Torque controller.
+- [**Quiet Drive 🤫**](#-quiet-drive) - Toggle to mute all notification sounds (excluding driver safety warnings)
+- [**Auto Lane Change Timer**](#Auto-Lane-Change-Timer) - Set a timer to delay the auto lane change operation when the blinker is used. No nudge on the steering wheel is required to auto lane change if a timer is set
+- [**Force Car Recognition (FCR)**](#Force-Car-Recognition-) - Use a selector to force your car to be recognized by sunnypilot
+- [**Fix sunnypilot No Offroad**](#Fix-sunnypilot-No-Offroad) - Enforce sunnypilot to go offroad and turns off after shutting down the car. This feature fixes non-official devices running sunnypilot without comma power
+- [**Enable ACC+MADS with RES+/SET-**](#Enable-ACC+MADS-with-RES+/SET-) - Engage both ACC and MADS with a single press of RES+ or SET- button
+- [**Offline OSM Maps**](#Offline-OSM-Maps) - OSM database can now be downloaded locally for offline use[^2]. This enables offline SLC, V-TSC and M-TSC. Currently available for US South, US West, US Northeast, Florida, Taiwan, South Africa and New Zealand
+- [**Various Live Tuning**](#Various-Live-Tuning) - Ability to tailor your driving experience on the fly:
+    * Enforce Torque Lateral Control - Use the newest [torque controller](https://blog.comma.ai/0815release/#torque-controller) for all vehicles.
+    * Torque Lateral Control Live Tune - Ability to adjust the torque controller’s `FRICTION` and `LAT_ACCEL_FACTOR` values to suit your vehicle.
+    * Torque Lateral Controller Self-Tune - Enable automatic turning for the Torque controller.
 
 ### Visual Enhancements
 * **M.A.D.S Status Icon** - Dedicated icon to display M.A.D.S. engagement status
@@ -223,7 +223,7 @@ The goal of Modified Assistive Driving Safety (MADS) is to enhance the user driv
 * `SET-` button enables ACC/SCC
 * `CANCEL` button only disables ACC/SCC
 * `CRUISE (MAIN)` must be `ON` to use ACC/SCC
-* `CRUISE (MAIN)` button disables sunnypilot completely when `OFF` **(strictly enforced in panda safety code)**
+* `CRUISE (MAIN)` button disables ACC/SCC completely when `OFF` **(strictly enforced in panda safety code)**
 
 ### Disengage Lateral ALC on Brake Press Mode toggle
 Dedicated toggle to handle Lateral state on brake pedal press and release:

SConstruct

@@ -14,6 +14,8 @@ SCons.Warnings.warningAsException(True)
 
 TICI = os.path.isfile('/TICI')
 AGNOS = TICI
+UBUNTU_FOCAL = int(subprocess.check_output('[ -f /etc/os-release ] && . /etc/os-release && [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ] && echo 1 || echo 0', shell=True, encoding='utf-8').rstrip())
+Export('UBUNTU_FOCAL')
 
 Decider('MD5-timestamp')
 
@@ -70,6 +72,12 @@ AddOption('--minimal',
           default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
+AddOption('--stock-ui',
+          action='store_true',
+          dest='stock_ui',
+          default=False,
+          help='Build stock UI instead of sunnypilot UI')
+
 ## Architecture name breakdown (arch)
 ## - larch64: linux tici aarch64
 ## - aarch64: linux pc aarch64
@@ -102,7 +110,6 @@ if arch == "larch64":
 
   libpath = [
     "/usr/local/lib",
-    "/usr/lib",
     "/system/vendor/lib64",
     f"#third_party/acados/{arch}/lib",
   ]
@@ -170,9 +177,12 @@ if arch != "Darwin":
   ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
 
 # Enable swaglog include in submodules
-cflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 
+if not GetOption('stock_ui'):
+  cflags += ["-DSUNNYPILOT"]
+  cxxflags += ["-DSUNNYPILOT"]
+
 ccflags_option = GetOption('ccflags')
 if ccflags_option:
   ccflags += ccflags_option.split(' ')
@@ -206,11 +216,12 @@ env = Environment(
     "#third_party/json11",
     "#third_party/linux/include",
     "#third_party/snpe/include",
-    "#third_party/qrcode",
     "#third_party",
     "#cereal",
     "#msgq",
     "#opendbc/can",
+    "#third_party/maplibre-native-qt/include",
+    f"#third_party/maplibre-native-qt/{arch}/include"
   ],
 
   CC='clang',
@@ -232,7 +243,7 @@ env = Environment(
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
-  toolpath=["#rednose_repo/site_scons/site_tools"],
+  toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
 
 if arch == "Darwin":
@@ -244,7 +255,8 @@ if GetOption('compile_db'):
   env.CompilationDatabase('compile_commands.json')
 
 # Setup cache dir
-cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
 CacheDir(cache_dir)
 Clean(["."], cache_dir)
 
@@ -271,11 +283,12 @@ if arch == "Darwin":
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
-Export('envCython')
+np_version = SCons.Script.Value(np.__version__)
+Export('envCython', 'np_version')
 
 # Qt build environment
 qt_env = env.Clone()
-qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "DBus", "Xml"]
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus", "Xml"]
 
 qt_libs = []
 if arch == "Darwin":
@@ -315,7 +328,7 @@ try:
 except SCons.Errors.UserError:
   qt_env.Tool('qt')
 
-qt_env['CPPPATH'] += qt_dirs# + ["#selfdrive/ui/qt/"]
+qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
 qt_flags = [
   "-D_REENTRANT",
   "-DQT_NO_DEBUG",
@@ -328,7 +341,8 @@ qt_flags = [
   "-DQT_MESSAGELOGCONTEXT",
 ]
 qt_env['CXXFLAGS'] += qt_flags
-qt_env['LIBPATH'] += ['#selfdrive/ui', ]
+qt_env['LIBPATH'] += ['#selfdrive/ui', f"#third_party/maplibre-native-qt/{arch}/lib"]
+qt_env['RPATH'] += [Dir(f"#third_party/maplibre-native-qt/{arch}/lib").srcnode().abspath]
 qt_env['LIBS'] = qt_libs
 
 if GetOption("clazy"):
@@ -363,7 +377,6 @@ Export('messaging')
 
 # Build other submodules
 SConscript([
-  'body/board/SConscript',
   'opendbc/can/SConscript',
   'panda/SConscript',
 ])

cereal/car.capnp

@@ -137,6 +137,8 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     speedLimitPreActive @139;
     speedLimitConfirmed @140;
     torqueNNLoad @141;
+    hyundaiRadarTracksAvailable @142;
+    spAutoBrakeHold @143;
 
     radarCanErrorDEPRECATED @15;
     communityFeatureDisallowedDEPRECATED @62;
@@ -250,7 +252,7 @@ struct CarState {
   struct CustomStockLong {
     cruiseButton @0 :Int16;
     finalSpeedKph @1 :Float32;
-    vCruiseKphPrev @2 :Float32;
+    vCruiseKphPrevDEPRECATED @2 :Float32;
     targetSpeed @3 :Float32;
     vSetDis @4 :Float32;
     speedDiff @5 :Float32;

cereal/custom.capnp

@@ -16,11 +16,36 @@ enum LongitudinalPersonalitySP {
   moderate @1;
   standard @2;
   relaxed @3;
+  overtake @4;
+}
+
+enum AccelerationPersonality {
+  sport @0;
+  normal @1;
+  eco @2;
+  stock @3;
+}
+
+enum ModelGeneration {
+  default @0;
+  one @1;
+  two @2;
+  three @3;
+  four @4;
+  five @5;
+}
+
+enum MpcSource {
+  acc @0;
+  blended @1;
 }
 
 struct ControlsStateSP @0x81c2f05a394cf4af {
   lateralState @0 :Text;
   personality @8 :LongitudinalPersonalitySP;
+  dynamicPersonality @9 :Bool;
+  accelPersonality @10 :AccelerationPersonality;
+  overtakingAccelerationAssist @11 :Bool;
 
   lateralControlState :union {
     indiState @1 :LateralINDIState;
@@ -72,8 +97,10 @@ struct LongitudinalPlanSP @0xaedffd8f31e7b55d {
   desiredTF @13 :Float32;
   notSpeedLimit @14 :Int16;
   e2eX @15 :List(Float32);
-  e2eBlended @18 :Text;
+  e2eBlendedDEPRECATED @18 :Text;
   e2eStatus @22 :Bool;
+  mpcSource @23 :MpcSource;
+  dynamicExperimentalControl @24 :Bool;
 
   distToTurn @7 :Float32;
   turnSpeed @8 :Float32;
@@ -180,6 +207,9 @@ struct E2eLongStateSP @0xa5cd762cd951a455 {
 struct ModelDataV2SP @0xf98d843bfd7004a3 {
   laneChangePrev @0 :Bool;
   laneChangeEdgeBlock @1 :Bool;
+  customModel @2 :Bool;
+  modelGeneration @3 :ModelGeneration;
+  modelCapabilities @4 :UInt32;
 }
 
 struct CustomReserved7 @0xb86e6369214c01c8 {

cereal/log.capnp

@@ -137,8 +137,6 @@ struct FrameData {
   requestId @28 :UInt32;
   encodeId @1 :UInt32;
 
-  frameType @7 :FrameType;
-
   # Timestamps
   timestampEof @2 :UInt64;
   timestampSof @8 :UInt64;
@@ -158,7 +156,7 @@ struct FrameData {
 
   temperaturesC @24 :List(Float32);
 
-  enum FrameType {
+  enum FrameTypeDEPRECATED {
     unknown @0;
     neo @1;
     chffrAndroid @2;
@@ -175,6 +173,7 @@ struct FrameData {
 
   frameLengthDEPRECATED @3 :Int32;
   globalGainDEPRECATED @5 :Int32;
+  frameTypeDEPRECATED @7 :FrameTypeDEPRECATED;
   androidCaptureResultDEPRECATED @9 :AndroidCaptureResult;
   lensPosDEPRECATED @11 :Int32;
   lensSagDEPRECATED @12 :Float32;
@@ -337,9 +336,9 @@ enum LaneChangeDirection {
 
 struct CanData {
   address @0 :UInt32;
-  busTime @1 :UInt16;
   dat     @2 :Data;
   src     @3 :UInt8;
+  busTimeDEPRECATED @1 :UInt16;
 }
 
 struct DeviceState @0xa4d8b5af2aa492eb {
@@ -1252,6 +1251,38 @@ struct LiveLocationKalman {
   }
 }
 
+
+struct LivePose {
+  # More info on reference frames:
+  # https://github.com/commaai/openpilot/tree/master/common/transformations
+  orientationNED @0 :XYZMeasurement;
+  velocityDevice @1 :XYZMeasurement;
+  accelerationDevice @2 :XYZMeasurement;
+  angularVelocityDevice @3 :XYZMeasurement;
+
+  inputsOK @4 :Bool = false;
+  posenetOK @5 :Bool = false;
+  sensorsOK @6 :Bool = false;
+
+  filterState @7 :FilterState;
+
+  struct XYZMeasurement {
+    x @0 :Float32;
+    y @1 :Float32;
+    z @2 :Float32;
+    xStd @3 :Float32;
+    yStd @4 :Float32;
+    zStd @5 :Float32;
+    valid @6 :Bool;
+  }
+
+  struct FilterState {
+    value @0 : List(Float64);
+    std @1 : List(Float64);
+    valid @2 : Bool;
+  }
+}
+
 struct ProcLog {
   cpuTimes @0 :List(CPUTimes);
   mem @1 :Mem;
@@ -2293,6 +2324,7 @@ struct Event {
     carParams @69: Car.CarParams;
     driverMonitoringState @71: DriverMonitoringState;
     liveLocationKalman @72 :LiveLocationKalman;
+    livePose @129 :LivePose;
     modelV2 @75 :ModelDataV2;
     drivingModelData @128 :DrivingModelData;
     driverStateV2 @92 :DriverStateV2;

cereal/messaging/__init__.py

@@ -2,7 +2,8 @@
 from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
                                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
 from msgq.ipc_pyx import MultiplePublishersError, IpcError
-from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw, context
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
+import msgq
 
 import os
 import capnp
@@ -17,8 +18,12 @@ from cereal.services import SERVICE_LIST
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
-  with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
+def reset_context():
+  msgq.context = Context()
+
+
+def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
+  with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
     return msg
 
 

cereal/messaging/tests/test_messaging.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 import os
 import capnp
 import multiprocessing
@@ -6,8 +5,8 @@ import numbers
 import random
 import threading
 import time
-import unittest
 from parameterized import parameterized
+import pytest
 
 from cereal import log, car
 import cereal.messaging as messaging
@@ -28,12 +27,6 @@ def zmq_sleep(t=1):
   if "ZMQ" in os.environ:
     time.sleep(t)
 
-def zmq_expected_failure(func):
-  if "ZMQ" in os.environ:
-    return unittest.expectedFailure(func)
-  else:
-    return func
-
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
@@ -58,12 +51,12 @@ def delayed_send(delay, sock, dat):
   threading.Timer(delay, send_func).start()
 
 
-class TestMessaging(unittest.TestCase):
+class TestMessaging:
   def setUp(self):
     # TODO: ZMQ tests are too slow; all sleeps will need to be
     # replaced with logic to block on the necessary condition
     if "ZMQ" in os.environ:
-      raise unittest.SkipTest
+      pytest.skip()
 
     # ZMQ pub socket takes too long to die
     # sleep to prevent multiple publishers error between tests
@@ -75,9 +68,9 @@ class TestMessaging(unittest.TestCase):
       msg = messaging.new_message(evt)
     except capnp.lib.capnp.KjException:
       msg = messaging.new_message(evt, random.randrange(200))
-    self.assertLess(time.monotonic() - msg.logMonoTime, 0.1)
-    self.assertFalse(msg.valid)
-    self.assertEqual(evt, msg.which())
+    assert (time.monotonic() - msg.logMonoTime) < 0.1
+    assert not msg.valid
+    assert evt == msg.which()
 
   @parameterized.expand(events)
   def test_pub_sock(self, evt):
@@ -99,8 +92,8 @@ class TestMessaging(unittest.TestCase):
 
     # no wait and no msgs in queue
     msgs = func(sub_sock)
-    self.assertIsInstance(msgs, list)
-    self.assertEqual(len(msgs), 0)
+    assert isinstance(msgs, list)
+    assert len(msgs) == 0
 
     # no wait but msgs are queued up
     num_msgs = random.randrange(3, 10)
@@ -108,9 +101,9 @@ class TestMessaging(unittest.TestCase):
       pub_sock.send(messaging.new_message(sock).to_bytes())
     time.sleep(0.1)
     msgs = func(sub_sock)
-    self.assertIsInstance(msgs, list)
-    self.assertTrue(all(isinstance(msg, expected_type) for msg in msgs))
-    self.assertEqual(len(msgs), num_msgs)
+    assert isinstance(msgs, list)
+    assert all(isinstance(msg, expected_type) for msg in msgs)
+    assert len(msgs) == num_msgs
 
   def test_recv_sock(self):
     sock = "carState"
@@ -120,14 +113,14 @@ class TestMessaging(unittest.TestCase):
 
     # no wait and no msg in queue, socket should timeout
     recvd = messaging.recv_sock(sub_sock)
-    self.assertTrue(recvd is None)
+    assert recvd is None
 
     # no wait and one msg in queue
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
     time.sleep(0.01)
     recvd = messaging.recv_sock(sub_sock)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert isinstance(recvd, capnp._DynamicStructReader)
     # https://github.com/python/mypy/issues/13038
     assert_carstate(msg.carState, recvd.carState)
 
@@ -139,16 +132,16 @@ class TestMessaging(unittest.TestCase):
 
     # no msg in queue, socket should timeout
     recvd = messaging.recv_one(sub_sock)
-    self.assertTrue(recvd is None)
+    assert recvd is None
 
     # one msg in queue
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
     recvd = messaging.recv_one(sub_sock)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert isinstance(recvd, capnp._DynamicStructReader)
     assert_carstate(msg.carState, recvd.carState)
 
-  @zmq_expected_failure
+  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
   def test_recv_one_or_none(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
@@ -157,13 +150,13 @@ class TestMessaging(unittest.TestCase):
 
     # no msg in queue, socket shouldn't block
     recvd = messaging.recv_one_or_none(sub_sock)
-    self.assertTrue(recvd is None)
+    assert recvd is None
 
     # one msg in queue
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
     recvd = messaging.recv_one_or_none(sub_sock)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert isinstance(recvd, capnp._DynamicStructReader)
     assert_carstate(msg.carState, recvd.carState)
 
   def test_recv_one_retry(self):
@@ -179,7 +172,7 @@ class TestMessaging(unittest.TestCase):
       p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
       p.start()
       time.sleep(sock_timeout*15)
-      self.assertTrue(p.is_alive())
+      assert p.is_alive()
       p.terminate()
 
     # wait 15 socket timeouts before sending
@@ -187,9 +180,6 @@ class TestMessaging(unittest.TestCase):
     delayed_send(sock_timeout*15, pub_sock, msg.to_bytes())
     start_time = time.monotonic()
     recvd = messaging.recv_one_retry(sub_sock)
-    self.assertGreaterEqual(time.monotonic() - start_time, sock_timeout*15)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert (time.monotonic() - start_time) >= sock_timeout*15
+    assert isinstance(recvd, capnp._DynamicStructReader)
     assert_carstate(msg.carState, recvd.carState)
-
-if __name__ == "__main__":
-  unittest.main()

cereal/messaging/tests/test_pub_sub_master.py

@@ -1,8 +1,6 @@
-#!/usr/bin/env python3
 import random
 import time
 from typing import Sized, cast
-import unittest
 
 import cereal.messaging as messaging
 from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
@@ -10,9 +8,9 @@ from cereal.messaging.tests.test_messaging import events, random_sock, random_so
                                                   zmq_sleep
 
 
-class TestSubMaster(unittest.TestCase):
+class TestSubMaster:
 
-  def setUp(self):
+  def setup_method(self):
     # ZMQ pub socket takes too long to die
     # sleep to prevent multiple publishers error between tests
     zmq_sleep(3)
@@ -21,21 +19,21 @@ class TestSubMaster(unittest.TestCase):
     sm = messaging.SubMaster(events)
     for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
               sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      self.assertEqual(len(cast(Sized, p)), len(events))
+      assert len(cast(Sized, p)) == len(events)
 
   def test_init_state(self):
     socks = random_socks()
     sm = messaging.SubMaster(socks)
-    self.assertEqual(sm.frame, -1)
-    self.assertFalse(any(sm.updated.values()))
-    self.assertFalse(any(sm.alive.values()))
-    self.assertTrue(all(t == 0. for t in sm.recv_time.values()))
-    self.assertTrue(all(f == 0 for f in sm.recv_frame.values()))
-    self.assertTrue(all(t == 0 for t in sm.logMonoTime.values()))
+    assert sm.frame == -1
+    assert not any(sm.updated.values())
+    assert not any(sm.alive.values())
+    assert all(t == 0. for t in sm.recv_time.values())
+    assert all(f == 0 for f in sm.recv_frame.values())
+    assert all(t == 0 for t in sm.logMonoTime.values())
 
     for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
               sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      self.assertEqual(len(cast(Sized, p)), len(socks))
+      assert len(cast(Sized, p)) == len(socks)
 
   def test_getitem(self):
     sock = "carState"
@@ -59,8 +57,8 @@ class TestSubMaster(unittest.TestCase):
       msg = messaging.new_message(sock)
       pub_sock.send(msg.to_bytes())
       sm.update(1000)
-      self.assertEqual(sm.frame, i)
-      self.assertTrue(all(sm.updated.values()))
+      assert sm.frame == i
+      assert all(sm.updated.values())
 
   def test_update_timeout(self):
     sock = random_sock()
@@ -70,9 +68,9 @@ class TestSubMaster(unittest.TestCase):
       start_time = time.monotonic()
       sm.update(timeout)
       t = time.monotonic() - start_time
-      self.assertGreaterEqual(t, timeout/1000.)
-      self.assertLess(t, 5)
-      self.assertFalse(any(sm.updated.values()))
+      assert t >= timeout/1000.
+      assert t < 5
+      assert not any(sm.updated.values())
 
   def test_avg_frequency_checks(self):
     for poll in (True, False):
@@ -118,12 +116,12 @@ class TestSubMaster(unittest.TestCase):
       pub_sock.send(msg.to_bytes())
       time.sleep(0.01)
     sm.update(1000)
-    self.assertEqual(sm[sock].vEgo, n)
+    assert sm[sock].vEgo == n
 
 
-class TestPubMaster(unittest.TestCase):
+class TestPubMaster:
 
-  def setUp(self):
+  def setup_method(self):
     # ZMQ pub socket takes too long to die
     # sleep to prevent multiple publishers error between tests
     zmq_sleep(3)
@@ -156,8 +154,4 @@ class TestPubMaster(unittest.TestCase):
         if capnp:
           msg.clear_write_flag()
           msg = msg.to_bytes()
-        self.assertEqual(msg, recvd, i)
-
-
-if __name__ == "__main__":
-  unittest.main()
+        assert msg == recvd, i

cereal/messaging/tests/test_services.py

@@ -1,26 +1,21 @@
-#!/usr/bin/env python3
 import os
 import tempfile
 from typing import Dict
-import unittest
 from parameterized import parameterized
 
 import cereal.services as services
 from cereal.services import SERVICE_LIST
 
 
-class TestServices(unittest.TestCase):
+class TestServices:
 
   @parameterized.expand(SERVICE_LIST.keys())
   def test_services(self, s):
     service = SERVICE_LIST[s]
-    self.assertTrue(service.frequency <= 104)
-    self.assertTrue(service.decimation != 0)
+    assert service.frequency <= 104
+    assert service.decimation != 0
 
   def test_generated_header(self):
     with tempfile.NamedTemporaryFile(suffix=".h") as f:
       ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}")
-      self.assertEqual(ret, 0, "generated services header is not valid C")
-
-if __name__ == "__main__":
-  unittest.main()
+      assert ret == 0, "generated services header is not valid C"

cereal/services.py

@@ -16,13 +16,13 @@ _services: dict[str, tuple] = {
   "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
   "accelerometer2": (True, 100., 100),
-  "magnetometer": (True, 25., 25),
+  "magnetometer": (True, 25.),
   "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
   "temperatureSensor2": (True, 2., 200),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "can": (True, 100., 1223),  # decimation gives ~5 msgs in a full segment
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
   "controlsState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
@@ -38,7 +38,7 @@ _services: dict[str, tuple] = {
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
-  "longitudinalPlan": (True, 20., 5),
+  "longitudinalPlan": (True, 20., 10),
   "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
@@ -47,9 +47,10 @@ _services: dict[str, tuple] = {
   "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
-  "liveLocationKalman": (True, 20., 5),
+  "livePose": (True, 20., 4),
+  "liveLocationKalman": (True, 20.),
   "liveParameters": (True, 20., 5),
-  "cameraOdometry": (True, 20., 5),
+  "cameraOdometry": (True, 20., 10),
   "lateralPlanDEPRECATED": (True, 20., 5),
   "thumbnail": (True, 0.2, 1),
   "onroadEvents": (True, 1., 1),

common/api/__init__.py

@@ -1,46 +1,27 @@
-import jwt
-import os
-import requests
-from datetime import datetime, timedelta, UTC
-from openpilot.system.hardware.hw import Paths
-from openpilot.system.version import get_version
+from .comma_connect import CommaConnectApi
+from .sunnylink import SunnylinkApi
 
-API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 
 class Api:
-  def __init__(self, dongle_id):
-    self.dongle_id = dongle_id
-    with open(Paths.persist_root()+'/comma/id_rsa') as f:
-      self.private_key = f.read()
+  def __init__(self, dongle_id, use_sunnylink=False):
+    if use_sunnylink:
+      self.service = SunnylinkApi(dongle_id)
+    else:
+      self.service = CommaConnectApi(dongle_id)
+
+  def request(self, method, endpoint, **params):
+    return self.service.request(method, endpoint, **params)
 
   def get(self, *args, **kwargs):
-    return self.request('GET', *args, **kwargs)
+    return self.service.get(*args, **kwargs)
 
   def post(self, *args, **kwargs):
-    return self.request('POST', *args, **kwargs)
-
-  def request(self, method, endpoint, timeout=None, access_token=None, **params):
-    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+    return self.service.post(*args, **kwargs)
 
   def get_token(self, expiry_hours=1):
-    now = datetime.now(UTC).replace(tzinfo=None)
-    payload = {
-      'identity': self.dongle_id,
-      'nbf': now,
-      'iat': now,
-      'exp': now + timedelta(hours=expiry_hours)
-    }
-    token = jwt.encode(payload, self.private_key, algorithm='RS256')
-    if isinstance(token, bytes):
-      token = token.decode('utf8')
-    return token
+    return self.service.get_token(expiry_hours)
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
-  headers = {}
-  if access_token is not None:
-    headers['Authorization'] = "JWT " + access_token
-
-  headers['User-Agent'] = "openpilot-" + get_version()
-
-  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)
+def api_get(endpoint, method='GET', timeout=None, access_token=None, use_sunnylink=False, **params):
+    return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, **params) if not use_sunnylink else \
+           SunnylinkApi(None).api_get(endpoint, method, timeout, access_token, **params)

common/api/base.py

@@ -0,0 +1,56 @@
+import jwt
+import requests
+import unicodedata
+from datetime import datetime, timedelta, UTC
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.version import get_version
+
+
+class BaseApi:
+  def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
+    self.dongle_id = dongle_id
+    self.api_host = api_host
+    self.user_agent = user_agent
+    with open(Paths.persist_root()+'/comma/id_rsa') as f:
+      self.private_key = f.read()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def _get_token(self, expiry_hours=1, **extra_payload):
+    now = datetime.now(UTC).replace(tzinfo=None)
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours),
+      **extra_payload
+    }
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+
+  def get_token(self, expiry_hours=1):
+    return self._get_token(expiry_hours)
+
+  def remove_non_ascii_chars(self, text):
+    normalized_text = unicodedata.normalize('NFD', text)
+    ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
+    return ascii_encoded_text.decode()
+
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, **params):
+    headers = {}
+    if access_token is not None:
+      headers['Authorization'] = "JWT " + access_token
+
+    version = self.remove_non_ascii_chars(get_version())
+    headers['User-Agent'] = self.user_agent + version
+
+    return requests.request(method, self.api_host + "/" + endpoint, timeout=timeout, headers=headers, params=params)

common/api/comma_connect.py

@@ -0,0 +1,11 @@
+import os
+
+from openpilot.common.api.base import BaseApi
+
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
+
+
+class CommaConnectApi(BaseApi):
+  def __init__(self, dongle_id):
+    super().__init__(dongle_id, API_HOST)
+    self.user_agent = "openpilot-"

common/api/sunnylink.py

@@ -0,0 +1,155 @@
+import os
+import time
+import jwt
+import json
+from pathlib import Path
+from datetime import datetime, timedelta
+from openpilot.common.params import Params
+from openpilot.system.hardware import HARDWARE
+from openpilot.system.hardware.hw import Paths
+
+from openpilot.common.api.base import BaseApi
+
+API_HOST = os.getenv('SUNNYLINK_API_HOST', 'https://stg.api.sunnypilot.ai')
+UNREGISTERED_SUNNYLINK_DONGLE_ID = "UnregisteredDevice"
+MAX_RETRIES = 6
+CRASH_LOG_DIR = '/data/community/crashes'
+
+class SunnylinkApi(BaseApi):
+  def __init__(self, dongle_id):
+    super().__init__(dongle_id, API_HOST)
+    self.user_agent = "sunnypilot-"
+    self.spinner = None
+    self.params = Params()
+
+  def api_get(self, endpoint, method='GET', timeout=10, access_token=None, **kwargs):
+    if not self.params.get_bool("SunnylinkEnabled"):
+      return None
+
+    return super().api_get(endpoint, method, timeout, access_token, **kwargs)
+
+  def resume_queued(self, timeout=10, **kwargs):
+    sunnylinkId, commaId = self._resolve_dongle_ids()
+    return self.api_get(f"ws/{sunnylinkId}/resume_queued", "POST", timeout, access_token=self.get_token(), **kwargs)
+
+  def get_token(self, expiry_hours=1):
+    # Add your additional data here
+    additional_data = {}
+    return super()._get_token(expiry_hours, **additional_data)
+
+  def _status_update(self, message):
+    print(message)
+    if self.spinner:
+      self.spinner.update(message)
+      time.sleep(0.5)
+
+  def _resolve_dongle_ids(self):
+    sunnylink_dongle_id = self.params.get("SunnylinkDongleId", encoding='utf-8')
+    comma_dongle_id = self.dongle_id or self.params.get("DongleId", encoding='utf-8')
+    return sunnylink_dongle_id, comma_dongle_id
+
+  def _resolve_imeis(self):
+    imei1, imei2 = None, None
+    imei_try = 0
+    while imei1 is None and imei2 is None and imei_try < MAX_RETRIES:
+      try:
+        imei1, imei2 = self.params.get("IMEI", encoding='utf8') or HARDWARE.get_imei(0), HARDWARE.get_imei(1)
+      except Exception:
+        self._status_update(f"Error getting imei, trying again... [{imei_try+1}/{MAX_RETRIES}]")
+        time.sleep(1)
+      imei_try += 1
+    return imei1, imei2
+
+  def _resolve_serial(self):
+    serial = self.params.get("HardwareSerial", encoding='utf8') or HARDWARE.get_serial()
+    return serial
+
+  def register_device(self, spinner=None, timeout=60, verbose=False):
+    self.spinner = spinner
+
+    sunnylink_dongle_id, comma_dongle_id = self._resolve_dongle_ids()
+
+    if comma_dongle_id is None:
+      self._status_update("Comma dongle ID not found, deferring sunnylink's registration to comma's registration process.")
+      return None
+
+    imei1, imei2 = self._resolve_imeis()
+    serial = self._resolve_serial()
+
+    if sunnylink_dongle_id not in (None, UNREGISTERED_SUNNYLINK_DONGLE_ID):
+      return sunnylink_dongle_id
+
+    privkey_path = Path(Paths.persist_root()+"/comma/id_rsa")
+    pubkey_path = Path(Paths.persist_root()+"/comma/id_rsa.pub")
+
+    start_time = time.monotonic()
+    successful_registration = False
+    if not pubkey_path.is_file():
+      sunnylink_dongle_id = UNREGISTERED_SUNNYLINK_DONGLE_ID
+      self._status_update("Public key not found, setting dongle ID to unregistered.")
+    else:
+      Params().put("LastSunnylinkPingTime", "0")  # Reset the last ping time to 0 if we are trying to register
+      with pubkey_path.open() as f1, privkey_path.open() as f2:
+        public_key = f1.read()
+        private_key = f2.read()
+
+      backoff = 1
+      while True:
+        register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256')
+        try:
+          if verbose or time.monotonic() - start_time < timeout / 2:
+            self._status_update("Registering device to sunnylink...")
+          elif time.monotonic() - start_time >= timeout / 2:
+            self._status_update("Still registering device to sunnylink...")
+
+          resp = self.api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, comma_dongle_id=comma_dongle_id, public_key=public_key, register_token=register_token)
+
+          if resp is None:
+            raise Exception("Unable to register device, request was None")
+
+          if resp.status_code in (409, 412):
+            timeout = time.monotonic() - start_time  # Don't retry if the public key is already in use
+            key_in_use = "Public key is already in use, is your key unique? Contact your vendor for a new key."
+            unsafe_key = "Public key is known to not be unique and it's unsafe. Contact your vendor for a new key."
+            error_message = key_in_use if resp.status_code == 409 else unsafe_key
+            raise Exception(error_message)
+
+          if resp.status_code != 200:
+            raise Exception(f"Failed to register with sunnylink. Status code: {resp.status_code}\nData\n:{resp.text}")
+
+          dongleauth = json.loads(resp.text)
+          sunnylink_dongle_id = dongleauth["device_id"]
+          if sunnylink_dongle_id:
+            self._status_update("Device registered successfully.")
+            successful_registration = True
+            break
+        except Exception as e:
+          if verbose:
+            self._status_update(f"Waiting {backoff}s before retry, Exception occurred during registration: [{str(e)}]")
+
+          if not os.path.exists(CRASH_LOG_DIR):
+            os.makedirs(CRASH_LOG_DIR)
+
+          with open(f'{CRASH_LOG_DIR}/error.txt', 'a') as f:
+            f.write(f"[{datetime.now()}] sunnylink: {str(e)}\n")
+
+          backoff = min(backoff * 2, 60)
+          time.sleep(backoff)
+
+        if time.monotonic() - start_time > timeout:
+          self._status_update(f"Giving up on sunnylink's registration after {timeout}s. Will retry on next boot.")
+          time.sleep(3)
+          break
+
+    self.params.put("SunnylinkDongleId", sunnylink_dongle_id or UNREGISTERED_SUNNYLINK_DONGLE_ID)
+
+    # Set the last ping time to the current time since we were just talking to the API
+    last_ping = int(time.monotonic() * 1e9) if successful_registration else start_time
+    Params().put("LastSunnylinkPingTime", str(last_ping))
+
+    # Disable sunnylink if registration was not successful
+    if not successful_registration:
+      Params().put_bool("SunnylinkEnabled", False)
+
+    self.spinner = None
+    return sunnylink_dongle_id

common/model.h

@@ -0,0 +1 @@
+#define CURRENT_MODEL "Notre Dame (July 1, 2024)"

common/params.cc

@@ -24,8 +24,8 @@ int fsync_dir(const std::string &path) {
   int result = -1;
   int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
   if (fd >= 0) {
-    result = fsync(fd);
-    close(fd);
+    result = HANDLE_EINTR(fsync(fd));
+    HANDLE_EINTR(close(fd));
   }
   return result;
 }
@@ -91,6 +91,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
     {"AlwaysOnDM", PERSISTENT},
     {"ApiCache_Device", PERSISTENT},
+    {"ApiCache_NavDestinations", PERSISTENT},
     {"AssistNowToken", PERSISTENT},
     {"AthenadPid", PERSISTENT},
     {"AthenadUploadQueue", PERSISTENT},
@@ -110,45 +111,46 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"CurrentBootlog", PERSISTENT},
     {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"DisablePowerDown", PERSISTENT},
-    {"DisableUpdates", PERSISTENT},
-    {"DisengageOnAccelerator", PERSISTENT},
+    {"DisablePowerDown", PERSISTENT | BACKUP},
+    {"DisableUpdates", PERSISTENT | BACKUP},
+    {"DisengageOnAccelerator", PERSISTENT | BACKUP},
     {"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
     {"DongleId", PERSISTENT},
     {"DoReboot", CLEAR_ON_MANAGER_START},
     {"DoShutdown", CLEAR_ON_MANAGER_START},
     {"DoUninstall", CLEAR_ON_MANAGER_START},
-    {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
-    {"ExperimentalMode", PERSISTENT},
-    {"ExperimentalModeConfirmed", PERSISTENT},
+    {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY | BACKUP},
+    {"ExperimentalMode", PERSISTENT | BACKUP},
+    {"ExperimentalModeConfirmed", PERSISTENT | BACKUP},
     {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"ForcePowerDown", PERSISTENT},
     {"GitBranch", PERSISTENT},
     {"GitCommit", PERSISTENT},
     {"GitCommitDate", PERSISTENT},
     {"GitDiff", PERSISTENT},
-    {"GithubSshKeys", PERSISTENT},
-    {"GithubUsername", PERSISTENT},
+    {"GithubSshKeys", PERSISTENT | BACKUP},
+    {"GithubUsername", PERSISTENT | BACKUP},
     {"GitRemote", PERSISTENT},
-    {"GsmApn", PERSISTENT},
-    {"GsmMetered", PERSISTENT},
-    {"GsmRoaming", PERSISTENT},
+    {"GsmApn", PERSISTENT | BACKUP},
+    {"GsmMetered", PERSISTENT | BACKUP},
+    {"GsmRoaming", PERSISTENT | BACKUP},
     {"HardwareSerial", PERSISTENT},
     {"HasAcceptedTerms", PERSISTENT},
     {"IMEI", PERSISTENT},
     {"InstallDate", PERSISTENT},
     {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
     {"IsEngaged", PERSISTENT},
-    {"IsLdwEnabled", PERSISTENT},
-    {"IsMetric", PERSISTENT},
+    {"IsLdwEnabled", PERSISTENT | BACKUP},
+    {"IsMetric", PERSISTENT | BACKUP},
     {"IsOffroad", CLEAR_ON_MANAGER_START},
     {"IsOnroad", PERSISTENT},
     {"IsRhdDetected", PERSISTENT},
     {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
+    {"IsReleaseSPBranch", CLEAR_ON_MANAGER_START},
     {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
     {"IsTestedBranch", CLEAR_ON_MANAGER_START},
     {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"LanguageSetting", PERSISTENT},
+    {"LanguageSetting", PERSISTENT | BACKUP},
     {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
     {"LastGPSPosition", PERSISTENT},
     {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
@@ -158,7 +160,12 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"LastUpdateTime", PERSISTENT},
     {"LiveParameters", PERSISTENT},
     {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
-    {"LongitudinalPersonality", PERSISTENT},
+    {"LongitudinalPersonality", PERSISTENT | BACKUP},
+    {"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"NavDestinationWaypoints", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"NavPastDestinations", PERSISTENT},
+    {"NavSettingLeftSide", PERSISTENT | BACKUP},
+    {"NavSettingTime24h", PERSISTENT | BACKUP},
     {"NetworkMetered", PERSISTENT},
     {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
@@ -174,17 +181,18 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
     {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
     {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
-    {"OpenpilotEnabledToggle", PERSISTENT},
+    {"Offroad_OSMUpdateRequired", CLEAR_ON_MANAGER_START},
+    {"OpenpilotEnabledToggle", PERSISTENT | BACKUP},
     {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"PandaSignatures", CLEAR_ON_MANAGER_START},
     {"PrimeType", PERSISTENT},
-    {"RecordFront", PERSISTENT},
+    {"RecordFront", PERSISTENT | BACKUP},
     {"RecordFrontLock", PERSISTENT},  // for the internal fleet
     {"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"RouteCount", PERSISTENT},
     {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"SshEnabled", PERSISTENT},
+    {"SshEnabled", PERSISTENT | BACKUP},
     {"TermsVersion", PERSISTENT},
     {"TrainingVersion", PERSISTENT},
     {"UbloxAvailable", PERSISTENT},
@@ -201,99 +209,163 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"UpdaterLastFetchTime", PERSISTENT},
     {"Version", PERSISTENT},
 
-    {"AccMadsCombo", PERSISTENT},
-    {"AmapKey1", PERSISTENT},
-    {"AmapKey2", PERSISTENT},
-    {"AutoLaneChangeTimer", PERSISTENT},
-    {"AutoLaneChangeBsmDelay", PERSISTENT},
-    {"BelowSpeedPause", PERSISTENT},
-    {"BrakeLights", PERSISTENT},
-    {"BrightnessControl", PERSISTENT},
-    {"ButtonAutoHide", PERSISTENT},
-    {"CameraControl", PERSISTENT},
-    {"CameraControlToggle", PERSISTENT},
-    {"CameraOffset", PERSISTENT},
-    {"CarModel", PERSISTENT},
-    {"CarModelText", PERSISTENT},
-    {"ChevronInfo", PERSISTENT},
-    {"CustomBootScreen", PERSISTENT},
-    {"CustomMapboxTokenPk", PERSISTENT},
-    {"CustomMapboxTokenSk", PERSISTENT},
-    {"CustomOffsets", PERSISTENT},
-    {"CustomStockLong", PERSISTENT},
-    {"CustomTorqueLateral", PERSISTENT},
-    {"DevUI", PERSISTENT},
-    {"DevUIInfo", PERSISTENT},
-    {"DisableOnroadUploads", PERSISTENT},
-    {"DisengageLateralOnBrake", PERSISTENT},
-    {"DynamicExperimentalControl", PERSISTENT},
-    {"DynamicExperimentalControlToggle", PERSISTENT},
-    {"DynamicLaneProfile", PERSISTENT},
-    {"DynamicLaneProfileToggle", PERSISTENT},
-    {"EnableAmap", PERSISTENT},
-    {"EnableDebugSnapshot", PERSISTENT},
-    {"EnableGmap", PERSISTENT},
-    {"EnableMads", PERSISTENT},
-    {"EndToEndLongAlertLead", PERSISTENT},
-    {"EndToEndLongAlertLight", PERSISTENT},
-    {"EndToEndLongAlertUI", PERSISTENT},
-    {"EndToEndLongToggle", PERSISTENT},
-    {"EnforceTorqueLateral", PERSISTENT},
-    {"EnhancedScc", PERSISTENT},
+    {"AccelPersonality", PERSISTENT | BACKUP},
+    {"AccMadsCombo", PERSISTENT | BACKUP},
+    {"AmapKey1", PERSISTENT | BACKUP},
+    {"AmapKey2", PERSISTENT | BACKUP},
+    {"ApiCache_DriveStats", PERSISTENT | BACKUP},
+    {"AutoLaneChangeTimer", PERSISTENT | BACKUP},
+    {"AutoLaneChangeBsmDelay", PERSISTENT | BACKUP},
+    {"BelowSpeedPause", PERSISTENT | BACKUP},
+    {"BrakeLights", PERSISTENT | BACKUP},
+    {"BrightnessControl", PERSISTENT | BACKUP},
+    {"ButtonAutoHide", PERSISTENT | BACKUP},
+    {"CameraControl", PERSISTENT | BACKUP},
+    {"CameraControlToggle", PERSISTENT | BACKUP},
+    {"CameraOffset", PERSISTENT | BACKUP},
+    {"CarModel", PERSISTENT | BACKUP},
+    {"CarModelText", PERSISTENT | BACKUP},
+    {"ChevronInfo", PERSISTENT | BACKUP},
+    {"CustomBootScreen", PERSISTENT | BACKUP},
+    {"CustomDrivingModel", PERSISTENT},
+    {"CustomMapboxTokenPk", PERSISTENT | BACKUP},
+    {"CustomMapboxTokenSk", PERSISTENT | BACKUP},
+    {"CustomOffsets", PERSISTENT | BACKUP},
+    {"CustomStockLong", PERSISTENT | BACKUP},
+    {"CustomStockLongPlanner", PERSISTENT | BACKUP},
+    {"CustomTorqueLateral", PERSISTENT | BACKUP},
+    {"DevUIInfo", PERSISTENT | BACKUP},
+    {"DisableOnroadUploads", PERSISTENT | BACKUP},
+    {"DisengageLateralOnBrake", PERSISTENT | BACKUP},
+    {"DrivingModelGeneration", PERSISTENT},
+    {"DrivingModelMetadataText", PERSISTENT},
+    {"DrivingModelName", PERSISTENT},
+    {"DrivingModelText", PERSISTENT},
+    {"DrivingModelUrl", PERSISTENT},
+    {"DynamicExperimentalControl", PERSISTENT | BACKUP},
+    {"DynamicLaneProfile", PERSISTENT | BACKUP},
+    {"DynamicPersonality", PERSISTENT | BACKUP},
+    {"EnableAmap", PERSISTENT | BACKUP},
+    {"EnableGmap", PERSISTENT | BACKUP},
+    {"EnableMads", PERSISTENT | BACKUP},
+    {"EnableSlc", PERSISTENT | BACKUP},
+    {"EndToEndLongAlertLead", PERSISTENT | BACKUP},
+    {"EndToEndLongAlertLight", PERSISTENT | BACKUP},
+    {"EndToEndLongAlertUI", PERSISTENT | BACKUP},
+    {"EndToEndLongToggle", PERSISTENT | BACKUP},
+    {"EnforceTorqueLateral", PERSISTENT | BACKUP},
+    {"EnhancedScc", PERSISTENT | BACKUP},
+    {"FeatureStatus", PERSISTENT | BACKUP},
     {"FleetManagerPin", PERSISTENT},
-    {"GapAdjustCruise", PERSISTENT},
-    {"GapAdjustCruiseMax", PERSISTENT},
-    {"GapAdjustCruiseMin", PERSISTENT},
-    {"GapAdjustCruiseMode", PERSISTENT},
-    {"GapAdjustCruiseTr", PERSISTENT},
-    {"GmapKey", PERSISTENT},
-    {"HandsOnWheelMonitoring", PERSISTENT},
-    {"HideVEgoUi", PERSISTENT},
-    {"HkgSmoothStop", PERSISTENT},
+    {"ForceOffroad", CLEAR_ON_MANAGER_START},
+    {"GmapKey", PERSISTENT | BACKUP},
+    {"HandsOnWheelMonitoring", PERSISTENT | BACKUP},
+    {"HasAcceptedTermsSP", PERSISTENT},
+    {"HideVEgoUi", PERSISTENT | BACKUP},
+    {"HyundaiCruiseMainDefault", PERSISTENT | BACKUP},
+    {"HkgCustomLongTuning", PERSISTENT | BACKUP},
+    {"HkgSmoothStop", PERSISTENT | BACKUP},
+    {"HyundaiCruiseMainDefault", PERSISTENT | BACKUP},
     {"HotspotOnBoot", PERSISTENT},
-    {"LastCarModel", PERSISTENT},
+    {"HotspotOnBootConfirmed", PERSISTENT},
+    {"HyundaiRadarTracksAvailable", PERSISTENT},
+    {"HyundaiRadarTracksAvailableCache", PERSISTENT},
+    {"HyundaiRadarTracksAvailablePersistent", PERSISTENT},
+    {"LastCarModel", PERSISTENT | BACKUP},
     {"LastSpeedLimitSignTap", PERSISTENT},
-    {"LiveTorque", PERSISTENT},
-    {"LkasToggle", PERSISTENT},
-    {"MadsCruiseMain", PERSISTENT},
-    {"MadsIconToggle", PERSISTENT},
-    {"MapboxFullScreen", PERSISTENT},
-    {"MaxTimeOffroad", PERSISTENT},
-    {"OnroadScreenOff", PERSISTENT},
-    {"OnroadScreenOffBrightness", PERSISTENT},
-    {"OnroadScreenOffEvent", PERSISTENT},
-    {"OsmDbUpdatesCheck", PERSISTENT},
+    {"LastSunnylinkPingTime", CLEAR_ON_MANAGER_START},
+    {"LiveTorque", PERSISTENT | BACKUP},
+    {"LiveTorqueRelaxed", PERSISTENT | BACKUP},
+    {"LkasToggle", PERSISTENT | BACKUP},
+    {"MadsCruiseMain", PERSISTENT | BACKUP},
+    {"MadsIconToggle", PERSISTENT | BACKUP},
+    {"MapboxFullScreen", PERSISTENT | BACKUP},
+    {"MapTargetVelocities", PERSISTENT},
+    {"Map3DBuildings", PERSISTENT | BACKUP},
+    {"MaxTimeOffroad", PERSISTENT | BACKUP},
+    {"NavModelText", PERSISTENT | BACKUP},
+    {"NavModelUrl", PERSISTENT | BACKUP},
+    {"NNFF", PERSISTENT | BACKUP},
+    {"NNFFCarModel", PERSISTENT | BACKUP},
+    {"NNFFNoLateralJerk", PERSISTENT | BACKUP},
+    {"OnroadScreenOff", PERSISTENT | BACKUP},
+    {"OnroadScreenOffBrightness", PERSISTENT | BACKUP},
+    {"OnroadScreenOffEvent", PERSISTENT | BACKUP},
+    {"OnroadSettings", PERSISTENT | BACKUP},
     {"OsmLocal", PERSISTENT},
-    {"OsmLocalDb", PERSISTENT},
     {"OsmLocationName", PERSISTENT},
     {"OsmLocationTitle", PERSISTENT},
     {"OsmLocationUrl", PERSISTENT},
     {"OsmWayTest", PERSISTENT},
-    {"PathOffset", PERSISTENT},
-    {"QuietDrive", PERSISTENT},
-    {"RoadEdge", PERSISTENT},
-    {"ReverseAccChange", PERSISTENT},
-    {"ReverseDmCam", PERSISTENT},
-    {"ScreenRecorder", PERSISTENT},
-    {"ShowDebugUI", PERSISTENT},
-    {"SidebarTemperature", PERSISTENT},
-    {"SidebarTemperatureOptions", PERSISTENT},
-    {"SpeedLimitControl", PERSISTENT},
-    {"SpeedLimitPercOffset", PERSISTENT},
-    {"SpeedLimitValueOffset", PERSISTENT},
-    {"SpeedLimitOffsetType", PERSISTENT},
-    {"StandStillTimer", PERSISTENT},
-    {"StockLongToyota", PERSISTENT},
-    {"TorqueDeadzoneDeg", PERSISTENT},
-    {"TorqueFriction", PERSISTENT},
-    {"TorqueMaxLatAccel", PERSISTENT},
-    {"ToyotaTSS2Long", PERSISTENT},
-    {"TrueVEgoUi", PERSISTENT},
-    {"TurnSpeedControl", PERSISTENT},
-    {"TurnVisionControl", PERSISTENT},
-    {"VisionCurveLaneless", PERSISTENT},
-    {"VwAccType", PERSISTENT},
+    {"OsmDownloadedDate", PERSISTENT},
+    {"OvertakingAccelerationAssist", PERSISTENT},
+    {"PathOffset", PERSISTENT | BACKUP},
+    {"PauseLateralSpeed", PERSISTENT | BACKUP},
+    {"PCMVCruiseOverride", PERSISTENT | BACKUP},
+    {"PCMVCruiseOverrideSpeed", PERSISTENT | BACKUP},
+    {"QuietDrive", PERSISTENT | BACKUP},
+    {"RoadEdge", PERSISTENT | BACKUP},
+    {"ReverseAccChange", PERSISTENT | BACKUP},
+    {"ReverseDmCam", PERSISTENT | BACKUP},
+    {"ScreenRecorder", PERSISTENT | BACKUP},
+    {"ShowDebugUI", PERSISTENT | BACKUP},
+    {"SidebarTemperatureOptions", PERSISTENT | BACKUP},
+    {"SpeedLimitControlPolicy", PERSISTENT | BACKUP},
+    {"SpeedLimitEngageType", PERSISTENT | BACKUP},
+    {"SpeedLimitValueOffset", PERSISTENT | BACKUP},
+    {"SpeedLimitOffsetType", PERSISTENT | BACKUP},
+    {"SpeedLimitWarningFlash", PERSISTENT | BACKUP},
+    {"SpeedLimitWarningType", PERSISTENT | BACKUP},
+    {"SpeedLimitWarningValueOffset", PERSISTENT | BACKUP},
+    {"SpeedLimitWarningOffsetType", PERSISTENT | BACKUP},
+    {"StandStillTimer", PERSISTENT | BACKUP},
+    {"StockLongToyota", PERSISTENT | BACKUP},
+    {"SubaruManualParkingBrakeSng", PERSISTENT | BACKUP},
+    {"SunnylinkDongleId", PERSISTENT},
+    {"SunnylinkEnabled", PERSISTENT},
+    {"SunnylinkdPid", PERSISTENT},
+    {"TermsVersionSunnypilot", PERSISTENT},
+    {"TorqueDeadzoneDeg", PERSISTENT | BACKUP},
+    {"TorqueFriction", PERSISTENT | BACKUP},
+    {"TorqueLateralJerk", PERSISTENT | BACKUP},
+    {"TorqueMaxLatAccel", PERSISTENT | BACKUP},
+    {"TorquedOverride", PERSISTENT | BACKUP},
+    {"ToyotaAutoHold", PERSISTENT | BACKUP},
+    {"ToyotaAutoLockBySpeed", PERSISTENT | BACKUP},
+    {"ToyotaAutoUnlockByShifter", PERSISTENT | BACKUP},
+    {"ToyotaDriveMode", PERSISTENT | BACKUP},
+    {"ToyotaEnhancedBsm", PERSISTENT | BACKUP},
+    {"ToyotaSnG", PERSISTENT | BACKUP},
+    {"ToyotaTSS2Long", PERSISTENT | BACKUP},
+    {"TrueVEgoUi", PERSISTENT | BACKUP},
+    {"TurnSpeedControl", PERSISTENT | BACKUP},
+    {"TurnVisionControl", PERSISTENT | BACKUP},
+    {"DriverCameraHardwareMissing", PERSISTENT},
+    {"VisionCurveLaneless", PERSISTENT | BACKUP},
+    {"VwAccType", PERSISTENT | BACKUP},
+    {"VwCCOnly", PERSISTENT | BACKUP},
+    {"Offroad_ForceStatus", CLEAR_ON_MANAGER_START},
     {"Offroad_SupersededUpdate", PERSISTENT},
+
+    {"SunnylinkCache_Users", PERSISTENT},
+    {"SunnylinkCache_Roles", PERSISTENT},
+    {"EnableGitlabRunner", PERSISTENT | BACKUP},
+    {"EnableSunnylinkUploader", PERSISTENT | BACKUP},
+
+    // PFEIFER - MAPD {{
+    {"MapdVersion", PERSISTENT},
+    {"RoadName", CLEAR_ON_ONROAD_TRANSITION},
+    {"MapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"MapAdvisorySpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"NextMapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"OSMDownloadBounds", PERSISTENT},
+    {"OSMDownloadLocations", PERSISTENT},
+    {"OsmDownloadedDate", PERSISTENT},
+    {"OsmStateTitle", PERSISTENT},
+    {"OsmStateName", PERSISTENT},
+    {"OSMDownloadProgress", CLEAR_ON_MANAGER_START},
+    {"OsmDbUpdatesCheck", CLEAR_ON_MANAGER_START},  // mapd database update happens with device ON, reset on boot
+    // }} PFEIFER - MAPD
 };
 
 } // namespace

common/params.h

@@ -16,6 +16,7 @@ enum ParamKeyType {
   CLEAR_ON_OFFROAD_TRANSITION = 0x10,
   DONT_LOG = 0x20,
   DEVELOPMENT_ONLY = 0x40,
+  BACKUP = 0x80,
   ALL = 0xFFFFFFFF
 };
 

common/params_pyx.pyx

@@ -3,6 +3,7 @@
 from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
+from libcpp.map cimport map
 
 cdef extern from "common/params.h":
   cpdef enum ParamKeyType:
@@ -11,6 +12,7 @@ cdef extern from "common/params.h":
     CLEAR_ON_ONROAD_TRANSITION
     CLEAR_ON_OFFROAD_TRANSITION
     DEVELOPMENT_ONLY
+    BACKUP
     ALL
 
   cdef cppclass c_Params "Params":
@@ -26,6 +28,7 @@ cdef extern from "common/params.h":
     string getParamPath(string) nogil
     void clearAll(ParamKeyType)
     vector[string] allKeys()
+    map[string, string] readAll()
 
 
 def ensure_bytes(v):
@@ -116,3 +119,6 @@ cdef class Params:
 
   def all_keys(self):
     return self.p.allKeys()
+
+  def read_all(self):
+    return self.p.readAll()

common/realtime.py

@@ -2,6 +2,8 @@
 import gc
 import os
 import time
+import threading
+import psutil
 from collections import deque
 
 from openpilot.common.threadname import getthreadname
@@ -44,6 +46,14 @@ def config_realtime_process(cores: int | list[int], priority: int) -> None:
   set_core_affinity(c)
 
 
+def set_thread_affinity(thread: threading.Thread, cores: list[int]) -> None:
+  try:
+    process = psutil.Process(thread.ident)
+    process.cpu_affinity(cores)
+  except Exception as e:
+    print(f"Error setting thread affinity: {e}")
+
+
 class Ratekeeper:
   def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
     """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""

common/transformations/camera.py

@@ -47,9 +47,9 @@ class DeviceCameraConfig:
         yield cam, getattr(self, cam)
 
 _ar_ox_fisheye = CameraConfig(1928, 1208, 567.0)  # focal length probably wrong? magnification is not consistent across frame
-_os_fisheye = CameraConfig(2688, 1520, 567.0 / 2 * 3)
+_os_fisheye = CameraConfig(2688 // 2, 1520 // 2, 567.0 / 4 * 3)
 _ar_ox_config = DeviceCameraConfig(CameraConfig(1928, 1208, 2648.0), _ar_ox_fisheye, _ar_ox_fisheye)
-_os_config = DeviceCameraConfig(CameraConfig(2688, 1520, 2648.0 * 2 / 3), _os_fisheye, _os_fisheye)
+_os_config = DeviceCameraConfig(CameraConfig(2688 // 2, 1520 // 2, 1522.0 * 3 / 4), _os_fisheye, _os_fisheye)
 _neo_config = DeviceCameraConfig(CameraConfig(1164, 874, 910.0), CameraConfig(816, 612, 650.0), _NoneCameraConfig())
 
 DEVICE_CAMERAS = {

docs/CARS.md

@@ -4,7 +4,7 @@
 
 A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
 
-# 288 Supported Cars
+# 287 Supported Cars
 
 |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br>&nbsp;|Video|
 |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
@@ -25,8 +25,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Chrysler|Pacifica 2017-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2017-18">Buy Here</a></sub></details>||
 |Chrysler|Pacifica 2019-20|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2019-20">Buy Here</a></sub></details>||
 |Chrysler|Pacifica 2021-23|All|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2021-23">Buy Here</a></sub></details>||
-|Chrysler|Pacifica Hybrid 2017|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2017">Buy Here</a></sub></details>||
-|Chrysler|Pacifica Hybrid 2018|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2018">Buy Here</a></sub></details>||
+|Chrysler|Pacifica Hybrid 2017-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2017-18">Buy Here</a></sub></details>||
 |Chrysler|Pacifica Hybrid 2019-24|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2019-24">Buy Here</a></sub></details>||
 |comma|body|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|None||
 |CUPRA|Ateca 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=CUPRA&model=Ateca 2018-23">Buy Here</a></sub></details>||

docs/README.md

@@ -1,3 +1,26 @@
-# openpilot-docs
+# openpilot docs
 
-These docs are autogenerated from [this folder](https://github.com/commaai/openpilot/tree/master/docs) in the main openpilot repository.
+This is the source for [docs.comma.ai](https://docs.comma.ai).
+The site is updated on pushes to master by this [workflow](../.github/workflows/docs.yaml).
+
+## Development
+NOTE: Those commands must be run in the root directory of openpilot, **not /docs**
+
+**1. Install the docs dependencies**
+``` bash
+pip install .[docs]
+```
+
+**2. Build the new site**
+``` bash
+mkdocs build
+```
+
+**3. Run the new site locally**
+``` bash
+mkdocs serve
+```
+
+References:
+* https://www.mkdocs.org/getting-started/
+* https://github.com/ntno/mkdocs-terminal

docs/WORKFLOW.md

@@ -29,7 +29,7 @@ pytest
 cd system/loggerd && pytest .
 
 # run the linter
-pre-commit run --all
+op lint
 ```
 
 ## Testing

docs/car-porting/what-is-a-car-port.md

@@ -0,0 +1,22 @@
+# What is a car port?
+
+A car port enables openpilot support on a particular car. Each car model openpilot supports needs to be individually ported. All car ports live in `openpilot/selfdrive/car/`.
+
+The complexity of a car port varies depending on many factors including:
+* existing openpilot support for similar cars
+* architecture and APIs available in the car
+
+
+# Structure of a car port
+* `interface.py`: Interface for the car, defines the CarInterface class
+* `carstate.py`: Reads CAN from car and builds openpilot CarState message
+* `carcontroller.py`: Builds CAN messages to send to car
+* `values.py`: Limits for actuation, general constants for cars, and supported car documentation
+* `radar_interface.py`: Interface for parsing radar points from the car
+
+
+# Overiew
+
+[Jason Young](https://github.com/jyoung8607) gave a talk at COMMA_CON with an overview of the car porting process. The talk is available on YouTube:
+
+https://youtu.be/KcfzEHB6ms4?si=5szh1PX6TksOCKmM

docs/concepts/glossary.md

@@ -0,0 +1,8 @@
+# openpilot glossary
+
+* **route**:
+* **segment**: routes are split into one minute chunks called segments.
+* **panda**: this is . See the repo.
+* **onroad**:
+* **offroad**:
+* **comma 3X**:

docs/concepts/logs.md

@@ -1,10 +1,8 @@
-# loggerd
+# Logging
 
 openpilot records routes in one minute chunks called segments. A route starts on the rising edge of ignition and ends on the falling edge.
 
-Check out our [python library](https://github.com/commaai/openpilot/blob/master/tools/lib/logreader.py) for reading openpilot logs. Also checkout our [tools](https://github.com/commaai/openpilot/tree/master/tools) to replay and view your data. These are the same tools we use to debug and develop openpilot.
-
-## log types
+Check out our [Python library](https://github.com/commaai/openpilot/blob/master/tools/lib/logreader.py) for reading openpilot logs. Also checkout our [tools](https://github.com/commaai/openpilot/tree/master/tools) to replay and view your data. These are the same tools we use to debug and develop openpilot.
 
 For each segment, openpilot records the following log types:
 
@@ -15,9 +13,10 @@ rlogs contain all the messages passed amongst openpilot's processes. See [cereal
 ## {f,e,d}camera.hevc
 
 Each camera stream is H.265 encoded and written to its respective file.
-* fcamera.hevc is the road camera
-* ecamera.hevc is the wide road camera
-* dcamera.hevc is the driver camera
+
+* `fcamera.hevc` is the road camera
+* `ecamera.hevc` is the wide road camera
+* `dcamera.hevc` is the driver camera
 
 ## qlog.bz2 & qcamera.ts
 

docs/concepts/safety.md

@@ -0,0 +1 @@
+../SAFETY.md

docs/contributing/architecture.md

@@ -0,0 +1 @@
+# Architecture

docs/contributing/roadmap.md

@@ -0,0 +1,30 @@
+# Roadmap
+
+This is the roadmap for the next major openpilot releases. Also check out
+
+* [Milestones](https://github.com/commaai/openpilot/milestones) for minor releases
+* [Projects](https://github.com/commaai/openpilot/projects?query=is%3Aopen) for shorter-term projects not tied to releases
+* [Bounties](https://comma.ai/bounties) for paid individual issues
+
+## openpilot 0.10
+
+openpilot 0.10 will be the first release with a driving policy trained in
+a [learned simulator](https://youtu.be/EqQNZXqzFSI).
+
+* Driving model trained in a learned simlator
+* Always-on driver monitoring (behind a toggle)
+* GPS removed from the driving stack
+* 100KB qlogs
+* `master-ci` pushed after 1000 hours of hardware-in-the-loop testing
+* Car interface code moved into [opendbc](https://github.com/commaai/opendbc)
+* openpilot on PC for Linux x86, Linux arm64, and Mac (Apple Silicon)
+
+## openpilot 1.0
+
+openpilot 1.0 will feature a fully end-to-end driving policy.
+
+* End-to-end longitudinal control in Chill mode
+* Automatic Emergency Braking (AEB)
+* Driver monitoring with sleep detection
+* Rolling updates/releases pushed out by CI
+* [panda safety 1.0](https://github.com/orgs/commaai/projects/27)

docs/getting-started/what-is-openpilot.md

@@ -1,6 +1,6 @@
 # What is openpilot?
 
-[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](docs/INTEGRATION.md) and [limitations](docs/LIMITATIONS.md).
+[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](https://github.com/commaai/openpilot/blob/master/docs/INTEGRATION.md) and [limitations](https://github.com/commaai/openpilot/blob/master/docs/LIMITATIONS.md).
 
 
 ## How do I use it?

docs/how-to/connect-to-comma.md

@@ -1,39 +1,42 @@
-# SSH
+# connect to a comma 3/3X
 
-## Quick Start
+A comma 3/3X is a normal [Linux](https://github.com/commaai/agnos-builder) computer that exposes [SSH](https://wiki.archlinux.org/title/Secure_Shell) and a [serial console](https://wiki.archlinux.org/title/Working_with_the_serial_console).
+
+## Serial Console
+
+On both the comma three and 3X, the serial console is accessible from the main OBD-C port.
+Connect the comma 3/3X to your computer with a normal USB C cable, or use a [comma serial](https://comma.ai/shop/comma-serial) for steady 12V power.
+
+On the comma three, the serial console is exposed through a UART-to-USB chip, and `tools/serial/connect.sh` can be used to connect.
+
+On the comma 3X, the serial console is accessible through the [panda](https://github.com/commaai/panda) using the `panda/tests/som_debug.sh` script.
+
+## SSH
 
 In order to SSH into your device, you'll need a GitHub account with SSH keys. See this [GitHub article](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh) for getting your account setup with SSH keys.
 
 * Enable SSH in your device's settings
 * Enter your GitHub username in the device's settings
 * Connect to your device
-  * Username: `comma`
-  * Port: `22` or `8022`
+    * Username: `comma`
+    * Port: `22`
 
 Here's an example command for connecting to your device using its tethered connection:<br />
 `ssh comma@192.168.43.1`
 
 For doing development work on device, it's recommended to use [SSH agent forwarding](https://docs.github.com/en/developers/overview/using-ssh-agent-forwarding).
 
-## Notes
+### Notes
 
 The public keys are only fetched from your GitHub account once. In order to update your device's authorized keys, you'll need to re-enter your GitHub username.
 
 The `id_rsa` key in this directory only works while your device is in the setup state with no software installed. After installation, that default key will be removed.
 
-See the [community wiki](https://github.com/commaai/openpilot/wiki/SSH) for more detailed instructions and information.
+#### ssh.comma.ai proxy
 
-# Connecting to ssh.comma.ai
-SSH into your comma device from anywhere with `ssh.comma.ai`. Requires a [comma prime subscription](https://comma.ai/connect).
+With a [comma prime subscription](https://comma.ai/connect), you can SSH into your comma device from anywhere.
 
-## Setup
-
-With software version 0.6.1 or newer, enter your GitHub username on your device under Developer Settings. Your GitHub authorized public keys will become your authorized SSH keys for `ssh.comma.ai`. You can add any additional keys in `/system/comma/home/.ssh/authorized_keys.persist`.
-
-## Recommended .ssh/config
-
-With the below SSH configuration, you can type `ssh comma-{dongleid}` to connect to your device through `ssh.comma.ai`.<br />
-For example: `ssh comma-ffffffffffffffff`
+With the below SSH configuration, you can type `ssh comma-{dongleid}` to connect to your device through `ssh.comma.ai`.
 
 ```
 Host comma-*
@@ -41,20 +44,21 @@ Host comma-*
   User comma
   IdentityFile ~/.ssh/my_github_key
   ProxyCommand ssh %h@ssh.comma.ai -W %h:%p
+
 Host ssh.comma.ai
   Hostname ssh.comma.ai
   Port 22
   IdentityFile ~/.ssh/my_github_key
 ```
 
-## One-off connection
+### One-off connection
 
 ```
 ssh -i ~/.ssh/my_github_key -o ProxyCommand="ssh -i ~/.ssh/my_github_key -W %h:%p -p %p %h@ssh.comma.ai" comma@ffffffffffffffff
 ```
 (Replace `ffffffffffffffff` with your dongle_id)
 
-## ssh.comma.ai host key fingerprint
+### ssh.comma.ai host key fingerprint
 
 ```
 Host key fingerprint is SHA256:X22GOmfjGb9J04IA2+egtdaJ7vW9Fbtmpz9/x8/W1X4

docs/how-to/replay-a-drive.md

@@ -0,0 +1,14 @@
+# Replay
+
+Replaying is a critical tool for openpilot development and debugging.
+
+## Replaying a route
+*Hardware required: none*
+
+Just run `tools/replay/replay --demo`.
+
+## Replaying CAN data
+*Hardware required: jungle and comma 3/3X*
+
+1. Connect your PC to a jungle.
+2.

docs/how-to/turn-the-speed-blue.md

@@ -0,0 +1,98 @@
+# Turn the speed blue
+*A getting started guide for openpilot development*
+
+In 30 minutes, we'll get an openpilot development environment setup on your computer and make some changes to openpilot's UI.
+
+And if you have a comma 3/3X, we'll deploy the change to your device for testing.
+
+## 1. Setup your development environment
+
+Run this to clone openpilot and install all the dependencies:
+```bash
+curl -fsSL openpilot.comma.ai | bash
+```
+
+Navigate to openpilot folder & activate a Python virtual environment
+```bash
+cd openpilot
+source .venv/bin/activate
+```
+
+Then, compile openpilot:
+```bash
+scons -j8
+```
+
+## 2. Run replay
+
+We'll run the `replay` tool with the demo route to get data streaming for testing our UI changes.
+```bash
+# in terminal 1
+tools/replay/replay --demo
+
+# in terminal 2
+selfdrive/ui/ui
+```
+
+The openpilot UI should launch and show a replay of the demo route.
+
+If you have your own comma device, you can replace `--demo` with one of your own routes from comma connect.
+
+## 3. Make the speed blue
+
+Search for “mph” with git grep in the `ui` folder.
+```bash
+$ git grep "mph" selfdrive/ui/
+paint.cc:  ui_draw_text(s, s->fb_w/2, 290, s->scene.is_metric ? "km/h" : "mph", 36 * 2.5, COLOR_WHITE_ALPHA(200), "sans-regular");
+```
+
+The line right above contains the actual speed. Unfortunately, COLOR_BLUE isn’t defined, but a git grep of COLOR_WHITE shows it’s nvgRGBA(255, 255, 255, 255). Personally, I like a lighter blue, so I went with #8080FF.
+```bash
+$ git diff
+diff --git a/selfdrive/ui/paint.cc b/selfdrive/ui/paint.cc
+index 821d95115..cc996eaa1 100644
+--- a/selfdrive/ui/paint.cc
++++ b/selfdrive/ui/paint.cc
+@@ -175,8 +175,8 @@ static void ui_draw_vision_speed(UIState *s) {
+   const float speed = std::max(0.0, (*s->sm)["carState"].getCarState().getVEgo() * (s->scene.is_metric ? 3.6 : 2.2369363));
+   const std::string speed_str = std::to_string((int)std::nearbyint(speed));
+   nvgTextAlign(s->vg, NVG_ALIGN_CENTER | NVG_ALIGN_BASELINE);
+-  ui_draw_text(s, s->fb_w/2, 210, speed_str.c_str(), 96 * 2.5, COLOR_WHITE, "sans-bold");
+-  ui_draw_text(s, s->fb_w/2, 290, s->scene.is_metric ? "km/h" : "mph", 36 * 2.5, COLOR_WHITE_ALPHA(200), "sans-regular");
++  ui_draw_text(s, s->fb_w/2, 210, speed_str.c_str(), 96 * 2.5, nvgRGBA(128, 128, 255, 255), "sans-bold");
++  ui_draw_text(s, s->fb_w/2, 290, s->scene.is_metric ? "km/h" : "mph", 36 * 2.5, nvgRGBA(128, 128, 255, 200), "sans-regular");
+ }
+
+ static void ui_draw_vision_event(UIState *s) {
+```
+
+
+## 4. Rebuild UI, and admire your work
+
+```
+scons -j8 && selfdrive/ui/ui
+```
+
+![](https://blog.comma.ai/img/blue_speed_ui.png)
+
+## 5. Push your fork to GitHub
+
+Click fork on GitHub. Then, push with:
+```bash
+git remote rm origin
+git remote add origin git@github.com:<your-github-username>/openpilot.git
+git add .
+git commit -m "Make the speed blue."
+git push --set-upstream origin master
+```
+
+## 6. Run your fork on device in your car!
+
+Uninstall openpilot from your device through the settings. Then, enter the URL for your very own installer:
+```
+installer.comma.ai/<your-github-username>/master
+```
+
+## 7. Admire your work IRL
+
+![](https://blog.comma.ai/img/c3_blue_ui.jpg)

docs/index.md

@@ -0,0 +1 @@
+getting-started/what-is-openpilot.md

docs/new/README.md

@@ -1,12 +0,0 @@
-This is the source for a new https://docs.comma.ai. It's not hosted anywhere yet, but it's easy to run locally.
-
-https://www.mkdocs.org/getting-started/
-
-```
-pip install mkdocs mkdocs-terminal
-mkdocs serve
-```
-
-inspiration:
-* https://rerun.io/docs/
-* https://docs.expo.dev/

docs/new/docs/car-porting/what-is-a-car-port.md

@@ -1,9 +0,0 @@
-# What is a car port?
-
-All car ports live in `openpilot/selfdrive/car/`.
-
-* interface.py: Interface for the car, defines the CarInterface class
-* carstate.py: Reads CAN from car and builds openpilot CarState message
-* carcontroller.py: Builds CAN messages to send to car
-* values.py: Limits for actuation, general constants for cars, and supported car documentation
-* radar_interface.py: Interface for parsing radar points from the car

docs/new/docs/how-to/turning-the-speed-blue.md

@@ -1,4 +0,0 @@
-This section is for how-to's on common workflows.
-
-They'll be like this blog post we wrote:
-https://blog.comma.ai/turning-the-speed-blue/

docs/new/mkdocs.yml

@@ -1,18 +0,0 @@
-site_name: openpilot docs
-docs_dir: docs
-repo_url: https://github.com/commaai/openpilot/
-
-theme:
-  name: terminal
-  features:
-    - navigation.side.toc.hide
-
-nav:
-  - Getting Started:
-    - What is openpilot?: getting-started/what-is-openpilot.md
-  - How-to:
-    - Turn the speed blue: how-to/turning-the-speed-blue.md
-  - Car Porting:
-    - What is a car port?: car-porting/what-is-a-car-port.md
-    - Porting a car brand: car-porting/brand-port.md
-    - Porting a car model: car-porting/model-port.md

launch_chffrplus.sh

@@ -86,7 +86,8 @@ function launch {
   if [ ! -f $DIR/prebuilt ]; then
     ./build.py
   fi
-  ./manager.py
+
+  ./mapd_installer.py; ./manager.py
 
   # if broken, keep on screen error
   while true; do sleep 1; done

mkdocs.yml

@@ -0,0 +1,38 @@
+site_name: openpilot docs
+repo_url: https://github.com/commaai/openpilot/
+site_url: https://docs.comma.ai
+
+exclude_docs: README.md
+
+strict: true
+docs_dir: docs
+site_dir: docs_site/
+
+theme:
+  name: readthedocs
+  navigation_depth: 3
+
+nav:
+  - Getting Started:
+    - What is openpilot?: getting-started/what-is-openpilot.md
+  - How-to:
+    - Turn the speed blue: how-to/turn-the-speed-blue.md
+    - Connect to a comma 3/3X: how-to/connect-to-comma.md
+    #- Replay a drive: how-to/replay-a-drive.md
+  - Concepts:
+    - Logs: concepts/logs.md
+    - Safety: concepts/safety.md
+  - Car Porting:
+    - What is a car port?: car-porting/what-is-a-car-port.md
+    - Porting a car brand: car-porting/brand-port.md
+    - Porting a car model: car-porting/model-port.md
+  - Contributing:
+    - Roadmap: contributing/roadmap.md
+    #- Architecture: contributing/architecture.md
+    - Contributing Guide →: https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md
+  - Links:
+    - Blog →: https://blog.comma.ai
+    - Bounties →: https://comma.ai/bounties
+    - GitHub →: https://github.com/commaai
+    - Discord →: https://discord.comma.ai
+    - X →: https://x.com/comma_ai

opendbc

@@ -0,0 +1 @@
+opendbc_repo/opendbc

pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
   "pycapnp",
   "Cython",
   "setuptools",
-  "numpy < 2.0.0", # control does not support numpy 2
+  "numpy",
 
   # body / webrtcd
   "aiohttp",
@@ -52,27 +52,30 @@ dependencies = [
   "websocket_client",
 
   # acados deps
-  "casadi",
+  "casadi @ https://github.com/commaai/casadi/releases/download/nightly-release-3.6.6/casadi-3.6.6-cp312-none-manylinux2014_aarch64.whl ; (python_version == '3.12' and platform_machine == 'aarch64')", # TODO: Go back to pypi casadi when they fix aarch64 for python312
+  "casadi; platform_machine != 'aarch64' or python_version != '3.12'",
   "future-fstrings",
 
   # these should be removed
   "psutil",
   "pycryptodome", # used in updated/casync, panda, body, and a test
 
-  #logreader
-  "zstd",
+  # logreader
+  "zstandard",
 ]
 
 [project.optional-dependencies]
 docs = [
   "Jinja2",
+  "natsort",
+  "mkdocs",
 ]
 
 testing = [
   "coverage",
   "hypothesis ==6.47.*",
+  "import-linter",
   "mypy",
-  "pre-commit",
   "pytest",
   "pytest-cov",
   "pytest-cpp",
@@ -83,30 +86,25 @@ testing = [
   "pytest-asyncio",
   "pytest-mock",
   "pytest-repeat",
-  "ruff"
+  "ruff",
+  "codespell",
 ]
 
 dev = [
   "av",
   "azure-identity",
   "azure-storage-blob",
-  "breathe",
-  "control",
   "dictdiffer",
   "flaky",
   "inputs",
   "lru-dict",
   "matplotlib",
-  "metadrive-simulator; platform_machine != 'aarch64'",
-  "mpld3",
-  "myst-parser",
-  "natsort",
-  "opencv-python-headless",
+  "metadrive-simulator@git+https://github.com/commaai/metadrive@opencv_headless ; platform_machine != 'aarch64'",
   "parameterized >=0.8, <0.9",
-  #pprofile = "*"
+  #"pprofile",
   "pyautogui",
-  "pygame",
   "pyopencl; platform_machine != 'aarch64'", # broken on arm64
+  "pytools < 2024.1.11; platform_machine != 'aarch64'", # pyopencl use a broken version
   "pywinctl",
   "pyprof2calltree",
   "rerun-sdk",
@@ -116,12 +114,8 @@ dev = [
 
   # this is only pinned since 5.15.11 is broken
   "pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
-
 ]
 
-[tool.uv.sources]
-metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "opencv_headless" }
-
 [project.urls]
 Homepage = "https://comma.ai"
 
@@ -132,9 +126,12 @@ build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
 packages = [ "." ]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/  -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
+addopts = "--ignore=openpilot/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/  -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
 cpp_files = "test_*"
 cpp_harness = "selfdrive/test/cpp_harness.py"
 python_files = "test_*.py"
@@ -167,6 +164,13 @@ testpaths = [
   "cereal/messaging/tests",
 ]
 
+[tool.codespell]
+count = true
+quiet-level = 3
+# if you've got a short variable name that's getting flagged, add it here
+ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn"
+builtin = "clear,rare,informal,usage,code,names,en-GB_to_en-US"
+
 [tool.mypy]
 python_version = "3.11"
 plugins = [
@@ -223,7 +227,7 @@ lint.ignore = [
   "UP038",   # (x, y) -> x|y for isinstance
 ]
 line-length = 160
-target-version="py311"
+target-version ="py311"
 exclude = [
   "body",
   "cereal",
@@ -235,7 +239,8 @@ exclude = [
   "teleoprtc_repo",
   "third_party",
 ]
-lint.flake8-implicit-str-concat.allow-multiline=false
+lint.flake8-implicit-str-concat.allow-multiline = false
+
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
 "selfdrive".msg = "Use openpilot.selfdrive"
 "common".msg = "Use openpilot.common"
@@ -247,5 +252,6 @@ lint.flake8-implicit-str-concat.allow-multiline=false
 
 [tool.coverage.run]
 concurrency = ["multiprocessing", "thread"]
+
 [tool.ruff.format]
 quote-style = "preserve"

release/build_dev.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/bash -e
+
+# git diff --name-status origin/release3-staging | grep "^A" | less
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+
+BUILD_DIR=/data/openpilot
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
+
+FILES_SRC="release/files_tici"
+DEV_BRANCH="dev-c3"
+
+
+# set git identity
+source $DIR/identity.sh
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+echo "[-] Setting up repo T=$SECONDS"
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git init
+# set git username/password
+source /data/identity.sh
+git remote add origin https://github.com/sunnyhaibin/sunnypilot.git
+git fetch origin $DEV_BRANCH
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
+
+# in the directory
+cd $BUILD_DIR
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+rm -f panda/board/obj/bootstub.panda.bin
+rm -f panda/board/obj/bootstub.panda_h7.bin
+
+VERSION=$(date '+%Y.%m.%d')
+echo "#define COMMA_VERSION \"$VERSION-dev\"" > common/version.h
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$DEV_BRANCH
+
+# Build panda firmware
+pushd panda/
+scons -u .
+mv board/obj/panda.bin.signed /tmp/panda.bin.signed
+mv board/obj/panda_h7.bin.signed /tmp/panda_h7.bin.signed
+mv board/obj/bootstub.panda.bin /tmp/bootstub.panda.bin
+mv board/obj/bootstub.panda_h7.bin /tmp/bootstub.panda_h7.bin
+popd
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+scons -j$(nproc)
+
+# Ensure no submodules in release
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '*.cc' -delete
+find . -name '__pycache__' -delete
+find selfdrive/ui/ -name '*.h' -delete
+rm -rf panda/board panda/certs panda/crypto
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+rm -rf selfdrive/ui/replay/
+# Move back signed panda fw
+mkdir -p panda/board/obj
+mv /tmp/panda.bin.signed panda/board/obj/panda.bin.signed
+mv /tmp/panda_h7.bin.signed panda/board/obj/panda_h7.bin.signed
+mv /tmp/bootstub.panda.bin panda/board/obj/bootstub.panda.bin
+mv /tmp/bootstub.panda_h7.bin panda/board/obj/bootstub.panda_h7.bin
+
+# Restore third_party
+git checkout third_party/
+
+# Mark as prebuilt release
+touch prebuilt
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+# Add built files to git
+git add -f .
+git commit --amend -m "sunnypilot v$VERSION
+version: sunnypilot v$SP_VERSION release
+date: $DATETIME
+master commit: $GIT_HASH
+"
+git branch -m dev-c3
+
+# Run tests
+#TEST_FILES="tools/"
+#cd $SOURCE_DIR
+#cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+#cd $BUILD_DIR
+#RELEASE=1 selfdrive/test/test_onroad.py
+#selfdrive/manager/test/test_manager.py
+#selfdrive/car/tests/test_car_interfaces.py
+#rm -rf $TEST_FILES
+
+if [ ! -z "$PUSH" ]; then
+  echo "[-] pushing T=$SECONDS"
+  git push -f origin $DEV_BRANCH
+fi
+
+echo "[-] done T=$SECONDS"

release/build_release.sh

@@ -19,14 +19,17 @@ fi
 
 # set git identity
 source $DIR/identity.sh
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
 echo "[-] Setting up repo T=$SECONDS"
 rm -rf $BUILD_DIR
 mkdir -p $BUILD_DIR
 cd $BUILD_DIR
 git init
-git remote add origin git@github.com:commaai/openpilot.git
-git checkout --orphan $RELEASE_BRANCH
+# set git username/password
+source /data/identity.sh
+git remote add origin https://github.com/sunnyhaibin/sunnypilot.git
+git fetch origin $RELEASE_BRANCH
 
 # do the files copy
 echo "[-] copying files T=$SECONDS"
@@ -38,20 +41,23 @@ cd $BUILD_DIR
 
 rm -f panda/board/obj/panda.bin.signed
 rm -f panda/board/obj/panda_h7.bin.signed
+rm -f panda/board/obj/bootstub.panda.bin
+rm -f panda/board/obj/bootstub.panda_h7.bin
 
 VERSION=$(cat common/version.h | awk -F[\"-]  '{print $2}')
 echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h
 
 echo "[-] committing version $VERSION T=$SECONDS"
 git add -f .
-git commit -a -m "openpilot v$VERSION release"
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$RELEASE_BRANCH
 
 # Build
 export PYTHONPATH="$BUILD_DIR"
 scons -j$(nproc) --minimal
 
 # release panda fw
-CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/
+scons -j$(nproc) panda/
 
 # Ensure no submodules in release
 if test "$(git submodule--helper list | wc -l)" -gt "0"; then
@@ -67,9 +73,12 @@ find . -name '*.o' -delete
 find . -name '*.os' -delete
 find . -name '*.pyc' -delete
 find . -name 'moc_*' -delete
+find . -name '*.cc' -delete
 find . -name '__pycache__' -delete
+find selfdrive/ui/ -name '*.h' -delete
 rm -rf .sconsign.dblite Jenkinsfile release/
 rm selfdrive/modeld/models/supercombo.onnx
+rm -rf selfdrive/ui/replay/
 
 find third_party/ -name '*x86*' -exec rm -r {} +
 find third_party/ -name '*Darwin*' -exec rm -r {} +
@@ -81,19 +90,25 @@ git checkout third_party/
 # Mark as prebuilt release
 touch prebuilt
 
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
 # Add built files to git
 git add -f .
-git commit --amend -m "openpilot v$VERSION"
+git commit --amend -m "sunnypilot v$VERSION"
+git branch -m release-c3
 
 # Run tests
-TEST_FILES="tools/"
-cd $SOURCE_DIR
-cp -pR -n --parents $TEST_FILES $BUILD_DIR/
-cd $BUILD_DIR
-RELEASE=1 pytest -n0 -s selfdrive/test/test_onroad.py
+#TEST_FILES="tools/"
+#cd $SOURCE_DIR
+#cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+#cd $BUILD_DIR
+#RELEASE=1 pytest -n0 -s selfdrive/test/test_onroad.py
 #system/manager/test/test_manager.py
-pytest selfdrive/car/tests/test_car_interfaces.py
-rm -rf $TEST_FILES
+#pytest selfdrive/car/tests/test_car_interfaces.py
+#rm -rf $TEST_FILES
 
 if [ ! -z "$RELEASE_BRANCH" ]; then
   echo "[-] pushing release T=$SECONDS"

release/build_staging.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/bash -e
+
+# git diff --name-status origin/release3-staging | grep "^A" | less
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+
+BUILD_DIR=/data/openpilot
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
+
+FILES_SRC="release/files_tici"
+STAGING_BRANCH="staging-c3"
+
+
+# set git identity
+source $DIR/identity.sh
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+echo "[-] Setting up repo T=$SECONDS"
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git init
+# set git username/password
+source /data/identity.sh
+git remote add origin https://github.com/sunnyhaibin/sunnypilot.git
+git fetch origin $STAGING_BRANCH
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
+
+# in the directory
+cd $BUILD_DIR
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+rm -f panda/board/obj/bootstub.panda.bin
+rm -f panda/board/obj/bootstub.panda_h7.bin
+
+VERSION=$(cat common/version.h | awk -F[\"-]  '{print $2}')
+echo "#define COMMA_VERSION \"$VERSION-staging\"" > common/version.h
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$STAGING_BRANCH
+
+# Build panda firmware
+pushd panda/
+scons -u .
+mv board/obj/panda.bin.signed /tmp/panda.bin.signed
+mv board/obj/panda_h7.bin.signed /tmp/panda_h7.bin.signed
+mv board/obj/bootstub.panda.bin /tmp/bootstub.panda.bin
+mv board/obj/bootstub.panda_h7.bin /tmp/bootstub.panda_h7.bin
+popd
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+scons -j$(nproc)
+
+# Ensure no submodules in release
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '*.cc' -delete
+find . -name '__pycache__' -delete
+find selfdrive/ui/ -name '*.h' -delete
+rm -rf panda/board panda/certs panda/crypto
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+#rm models/supercombo_badweights.thneed
+rm -rf selfdrive/ui/replay/
+# Move back signed panda fw
+mkdir -p panda/board/obj
+mv /tmp/panda.bin.signed panda/board/obj/panda.bin.signed
+mv /tmp/panda_h7.bin.signed panda/board/obj/panda_h7.bin.signed
+mv /tmp/bootstub.panda.bin panda/board/obj/bootstub.panda.bin
+mv /tmp/bootstub.panda_h7.bin panda/board/obj/bootstub.panda_h7.bin
+
+# Restore third_party
+git checkout third_party/
+
+# Mark as prebuilt release
+touch prebuilt
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+# Add built files to git
+git add -f .
+git commit --amend -m "sunnypilot v$VERSION"
+git branch -m staging-c3
+
+# Run tests
+#TEST_FILES="tools/"
+#cd $SOURCE_DIR
+#cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+#cd $BUILD_DIR
+#RELEASE=1 selfdrive/test/test_onroad.py
+#selfdrive/manager/test/test_manager.py
+#selfdrive/car/tests/test_car_interfaces.py
+#rm -rf $TEST_FILES
+
+if [ ! -z "$PUSH" ]; then
+  echo "[-] pushing T=$SECONDS"
+  git push -f origin $STAGING_BRANCH
+fi
+
+echo "[-] done T=$SECONDS"

release/ci/build.sh

@@ -0,0 +1,45 @@
+# UNUSED YET BUT I WANT TO KEEP IT BECAUSE I INTEND TO USE IT SOON
+
+#!/usr/bin/env bash
+
+# set -e
+
+# DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+# cd $DIR
+
+# # Take parameters as arguments
+# SOURCE_DIR=$1
+# OUTPUT_DIR=$2
+
+# # Check parameters
+# if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then
+#     echo "Error: No source or output directory provided."
+#     exit 1
+# fi
+
+# if [ -z "$TARGETPLATFORM" ]; then
+#     # Detect the platform (e.g., x86_64, aarch64, etc.)
+#     platform=$(uname -m)
+
+#     # Map to Docker's platform syntax
+#     if [ "$platform" = "x86_64" ]; then
+#         TARGETPLATFORM="linux/amd64"
+#     elif [ "$platform" = "aarch64" ]; then
+#         TARGETPLATFORM="linux/arm64"
+#     # Add here more elif statements for other architectures if needed.
+#     else
+#         echo "Unknown platform: $platform"
+#         exit 1
+#     fi
+
+#     export TARGETPLATFORM
+# fi
+
+# echo "TARGETPLATFORM IS: [${TARGETPLATFORM}]"
+
+# # Let's bring the submodules!
+# git submodule update --init --recursive
+
+# # Build
+# docker compose -f ${SOURCE_DIR}/docker-compose.yml build
+# docker compose -f ${SOURCE_DIR}/docker-compose.yml up

release/ci/build_and_publish.sh

@@ -0,0 +1,27 @@
+# UNUSED YET BUT I WANT TO KEEP IT BECAUSE I INTEND TO USE IT SOON
+
+# #!/usr/bin/env bash
+# DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+# cd $DIR
+
+# DEV_BRANCH=$1
+# GIT_ORIGIN=$2
+# if [ -z "$DEV_BRANCH" ]; then
+#     DEV_BRANCH="dev-c3-test"
+# fi
+
+# if [ -z "$GIT_ORIGIN" ]; then
+#     GIT_ORIGIN="git@github.com:devtekve/openpilot-og.git" #This is a default that should be changed
+#     echo "No GIT_ORIGIN provided, we will use the default [${GIT_ORIGIN}]"
+# fi
+
+# VERSION=$(date '+%Y.%m.%d')
+# SOURCE_DIR="$(git rev-parse --show-toplevel)"
+# OUTPUT_DIR=${SOURCE_DIR}/output
+
+
+# echo "Calling to build [${DIR}/build.sh ${SOURCE_DIR} ${OUTPUT_DIR} ${VERSION}]"
+# $DIR/build.sh "${SOURCE_DIR}" "${OUTPUT_DIR}" "${VERSION}"
+
+# echo "Calling to publish [${DIR}/publish.sh ${SOURCE_DIR} ${OUTPUT_DIR} ${DEV_BRANCH} ${VERSION} ${GIT_ORIGIN}]"
+# $DIR/publish.sh "${SOURCE_DIR}" "${OUTPUT_DIR}" "${DEV_BRANCH}" "${VERSION}" "${GIT_ORIGIN}"

release/ci/discord_template_notify_dev_private.json

@@ -0,0 +1,13 @@
+{
+  "embeds": [
+    {
+      "title": "❗️ Action Required for `${CI_COMMIT_REF_NAME}` ❗️",
+      "description": "[${CI_PROJECT_NAME}](${CI_PROJECT_URL}): Pipeline [#${CI_PIPELINE_ID}](${CI_PROJECT_URL}/-/pipelines/${CI_PIPELINE_ID}) of branch [${CI_COMMIT_REF_NAME}](${CI_PROJECT_URL}/-/commits/${CI_COMMIT_REF_NAME}) by ${GITLAB_USER_NAME} (${GITLAB_USER_LOGIN}) is ready to publish manually as [${NEW_BRANCH}](${PUBLIC_REPO_URL}/tree/${NEW_BRANCH})",
+      "color": 16763904,
+      "author": {
+        "name": "${GITLAB_USER_LOGIN}",
+        "icon_url": "${AVATAR_URL}"
+      }
+    }
+  ]
+}

release/ci/discord_template_notify_dev_public.json

@@ -0,0 +1,9 @@
+{
+  "embeds": [
+    {
+      "title": "🎉 sunnypilot `${NEW_BRANCH}` New Update 🎉",
+      "description": "[sunnypilot](${PUBLIC_REPO_URL}): Build #${EXTRA_VERSION_IDENTIFIER} of branch [${NEW_BRANCH}](${PUBLIC_REPO_URL}/tree/${NEW_BRANCH}) has been published.\n\nDrive safe! 🚗💨",
+      "color": 4321431
+    }
+  ]
+}

release/ci/install_gitlab_runner.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+set -e
+
+# Check if script arguments are present, if not exit the script
+if [ $# -eq 0 ]; then
+    echo "No arguments provided. A GitLab token is required to run this script."
+    exit 1
+fi
+
+# Constants
+GITLAB_RUNNER_DOWNLOAD_URL="https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64"
+GITLAB_RUNNER_USER_NAME="gitlab-runner"
+USER_GROUPS="comma,gpu,gpio,sudo"
+GITLAB_BASE_DIR="/data/gitlab"
+GITLAB_BIN_DIR="${GITLAB_BASE_DIR}/bin"
+GITLAB_BUILDS_DIR="${GITLAB_BASE_DIR}/builds"
+GITLAB_LOGS_DIR="${GITLAB_BASE_DIR}/logs"
+GITLAB_CACHE_DIR="${GITLAB_BASE_DIR}/cache"
+GITLAB_OPENPILOT_DIR="${GITLAB_BASE_DIR}/openpilot"
+SERVICE_NAME="gitlab-runner"
+
+create_gitlab_runner_directories() {
+    sudo mkdir -p "$GITLAB_BIN_DIR" "$GITLAB_BUILDS_DIR" "$GITLAB_LOGS_DIR" "$GITLAB_CACHE_DIR" "$GITLAB_OPENPILOT_DIR"
+    mkdir -p "/data/openpilot"
+    sudo chown -R comma:comma "/data/openpilot"
+}
+
+download_and_setup_gitlab_runner() {
+    sudo curl -L --output "$GITLAB_BIN_DIR/gitlab-runner" "$GITLAB_RUNNER_DOWNLOAD_URL"
+    sudo chmod +x "$GITLAB_BIN_DIR/gitlab-runner"
+}
+
+setup_gitlab_runner_user() {
+    sudo useradd --comment 'GitLab Runner' --create-home --home-dir ${GITLAB_BASE_DIR} ${GITLAB_RUNNER_USER_NAME} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${GITLAB_RUNNER_USER_NAME}
+    export GITLAB_BASE_DIR # Export it to make it available to sub-processes
+    sudo -u ${GITLAB_RUNNER_USER_NAME} bash -c "truncate -s 0 '${GITLAB_BASE_DIR}/.bash_logout'"
+}
+
+create_sudoers_entry() {
+    sudo grep -qxF "${GITLAB_RUNNER_USER_NAME} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${GITLAB_RUNNER_USER_NAME} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers
+}
+
+generate_gitlab_config_file() {
+    cat <<EOL | sudo tee "$GITLAB_BASE_DIR/config.toml"
+[[runners]]
+  name = "tici"
+  url = "https://gitlab.com/"
+  token = "$1"
+  executor = "shell"
+  builds_dir = "$GITLAB_BUILDS_DIR"
+  [runners.custom_build_dir]
+  [runners.docker]
+    volumes = ["$GITLAB_CACHE_DIR:/cache"]
+  [runners.cache]
+    MaxUploadedArchiveSize = 0
+  [runners.custom]
+    config_exec = "$GITLAB_LOGS_DIR"
+EOL
+}
+
+set_gitlab_directory_permissions() {
+    sudo chown -R ${GITLAB_RUNNER_USER_NAME}:comma "$GITLAB_BASE_DIR"
+    sudo chmod g+rwx "$GITLAB_BASE_DIR"
+    sudo chmod g+s "$GITLAB_BASE_DIR"
+}
+
+# Please note that when the gitlab runner is started, linux implicitly executes "source /etc/profile" for every logged shell. This will make scons available
+create_gitlab_runner_service() {
+    cat <<EOL | sudo tee /etc/systemd/system/${SERVICE_NAME}.service
+[Unit]
+Description=GitLab Runner
+After=syslog.target network.target
+ConditionFileIsExecutable=$GITLAB_BIN_DIR/gitlab-runner
+[Service]
+StartLimitInterval=5
+StartLimitBurst=10
+ExecStart=/usr/bin/unshare -m -- sh -c 'mount --bind $GITLAB_OPENPILOT_DIR /data/openpilot && exec $GITLAB_BIN_DIR/gitlab-runner "run" "--working-directory" "$GITLAB_BUILDS_DIR" "--config" "$GITLAB_BASE_DIR/config.toml" "--service" "gitlab-runner" "--syslog" "--user" "${GITLAB_RUNNER_USER_NAME}"'
+Restart=always
+RestartSec=120
+[Install]
+WantedBy=multi-user.target
+EOL
+}
+
+start_gitlab_runner_service() {
+    sudo systemctl daemon-reload
+    sudo systemctl disable gitlab-runner # Intentionally making sure the service is NOT enabled on boot.
+    sudo systemctl start gitlab-runner
+}
+
+# Make the filesystem writable
+sudo mount -o remount,rw /
+
+# Ensure filesystem is remounted as read-only on script exit
+trap "sudo mount -o remount,ro /" EXIT
+
+# Call functions
+setup_gitlab_runner_user
+create_sudoers_entry
+create_gitlab_runner_directories
+download_and_setup_gitlab_runner
+generate_gitlab_config_file "$1"
+set_gitlab_directory_permissions
+create_gitlab_runner_service
+start_gitlab_runner_service
+
+# End of install script

release/ci/publish.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+# Take parameters as arguments
+SOURCE_DIR=$1
+OUTPUT_DIR=$2
+DEV_BRANCH=$3
+VERSION=$4
+GIT_ORIGIN=$5
+EXTRA_VERSION_IDENTIFIER=$6
+
+# Check parameters
+if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then
+    echo "Error: No source or output directory provided."
+    exit 1
+fi
+
+if [ -z "$DEV_BRANCH" ] || [ -z "$VERSION" ]; then
+    echo "Error: No dev branch or version provided."
+    exit 1
+fi
+
+if [ -z "$GIT_ORIGIN" ]; then
+    echo "Error: No GIT_ORIGIN provided"
+    exit 1
+fi
+
+# "Tagging"
+echo "#define COMMA_VERSION \"$VERSION\"" > ${OUTPUT_DIR}/common/version.h
+
+## set git identity
+#source $DIR/identity.sh
+#export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+echo "[-] Setting up repo T=$SECONDS"
+cd $OUTPUT_DIR
+git init
+
+# set git username/password
+#source /data/identity.sh
+
+git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .git doesn't exist or not allowed to delete
+git remote remove origin || true # ensure cleanup
+git remote add origin $GIT_ORIGIN
+#git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out
+git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH)
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$DEV_BRANCH
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+# Add built files to git
+git add -f .
+if [ "$EXTRA_VERSION_IDENTIFIER" = "-release" ] || [ "$EXTRA_VERSION_IDENTIFIER" = "-staging" ]; then
+  export VERSION=${VERSION%"$EXTRA_VERSION_IDENTIFIER"}
+  git commit --amend -m "sunnypilot v$VERSION"
+else
+  git commit --amend -m "sunnypilot v$VERSION
+  version: sunnypilot v$SP_VERSION release
+  date: $DATETIME
+  master commit: $GIT_HASH
+  "
+fi
+git branch -m $DEV_BRANCH
+
+# Push!
+echo "[-] pushing T=$SECONDS"
+git push -f origin $DEV_BRANCH

release/ci/sanity_check.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# Check if the correct number of arguments are provided
+if [ "$#" -lt 2 ]; then
+  echo "Error: Missing required arguments."
+  echo "Usage: $0 \"ext1,ext2,...\" \"dir1,dir2,...\" [expect_no_match]"
+  exit 1
+fi
+
+# Convert comma-separated lists of extensions and directory patterns to arrays
+IFS=',' read -r -a file_extensions <<< "$1"
+IFS=',' read -r -a dir_patterns <<< "$2"
+
+# Boolean argument to determine the script's behavior when matches are found
+expect_no_match=${3:-true} # Default to true if not provided
+
+# Validate that file_extensions and dir_patterns are not empty
+if [ ${#file_extensions[@]} -eq 0 ]; then
+  echo "Error: No file extensions provided."
+  exit 1
+fi
+
+if [ ${#dir_patterns[@]} -eq 0 ]; then
+  echo "Error: No directory patterns provided."
+  exit 1
+fi
+
+# Define the command and arguments as an array
+find_command=(find . \( -false)
+
+for ext in "${file_extensions[@]}"; do
+  find_command+=(-o -type f -name "*$ext")
+done
+for dir in "${dir_patterns[@]}"; do
+  find_command+=(-o -type d -path "$dir")
+done
+
+find_command+=(\) -print)
+
+# Debug print
+echo "Executing find_command ${find_command[@]}"
+# Execute the find command
+FOUND=$("${find_command[@]}")
+
+# Function to handle found matches
+handle_found() {
+  local count=$(echo "$FOUND" | grep -c '^')
+  if [ "$expect_no_match" = true ]; then
+    echo "Failure: Unexpected match$( [ "$count" -gt 1 ] && echo "es" ) found ($count):"
+    echo "$FOUND"
+    exit 1
+  else
+    echo "Success: Expected match$( [ "$count" -gt 1 ] && echo "es" ) found ($count):"
+    echo "$FOUND"
+    exit 0
+  fi
+}
+
+# Function to handle no matches found
+handle_not_found() {
+  if [ "$expect_no_match" = false ]; then
+    echo "Failure: Expected matches not found."
+    exit 1
+  else
+    echo "Success: No matches found as expected."
+    exit 0
+  fi
+}
+
+# Check if any files or directories are found and handle accordingly
+if [ -n "$FOUND" ]; then
+  handle_found
+else
+  handle_not_found
+fi

release/ci/sync-lfs.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+mv .lfsconfig .lfsconfig.bak
+mv .lfsconfig-comma .lfsconfig
+git lfs fetch --all; git lfs pull
+mv .lfsconfig .lfsconfig-comma
+mv .lfsconfig.bak .lfsconfig
+git lfs fetch --all; git lfs push --all origin

release/ci/uninstall_gitlab_runner.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Define directories and user
+GITLAB_BASE_DIR="/data/gitlab"
+BIN_DIR="$GITLAB_BASE_DIR/bin"
+BUILDS_DIR="$GITLAB_BASE_DIR/builds"
+OPENPILOT_DIR="$GITLAB_BASE_DIR/openpilot"
+LOGS_DIR="$GITLAB_BASE_DIR/logs"
+CACHE_DIR="$GITLAB_BASE_DIR/cache"
+RUNNER_USERNAME="gitlab-runner"
+# Define the systemd service name
+SERVICE_NAME="gitlab-runner"
+USER_GROUPS="comma,gpu,gpio,sudo"
+
+# Function to stop and disable the systemd service
+stop_service() {
+    sudo systemctl stop ${SERVICE_NAME}
+    sudo systemctl disable ${SERVICE_NAME}
+}
+
+# Function to remove the systemd service file
+remove_service_file() {
+    sudo rm /etc/systemd/system/${SERVICE_NAME}.service
+    sudo systemctl daemon-reload
+}
+
+# Function to delete the GitLab Runner directories
+delete_directories() {
+    sudo rm -rf "$BIN_DIR/gitlab-runner"
+    sudo rm -rf "$GITLAB_BASE_DIR" "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR"
+}
+
+# Function to remove the GitLab Runner user
+delete_user() {
+    for group in ${USER_GROUPS//,/ }
+    do
+       sudo gpasswd -d ${RUNNER_USERNAME} ${group}
+    done
+    sudo userdel -r ${RUNNER_USERNAME}
+}
+
+# Function to remove sudoers entry
+remove_sudoers_entry() {
+    sudo sed -i.bak "/${RUNNER_USERNAME} ALL=(ALL) NOPASSWD: ALL/d" /etc/sudoers
+}
+
+# Make filesystem writable
+sudo mount -o remount rw /
+
+# Ensure filesystem is remounted as read-only on script exit
+trap "sudo mount -o remount ro /" EXIT
+
+# Call functions
+stop_service
+remove_service_file
+delete_directories
+delete_user
+remove_sudoers_entry
+# End of uninstall script

release/files_pc

@@ -0,0 +1,4 @@
+third_party/libyuv/x86_64/**
+third_party/snpe/x86_64/**
+third_party/snpe/x86_64-linux-clang/**
+third_party/acados/x86_64/**

release/files_tici

@@ -0,0 +1,15 @@
+third_party/libyuv/larch64/**
+third_party/snpe/larch64**
+third_party/snpe/aarch64-ubuntu-gcc7.5/*
+third_party/acados/larch64/**
+
+system/camerad/cameras/camera_qcom2.cc
+system/camerad/cameras/camera_qcom2.h
+system/camerad/cameras/camera_util.cc
+system/camerad/cameras/camera_util.h
+system/camerad/cameras/process_raw.cl
+
+system/qcomgpsd/*
+
+selfdrive/ui/qt/spinner_larch64
+selfdrive/ui/qt/text_larch64

release/identity.sh

@@ -1,4 +1,4 @@
-export GIT_COMMITTER_NAME="Vehicle Researcher"
-export GIT_COMMITTER_EMAIL="user@comma.ai"
-export GIT_AUTHOR_NAME="Vehicle Researcher"
-export GIT_AUTHOR_EMAIL="user@comma.ai"
+export GIT_COMMITTER_NAME="Jason Wen"
+export GIT_COMMITTER_EMAIL="haibin.wen3@gmail.com"
+export GIT_AUTHOR_NAME="Jason Wen"
+export GIT_AUTHOR_EMAIL="haibin.wen3@gmail.com"

release/release_files.py

@@ -10,14 +10,12 @@ ROOT = HERE + "/.."
 # - minimizing release download size
 # - keeping the diff readable
 blacklist = [
-  "body/STL/",
-
   "panda/drivers/",
   "panda/examples/",
   "panda/tests/safety/",
 
-  "opendbc/.*.dbc$",
-  "opendbc/generator/",
+  "opendbc_repo/dbc/.*.dbc$",
+  "opendbc_repo/dbc/generator/",
 
   "cereal/.*test.*",
   "^common/tests/",
@@ -28,7 +26,6 @@ blacklist = [
   "selfdrive/car/tests/test_models.*",
 
   "^tools/",
-  "^scripts/",
   "^tinygrad_repo/",
 
   "matlab.*.md",
@@ -37,7 +34,7 @@ blacklist = [
   ".github/",
   ".devcontainer/",
   "Darwin/",
-  ".vscode",
+  ".vscode/",
 
   # common things
   "LICENSE",
@@ -51,10 +48,52 @@ blacklist = [
   ".gitmodules",
 ]
 
+# Sunnypilot blacklist
+sunnypilot_blacklist = [
+  "system/loggerd/sunnylink_uploader.py",  # Temporarily, until we are ready to roll it out widely
+  "system/manager/gitlab_runner.sh",
+  ".idea/",
+  ".run/",
+  ".run/",
+  ".*__pycache__/.*",
+  ".*\.pyc",
+  "tinygrad/*",
+  "teleoprtc/*",
+  "third_party/snpe/x86_64/*",
+  "body/board/canloader.py",
+  "body/board/flash_base.sh",
+  "body/board/flash_knee.sh",
+  "body/board/recover.sh",
+  ".*/test/",
+  ".*/tests/",
+  ".*tinygrad_repo/tinygrad/renderer/",
+  "README.md",
+  ".*internal/",
+  "docs/.*",
+  ".sconsign.dblite",
+  "release/ci/scons_cache/",
+  ".gitlab-ci.yml",
+  ".clang-tidy",
+  ".dockerignore",
+  ".editorconfig",
+  ".gitmodules",
+  ".pre-commit-config.yaml",
+  ".python-version",
+  "Dockerfile",
+  "dockerfile",
+  "SECURITY.md",
+  "codecov.yml",
+  "conftest.py",
+  "poetry.lock",
+]
+
+# Merge the blacklists
+blacklist += sunnypilot_blacklist
+
 # gets you through the blacklist
 whitelist = [
-  "tools/lib/",
-  "tools/bodyteleop/",
+  "^tools/lib/(?!.*__pycache__).*$",
+  "tools/bodyteleop/(?!.*__pycache__).*$",
 
   "tinygrad_repo/openpilot/compile2.py",
   "tinygrad_repo/extra/onnx.py",
@@ -76,57 +115,98 @@ whitelist = [
   "tinygrad_repo/tinygrad/.*.py",
 
   # TODO: do this automatically
-  "opendbc/comma_body.dbc",
-  "opendbc/chrysler_ram_hd_generated.dbc",
-  "opendbc/chrysler_ram_dt_generated.dbc",
-  "opendbc/chrysler_pacifica_2017_hybrid_generated.dbc",
-  "opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc",
-  "opendbc/gm_global_a_powertrain_generated.dbc",
-  "opendbc/gm_global_a_object.dbc",
-  "opendbc/gm_global_a_chassis.dbc",
-  "opendbc/FORD_CADS.dbc",
-  "opendbc/ford_fusion_2018_adas.dbc",
-  "opendbc/ford_lincoln_base_pt.dbc",
-  "opendbc/honda_accord_2018_can_generated.dbc",
-  "opendbc/acura_ilx_2016_can_generated.dbc",
-  "opendbc/acura_rdx_2018_can_generated.dbc",
-  "opendbc/acura_rdx_2020_can_generated.dbc",
-  "opendbc/honda_civic_touring_2016_can_generated.dbc",
-  "opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc",
-  "opendbc/honda_crv_touring_2016_can_generated.dbc",
-  "opendbc/honda_crv_ex_2017_can_generated.dbc",
-  "opendbc/honda_crv_ex_2017_body_generated.dbc",
-  "opendbc/honda_crv_executive_2016_can_generated.dbc",
-  "opendbc/honda_fit_ex_2018_can_generated.dbc",
-  "opendbc/honda_odyssey_exl_2018_generated.dbc",
-  "opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc",
-  "opendbc/honda_insight_ex_2019_can_generated.dbc",
-  "opendbc/acura_ilx_2016_nidec.dbc",
-  "opendbc/honda_civic_ex_2022_can_generated.dbc",
-  "opendbc/hyundai_canfd.dbc",
-  "opendbc/hyundai_kia_generic.dbc",
-  "opendbc/hyundai_kia_mando_front_radar_generated.dbc",
-  "opendbc/mazda_2017.dbc",
-  "opendbc/nissan_x_trail_2017_generated.dbc",
-  "opendbc/nissan_leaf_2018_generated.dbc",
-  "opendbc/subaru_global_2017_generated.dbc",
-  "opendbc/subaru_global_2020_hybrid_generated.dbc",
-  "opendbc/subaru_outback_2015_generated.dbc",
-  "opendbc/subaru_outback_2019_generated.dbc",
-  "opendbc/subaru_forester_2017_generated.dbc",
-  "opendbc/toyota_tnga_k_pt_generated.dbc",
-  "opendbc/toyota_new_mc_pt_generated.dbc",
-  "opendbc/toyota_nodsu_pt_generated.dbc",
-  "opendbc/toyota_adas.dbc",
-  "opendbc/toyota_tss2_adas.dbc",
-  "opendbc/vw_golf_mk4.dbc",
-  "opendbc/vw_mqb_2010.dbc",
-  "opendbc/tesla_can.dbc",
-  "opendbc/tesla_radar_bosch_generated.dbc",
-  "opendbc/tesla_radar_continental_generated.dbc",
-  "opendbc/tesla_powertrain.dbc",
+  "opendbc_repo/dbc/comma_body.dbc",
+  "opendbc_repo/dbc/chrysler_ram_hd_generated.dbc",
+  "opendbc_repo/dbc/chrysler_ram_dt_generated.dbc",
+  "opendbc_repo/dbc/chrysler_pacifica_2017_hybrid_generated.dbc",
+  "opendbc_repo/dbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc",
+  "opendbc_repo/dbc/gm_global_a_powertrain_generated.dbc",
+  "opendbc_repo/dbc/gm_global_a_object.dbc",
+  "opendbc_repo/dbc/gm_global_a_chassis.dbc",
+  "opendbc_repo/dbc/FORD_CADS.dbc",
+  "opendbc_repo/dbc/ford_fusion_2018_adas.dbc",
+  "opendbc_repo/dbc/ford_lincoln_base_pt.dbc",
+  "opendbc_repo/dbc/honda_accord_2018_can_generated.dbc",
+  "opendbc_repo/dbc/acura_ilx_2016_can_generated.dbc",
+  "opendbc_repo/dbc/acura_rdx_2018_can_generated.dbc",
+  "opendbc_repo/dbc/acura_rdx_2020_can_generated.dbc",
+  "opendbc_repo/dbc/honda_civic_touring_2016_can_generated.dbc",
+  "opendbc_repo/dbc/honda_civic_hatchback_ex_2017_can_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_touring_2016_can_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_ex_2017_can_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_ex_2017_body_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_executive_2016_can_generated.dbc",
+  "opendbc_repo/dbc/honda_fit_ex_2018_can_generated.dbc",
+  "opendbc_repo/dbc/honda_odyssey_exl_2018_generated.dbc",
+  "opendbc_repo/dbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc",
+  "opendbc_repo/dbc/honda_insight_ex_2019_can_generated.dbc",
+  "opendbc_repo/dbc/acura_ilx_2016_nidec.dbc",
+  "opendbc_repo/dbc/honda_civic_ex_2022_can_generated.dbc",
+  "opendbc_repo/dbc/hyundai_canfd.dbc",
+  "opendbc_repo/dbc/hyundai_kia_generic.dbc",
+  "opendbc_repo/dbc/hyundai_kia_mando_front_radar_generated.dbc",
+  "opendbc_repo/dbc/mazda_2017.dbc",
+  "opendbc_repo/dbc/nissan_x_trail_2017_generated.dbc",
+  "opendbc_repo/dbc/nissan_leaf_2018_generated.dbc",
+  "opendbc_repo/dbc/subaru_global_2017_generated.dbc",
+  "opendbc_repo/dbc/subaru_global_2020_hybrid_generated.dbc",
+  "opendbc_repo/dbc/subaru_outback_2015_generated.dbc",
+  "opendbc_repo/dbc/subaru_outback_2019_generated.dbc",
+  "opendbc_repo/dbc/subaru_forester_2017_generated.dbc",
+  "opendbc_repo/dbc/toyota_tnga_k_pt_generated.dbc",
+  "opendbc_repo/dbc/toyota_new_mc_pt_generated.dbc",
+  "opendbc_repo/dbc/toyota_nodsu_pt_generated.dbc",
+  "opendbc_repo/dbc/toyota_adas.dbc",
+  "opendbc_repo/dbc/toyota_tss2_adas.dbc",
+  "opendbc_repo/dbc/vw_golf_mk4.dbc",
+  "opendbc_repo/dbc/vw_mqb_2010.dbc",
+  "opendbc_repo/dbc/tesla_can.dbc",
+  "opendbc_repo/dbc/tesla_radar_bosch_generated.dbc",
+  "opendbc_repo/dbc/tesla_radar_continental_generated.dbc",
+  "opendbc_repo/dbc/tesla_powertrain.dbc",
 ]
 
+# Sunnypilot whitelist
+sunnypilot_whitelist = [
+  "^README.md",
+  ".*selfdrive/test/fuzzy_generation.py",
+  ".*selfdrive/test/helpers.py",
+  ".*selfdrive/test/__init__.py",
+  ".*selfdrive/test/setup_device_ci.sh",
+  ".*selfdrive/test/test_time_to_onroad.py",
+  ".*selfdrive/test/test_onroad.py",
+  ".*system/manager/test/test_manager.py",
+  ".*system/manager/test/__init__.py",
+  ".*system/qcomgpsd/tests/test_qcomgpsd.py",
+  ".*system/updated/casync/tests/test_casync.py",
+  ".*system/updated/tests/test_git.py",
+  ".*system/updated/tests/test_base.py",
+  ".*tools/lib/tests/test_route_library.py",
+  ".*tools/lib/tests/test_caching.py",
+  ".*tools/lib/tests/test_logreader.py",
+  ".*tools/lib/tests/test_readers.py",
+  ".*tools/lib/tests/__init__.py",
+  ".*tools/lib/tests/test_comma_car_segments.py",
+  ".*selfdrive/ui/tests/test_translations.py",
+  ".*selfdrive/car/tests/__init__.py",
+  ".*selfdrive/car/tests/test_car_interfaces.py",
+  ".*selfdrive/navd/tests/test_navd.py",
+  ".*selfdrive/navd/tests/test_map_renderer.py",
+  ".*selfdrive/boardd/tests/test_boardd_loopback.py",
+  ".*tinygrad_repo/tinygrad/renderer/opencl.py",
+  ".*tinygrad_repo/tinygrad/renderer/cstyle.py",
+  ".*INTEGRATION.md",
+  ".*HOW-TOS.md",
+  ".*CARS.md",
+  ".*LIMITATIONS.md",
+  ".*CONTRIBUTING.md",
+  ".*sunnyhaibin0850_qrcode_paypal.me.png",
+  "opendbc/.*.dbc",
+]
+
+# Merge the whitelists
+whitelist += sunnypilot_whitelist
+
 
 if __name__ == "__main__":
   for f in Path(ROOT).rglob("**/*"):

release/sunnypilot/build_dev.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/bash -e
+
+# git diff --name-status origin/release3-staging | grep "^A" | less
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+
+BUILD_DIR=/data/media/openpilot-dev
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
+
+FILES_SRC="release/files_tici"
+DEV_BRANCH="dev-c3"
+
+BUILD_PANDA_DIR=/data/media/panda/openpilot-dev
+
+
+# set git identity
+source $DIR/identity.sh
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+echo "[-] Setting up repo T=$SECONDS"
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git init
+# set git username/password
+source /data/identity.sh
+git remote add origin https://github.com/sunnyhaibin/sunnypilot.git
+git fetch origin $DEV_BRANCH
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
+
+# in the directory
+cd $BUILD_DIR
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+rm -f panda/board/obj/bootstub.panda.bin
+rm -f panda/board/obj/bootstub.panda_h7.bin
+
+VERSION=$(date '+%Y.%m.%d')
+echo "#define COMMA_VERSION \"$VERSION-dev\"" > common/version.h
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$DEV_BRANCH
+
+# Build panda firmware
+pushd panda/
+scons -u .
+mv board/obj/panda.bin.signed $BUILD_PANDA_DIR/panda.bin.signed
+mv board/obj/panda_h7.bin.signed $BUILD_PANDA_DIR/panda_h7.bin.signed
+mv board/obj/bootstub.panda.bin $BUILD_PANDA_DIR/bootstub.panda.bin
+mv board/obj/bootstub.panda_h7.bin $BUILD_PANDA_DIR/bootstub.panda_h7.bin
+popd
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+scons -j$(nproc)
+
+# Ensure no submodules in release
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '*.cc' -delete
+find . -name '__pycache__' -delete
+find selfdrive/ui/ -name '*.h' -delete
+rm -rf panda/board panda/certs panda/crypto
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+rm -rf selfdrive/ui/replay/
+# Move back signed panda fw
+mkdir -p panda/board/obj
+mv $BUILD_PANDA_DIR/panda.bin.signed panda/board/obj/panda.bin.signed
+mv $BUILD_PANDA_DIR/panda_h7.bin.signed panda/board/obj/panda_h7.bin.signed
+mv $BUILD_PANDA_DIR/bootstub.panda.bin panda/board/obj/bootstub.panda.bin
+mv $BUILD_PANDA_DIR/bootstub.panda_h7.bin panda/board/obj/bootstub.panda_h7.bin
+
+# Restore third_party
+git checkout third_party/
+
+# Mark as prebuilt release
+touch prebuilt
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+# Add built files to git
+git add -f .
+git commit --amend -m "sunnypilot v$VERSION
+version: sunnypilot v$SP_VERSION release
+date: $DATETIME
+master commit: $GIT_HASH
+"
+git branch -m dev-c3
+
+# Run tests
+#TEST_FILES="tools/"
+#cd $SOURCE_DIR
+#cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+#cd $BUILD_DIR
+#RELEASE=1 selfdrive/test/test_onroad.py
+#selfdrive/manager/test/test_manager.py
+#selfdrive/car/tests/test_car_interfaces.py
+#rm -rf $TEST_FILES
+
+if [ ! -z "$PUSH" ]; then
+  echo "[-] pushing T=$SECONDS"
+  git push -f origin $DEV_BRANCH
+fi
+
+echo "[-] done T=$SECONDS"

release/sunnypilot/build_release.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/bash -e
+
+# git diff --name-status origin/release3-staging | grep "^A" | less
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+
+BUILD_DIR=/data/media/openpilot-release
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
+
+FILES_SRC="release/files_tici"
+RELEASE_BRANCH="release-c3"
+
+BUILD_PANDA_DIR=/data/media/panda/openpilot-release
+
+
+# set git identity
+source $DIR/identity.sh
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+echo "[-] Setting up repo T=$SECONDS"
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git init
+# set git username/password
+source /data/identity.sh
+git remote add origin https://github.com/sunnyhaibin/sunnypilot.git
+git fetch origin $RELEASE_BRANCH
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
+
+# in the directory
+cd $BUILD_DIR
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+rm -f panda/board/obj/bootstub.panda.bin
+rm -f panda/board/obj/bootstub.panda_h7.bin
+
+VERSION=$(cat common/version.h | awk -F[\"-]  '{print $2}')
+echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$RELEASE_BRANCH
+
+# Build panda firmware
+pushd panda/
+scons -u .
+mv board/obj/panda.bin.signed $BUILD_PANDA_DIR/panda.bin.signed
+mv board/obj/panda_h7.bin.signed $BUILD_PANDA_DIR/panda_h7.bin.signed
+mv board/obj/bootstub.panda.bin $BUILD_PANDA_DIR/bootstub.panda.bin
+mv board/obj/bootstub.panda_h7.bin $BUILD_PANDA_DIR/bootstub.panda_h7.bin
+popd
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+scons -j$(nproc)
+
+# Ensure no submodules in release
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '*.cc' -delete
+find . -name '__pycache__' -delete
+find selfdrive/ui/ -name '*.h' -delete
+rm -rf panda/board panda/certs panda/crypto
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+#rm models/supercombo_badweights.thneed
+rm -rf selfdrive/ui/replay/
+# Move back signed panda fw
+mkdir -p panda/board/obj
+mv $BUILD_PANDA_DIR/panda.bin.signed panda/board/obj/panda.bin.signed
+mv $BUILD_PANDA_DIR/panda_h7.bin.signed panda/board/obj/panda_h7.bin.signed
+mv $BUILD_PANDA_DIR/bootstub.panda.bin panda/board/obj/bootstub.panda.bin
+mv $BUILD_PANDA_DIR/bootstub.panda_h7.bin panda/board/obj/bootstub.panda_h7.bin
+
+# Restore third_party
+git checkout third_party/
+
+# Mark as prebuilt release
+touch prebuilt
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+# Add built files to git
+git add -f .
+git commit --amend -m "sunnypilot v$VERSION"
+git branch -m release-c3
+
+# Run tests
+#TEST_FILES="tools/"
+#cd $SOURCE_DIR
+#cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+#cd $BUILD_DIR
+#RELEASE=1 selfdrive/test/test_onroad.py
+#selfdrive/manager/test/test_manager.py
+#selfdrive/car/tests/test_car_interfaces.py
+#rm -rf $TEST_FILES
+
+if [ ! -z "$PUSH" ]; then
+  echo "[-] pushing T=$SECONDS"
+  git push -f origin $RELEASE_BRANCH
+fi
+
+echo "[-] done T=$SECONDS"

release/sunnypilot/build_staging.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/bash -e
+
+# git diff --name-status origin/release3-staging | grep "^A" | less
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+
+BUILD_DIR=/data/media/openpilot-staging
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
+
+FILES_SRC="release/files_tici"
+STAGING_BRANCH="staging-c3"
+
+BUILD_PANDA_DIR=/data/media/panda/openpilot-staging
+
+
+# set git identity
+source $DIR/identity.sh
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+echo "[-] Setting up repo T=$SECONDS"
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git init
+# set git username/password
+source /data/identity.sh
+git remote add origin https://github.com/sunnyhaibin/sunnypilot.git
+git fetch origin $STAGING_BRANCH
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
+
+# in the directory
+cd $BUILD_DIR
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+rm -f panda/board/obj/bootstub.panda.bin
+rm -f panda/board/obj/bootstub.panda_h7.bin
+
+VERSION=$(cat common/version.h | awk -F[\"-]  '{print $2}')
+echo "#define COMMA_VERSION \"$VERSION-staging\"" > common/version.h
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "sunnypilot v$VERSION release"
+git branch --set-upstream-to=origin/$STAGING_BRANCH
+
+# Build panda firmware
+pushd panda/
+scons -u .
+mv board/obj/panda.bin.signed $BUILD_PANDA_DIR/panda.bin.signed
+mv board/obj/panda_h7.bin.signed $BUILD_PANDA_DIR/panda_h7.bin.signed
+mv board/obj/bootstub.panda.bin $BUILD_PANDA_DIR/bootstub.panda.bin
+mv board/obj/bootstub.panda_h7.bin $BUILD_PANDA_DIR/bootstub.panda_h7.bin
+popd
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+scons -j$(nproc)
+
+# Ensure no submodules in release
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '*.cc' -delete
+find . -name '__pycache__' -delete
+find selfdrive/ui/ -name '*.h' -delete
+rm -rf panda/board panda/certs panda/crypto
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+#rm models/supercombo_badweights.thneed
+rm -rf selfdrive/ui/replay/
+# Move back signed panda fw
+mkdir -p panda/board/obj
+mv $BUILD_PANDA_DIR/panda.bin.signed panda/board/obj/panda.bin.signed
+mv $BUILD_PANDA_DIR/panda_h7.bin.signed panda/board/obj/panda_h7.bin.signed
+mv $BUILD_PANDA_DIR/bootstub.panda.bin panda/board/obj/bootstub.panda.bin
+mv $BUILD_PANDA_DIR/bootstub.panda_h7.bin panda/board/obj/bootstub.panda_h7.bin
+
+# Restore third_party
+git checkout third_party/
+
+# Mark as prebuilt release
+touch prebuilt
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+# Add built files to git
+git add -f .
+git commit --amend -m "sunnypilot v$VERSION"
+git branch -m staging-c3
+
+# Run tests
+#TEST_FILES="tools/"
+#cd $SOURCE_DIR
+#cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+#cd $BUILD_DIR
+#RELEASE=1 selfdrive/test/test_onroad.py
+#selfdrive/manager/test/test_manager.py
+#selfdrive/car/tests/test_car_interfaces.py
+#rm -rf $TEST_FILES
+
+if [ ! -z "$PUSH" ]; then
+  echo "[-] pushing T=$SECONDS"
+  git push -f origin $STAGING_BRANCH
+fi
+
+echo "[-] done T=$SECONDS"

release/sunnypilot/identity.sh

@@ -0,0 +1,4 @@
+export GIT_COMMITTER_NAME="Jason Wen"
+export GIT_COMMITTER_EMAIL="haibin.wen3@gmail.com"
+export GIT_AUTHOR_NAME="Jason Wen"
+export GIT_AUTHOR_EMAIL="haibin.wen3@gmail.com"

scripts/git_rewrite/rewrite-git-history.sh

@@ -0,0 +1,390 @@
+#!/bin/bash -e
+
+SRC=/tmp/openpilot/
+SRC_CLONE=/tmp/openpilot-clone/
+OUT=/tmp/openpilot-tiny/
+
+REWRITE_IGNORE_BRANCHES=(
+  dashcam3
+  devel
+  master-ci
+  nightly
+  release2
+  release3
+  release3-staging
+)
+
+VALIDATE_IGNORE_FILES=(
+  ".github/ISSUE_TEMPLATE/bug_report.md"
+  ".github/pull_request_template.md"
+)
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+LOGS_DIR=$DIR/git-rewrite-$(date +"%Y-%m-%dT%H:%M:%S%z")
+mkdir -p $LOGS_DIR
+
+GIT_REWRITE_LOG=$LOGS_DIR/git-rewrite-log.txt
+BRANCH_DIFF_LOG=$LOGS_DIR/branch-diff-log.txt
+COMMIT_DIFF_LOG=$LOGS_DIR/commit-diff-log.txt
+
+START_TIME=$(date +%s)
+exec > >(while IFS= read -r line; do
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  echo "[${ELAPSED_TIME}s] $line"
+done | tee -a "$GIT_REWRITE_LOG") 2>&1
+
+# INSTALL git-filter-repo
+if [ ! -f /tmp/git-filter-repo ]; then
+  echo "Installing git-filter-repo..."
+  curl -sSo /tmp/git-filter-repo https://raw.githubusercontent.com/newren/git-filter-repo/main/git-filter-repo
+  chmod +x /tmp/git-filter-repo
+fi
+
+# MIRROR openpilot
+if [ ! -d $SRC ]; then
+  echo "Mirroring openpilot..."
+  git clone --mirror https://github.com/commaai/openpilot.git $SRC # 4.18 GiB (488034 objects)
+
+  cd $SRC
+
+  echo "Starting size $(du -sh .)"
+
+  git remote update
+
+  # the git-filter-repo analysis is bliss - can be found in the repo root/filter-repo/analysis
+  echo "Analyzing with git-filter-repo..."
+  /tmp/git-filter-repo --force --analyze
+
+  echo "Pushing to openpilot-archive..."
+  # push to archive repo - in smaller parts because the 2 GB push limit - https://docs.github.com/en/get-started/using-git/troubleshooting-the-2-gb-push-limit
+  ARCHIVE_REPO=git@github.com:commaai/openpilot-archive.git
+  git push --prune $ARCHIVE_REPO +refs/heads/master:refs/heads/master # push master first so it's the default branch (when openpilot-archive is an empty repo)
+  git push --prune $ARCHIVE_REPO +refs/heads/*:refs/heads/* # 956.39 MiB (110725 objects)
+  git push --prune $ARCHIVE_REPO +refs/tags/*:refs/tags/* # 1.75 GiB (21694 objects)
+  # git push --mirror $ARCHIVE_REPO || true # fails to push refs/pull/* (deny updating a hidden ref) for pull requests
+  # we fail and continue - more reading: https://stackoverflow.com/a/34266401/639708 and https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
+fi
+
+# REWRITE master and tags
+if [ ! -d $SRC_CLONE ]; then
+  echo "Cloning $SRC..."
+  GIT_LFS_SKIP_SMUDGE=1 git clone $SRC $SRC_CLONE
+
+  cd $SRC_CLONE
+
+  echo "Checking out old history..."
+
+  git checkout tags/v0.7.1 > /dev/null 2>&1
+  # checkout as main, since we need master ref later
+  git checkout -b main
+
+  echo "Creating setup commits..."
+
+  # rm these so we don't get conflicts later
+  git rm -r cereal opendbc panda selfdrive/ui/ui > /dev/null
+  git commit -m "removed conflicting files" > /dev/null
+
+  # skip-smudge to get rid of some lfs errors that it can't find the reference of some lfs files
+  # we don't care about fetching/pushing lfs right now
+  git lfs install --skip-smudge --local
+
+  # squash initial setup commits
+  git cherry-pick -n -X theirs 6c33a5c..59b3d06 > /dev/null
+  git commit -m "switching to master" > /dev/null
+
+  # squash the two commits
+  git reset --soft HEAD~2
+  git commit -m "switching to master" -m "$(git log --reverse --format=%B 6c33a5c..59b3d06)" -m "removed conflicting files" > /dev/null
+
+  # get commits we want to cherry-pick
+  # will start with the next commit after #59b3d06 tools is local now
+  COMMITS=$(git rev-list --reverse 59b3d06..master)
+
+  # we need this for logging
+  TOTAL_COMMITS=$(echo $COMMITS | wc -w | xargs)
+  CURRENT_COMMIT_NUMBER=0
+
+  # empty this file
+  > commit-map.txt
+
+  echo "Rewriting master commits..."
+
+  for COMMIT in $COMMITS; do
+      CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
+      # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"\\r
+      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"
+
+      # set environment variables to preserve author/committer and dates
+      export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
+      export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
+      export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
+      export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
+      export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
+      export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
+
+      # cherry-pick the commit
+      if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
+        # check if the failure is because of an empty commit
+        if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
+          echo "Empty commit detected. Skipping commit $COMMIT"
+          git cherry-pick --skip
+          # log it was empty to the mapping file
+          echo "$COMMIT EMPTY" >> commit-map.txt
+        else
+          # handle other errors or conflicts
+          echo "Cherry-pick failed. Handling error..."
+          echo "$GIT_OUTPUT"
+          exit 1
+        fi
+      else
+        # capture the new commit hash
+        NEW_COMMIT=$(git rev-parse HEAD)
+
+        # save the old and new commit hashes to the mapping file
+        echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
+
+        # append the old commit ID to the commit message
+        git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
+      fi
+
+    # prune every 3000 commits to avoid gc errors
+    if [ $((CURRENT_COMMIT_NUMBER % 3000)) -eq 0 ]; then
+      echo "Pruning repo..."
+      git gc
+    fi
+  done
+
+  echo "Rewriting tags..."
+
+  # remove all old tags
+  git tag -l | xargs git tag -d
+
+  # read each line from the tag-commit-map.txt
+  while IFS=' ' read -r TAG OLD_COMMIT; do
+    # search for the new commit in commit-map.txt corresponding to the old commit
+    NEW_COMMIT=$(grep "^$OLD_COMMIT " "commit-map.txt" | awk '{print $2}')
+
+    # check if this is a rebased commit
+    if [ -z "$NEW_COMMIT" ]; then
+      # if not, then just use old commit hash
+      NEW_COMMIT=$OLD_COMMIT
+    fi
+
+    echo "Rewriting tag $TAG from commit $NEW_COMMIT"
+    git tag -f "$TAG" "$NEW_COMMIT"
+  done < "$DIR/tag-commit-map.txt"
+
+  # uninstall lfs since we don't want to touch (push to) lfs right now
+  # git push will also push lfs, if we don't uninstall (--local so just for this repo)
+  git lfs uninstall --local
+
+  # force push new master
+  git push --force origin main:master
+
+  # force push new tags
+  git push --force --tags
+fi
+
+# REWRITE branches based on master
+if [ ! -f "$SRC_CLONE/rewrite-branches-done" ]; then
+  cd $SRC_CLONE
+  > rewrite-branches-done
+
+  # empty file
+  > $BRANCH_DIFF_LOG
+
+  echo "Rewriting branches based on master..."
+
+  # will store raw diffs here, if exist
+  mkdir -p differences
+
+  # get a list of all branches except master and REWRITE_IGNORE_BRANCHES
+  BRANCHES=$(git branch -r | grep -v ' -> ' | sed 's/.*origin\///' | grep -v '^master$' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n'))
+
+  for BRANCH in $BRANCHES; do
+    # check if the branch is based on master history
+    MERGE_BASE=$(git merge-base master origin/$BRANCH) || true
+    if [ -n "$MERGE_BASE" ]; then
+      echo "Rewriting branch: $BRANCH"
+      
+      # create a new branch based on the new master
+      NEW_MERGE_BASE=$(grep "^$MERGE_BASE " "commit-map.txt" | awk '{print $2}')
+      if [ -z "$NEW_MERGE_BASE" ]; then
+        echo "Error: could not find new merge base for branch $BRANCH" >> $BRANCH_DIFF_LOG
+        continue
+      fi
+      git checkout -b ${BRANCH}_new $NEW_MERGE_BASE
+      
+      # get the range of commits unique to this branch
+      COMMITS=$(git rev-list --reverse $MERGE_BASE..origin/${BRANCH})
+      
+      HAS_ERROR=0
+
+      # simple delimiter
+      echo "BRANCH ${BRANCH}" >> commit-map.txt
+
+      for COMMIT in $COMMITS; do
+        # set environment variables to preserve author/committer and dates
+        export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
+        export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
+        export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
+        export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
+        export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
+        export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
+
+        # cherry-pick the commit
+        if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
+          # check if the failure is because of an empty commit
+          if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
+            echo "Empty commit detected. Skipping commit $COMMIT"
+            git cherry-pick --skip
+            # log it was empty to the mapping file
+            echo "$COMMIT EMPTY" >> commit-map.txt
+          else
+            # handle other errors or conflicts
+            echo "Cherry-pick of ${BRANCH} branch failed. Removing branch upstream..." >> $BRANCH_DIFF_LOG
+            echo "$GIT_OUTPUT" > "$LOGS_DIR/branch-${BRANCH}"
+            git cherry-pick --abort
+            git push --delete origin ${BRANCH}
+            HAS_ERROR=1
+            break
+          fi
+        else
+          # capture the new commit hash
+          NEW_COMMIT=$(git rev-parse HEAD)
+
+          # save the old and new commit hashes to the mapping file
+          echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
+
+          # append the old commit ID to the commit message
+          git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
+        fi
+      done
+      
+      # force push the new branch
+      if [ $HAS_ERROR -eq 0 ]; then
+        # git lfs goes haywire here, so we need to install and uninstall
+        # git lfs install --skip-smudge --local
+        git lfs uninstall --local > /dev/null
+        git push -f origin ${BRANCH}_new:${BRANCH}
+      fi
+      
+      # clean up local branch
+      git checkout master > /dev/null
+      git branch -D ${BRANCH}_new > /dev/null
+    else
+      echo "Deleting branch $BRANCH as it's not based on master history" >> $BRANCH_DIFF_LOG
+      git push --delete origin ${BRANCH}
+    fi
+  done
+fi
+
+# VALIDATE cherry-pick
+if [ ! -f "$SRC_CLONE/validation-done" ]; then
+  cd $SRC_CLONE
+  > validation-done
+
+  TOTAL_COMMITS=$(grep -cve '^\s*$' commit-map.txt)
+  CURRENT_COMMIT_NUMBER=0
+  COUNT_SAME=0
+  COUNT_DIFF=0
+
+  # empty file
+  > $COMMIT_DIFF_LOG
+
+  echo "Validating commits..."
+
+  # will store raw diffs here, if exist
+  mkdir -p differences
+
+  # read each line from commit-map.txt
+  while IFS=' ' read -r OLD_COMMIT NEW_COMMIT; do
+    if [ "$NEW_COMMIT" == "EMPTY" ]; then
+      continue
+    fi
+    if [ "$OLD_COMMIT" == "BRANCH" ]; then
+      echo "Branch ${NEW_COMMIT} below:" >> $COMMIT_DIFF_LOG
+      continue
+    fi
+    CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
+    # retrieve short hashes and dates for the old and new commits
+    OLD_COMMIT_SHORT=$(git rev-parse --short $OLD_COMMIT)
+    NEW_COMMIT_SHORT=$(git rev-parse --short $NEW_COMMIT)
+    OLD_DATE=$(git show -s --format='%cd' $OLD_COMMIT)
+    NEW_DATE=$(git show -s --format='%cd' $NEW_COMMIT)
+
+    # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"\\r
+    echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"
+    
+    # generate lists of files and their hashes for the old and new commits, excluding ignored files
+    OLD_FILES=$(git ls-tree -r $OLD_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
+    NEW_FILES=$(git ls-tree -r $NEW_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
+
+    # Compare the diffs
+    if diff <(echo "$OLD_FILES") <(echo "$NEW_FILES") > /dev/null; then
+      # echo "Old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT are equivalent."
+      COUNT_SAME=$((COUNT_SAME + 1))
+    else
+      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Difference found between old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT" >> $COMMIT_DIFF_LOG
+      COUNT_DIFF=$((COUNT_DIFF + 1))
+      set +e
+      diff -u <(echo "$OLD_FILES") <(echo "$NEW_FILES") > "$LOGS_DIR/commit-$CURRENT_COMMIT_NUMBER-$OLD_COMMIT_SHORT-$NEW_COMMIT_SHORT"
+      set -e
+    fi
+  done < "commit-map.txt"
+
+  echo "Summary:" >> $COMMIT_DIFF_LOG
+  echo "Equivalent commits: $COUNT_SAME" >> $COMMIT_DIFF_LOG
+  echo "Different commits: $COUNT_DIFF" >> $COMMIT_DIFF_LOG
+fi
+
+if [ ! -d $OUT ]; then
+  cp -r $SRC $OUT
+
+  cd $OUT
+
+  # remove all non-master branches
+  # git branch | grep -v "^  master$" | grep -v "\*" | xargs git branch -D
+
+  # echo "cleaning up refs"
+  # delete pull request refs since we can't alter them anyway (https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#error-failed-to-push-some-refs)
+  # git for-each-ref --format='%(refname)' | grep '^refs/pull/' | xargs -I {} git update-ref -d {}
+
+  echo "importing new lfs files"
+  # import "almost" everything to lfs
+  BRANCHES=$(git for-each-ref --format='%(refname)' refs/heads/ | sed 's%refs/heads/%%g' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n') | tr '\n' ' ')
+  git lfs migrate import --include="*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,selfdrive/car/tests/test_models_segs.txt,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,flask/**/*,panda/**/*,board/**/*,messaging/**/*,opendbc/**/*,tools/cabana/chartswidget.cc,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,selfdrive/ui/paint.cc,werkzeug/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,selfdrive/locationd/laikad.py,selfdrive/locationd/test/test_laikad.py,tools/gpstest/test_laikad.py,selfdrive/locationd/laikad_helpers.py,tools/nui/**/*,jsonrpc/**/*,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,selfdrive/camerad/cameras/camera_qcom.cc,selfdrive/manager.py,selfdrive/modeld/models/driving.cc,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,gunicorn/**/*,*.qm,jinja2/**/*,click/**/*,dbcs/**/*,websocket/**/*" $BRANCHES
+
+  echo "reflog and gc"
+  # this is needed after lfs import
+  git reflog expire --expire=now --all
+  git gc --prune=now --aggressive
+
+  # check the git-filter-repo analysis again - can be found in the repo root/filter-repo/analysis
+  echo "Analyzing with git-filter-repo..."
+  /tmp/git-filter-repo --force --analyze
+
+  echo "New size is $(du -sh .)"
+fi
+
+cd $OUT
+
+# fetch all lfs files from https://github.com/commaai/openpilot.git
+# some lfs files are missing on gitlab, but they can be found on github
+git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
+git config lfs.pushurl ssh://git@github.com/commaai/openpilot.git
+git lfs fetch --all || true
+
+# also fetch all lfs files from https://gitlab.com/commaai/openpilot-lfs.git
+git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
+git lfs fetch --all || true
+
+# final push - will also push lfs
+# TODO: switch to git@github.com:commaai/openpilot.git when ready
+# git push --mirror git@github.com:commaai/openpilot-tiny.git
+# using this instead to ignore refs/pull/* - since this is also what --mirror does - https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
+git push --prune git@github.com:commaai/openpilot-tiny.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/tag-commit-map.txt

@@ -0,0 +1,82 @@
+v0.1 e94a30bec07e719c5a7b037ca1f4db8312702cce
+v0.2 449b482cc3236ccf31829830b4f6a44b2dcc06c2
+v0.2.1 17d9becd3c673091b22f09aa02559a9ed9230f50
+v0.2.2 a64b9aa9b8cb5863c917b6926516291a63c02fe5
+v0.2.3 adaa4ed350acda4067fc0b455ad15b54cdf4c768
+v0.2.4 ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0
+v0.2.5 29c58b45882ac79595356caf98580c1d2a626011
+v0.2.6 6c3afeec0fb439070b2912978b8dbb659033b1d9
+v0.2.7 c6ba5dc5391d3ca6cda479bf1923b88ce45509a0
+v0.2.8 95a349abcc050712c50d4d85a1c8a804eee7f6c2
+v0.2.9 693bcb0f83478f2651db6bac9be5ca5ad60d03f3
+v0.3.0 c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8
+v0.3.1 41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24
+v0.3.2 7fe46f1e1df5dec08a940451ba0feefd5c039165
+v0.3.3 5cf91d0496688fed4f2a6c7021349b1fc0e057a2
+v0.3.4 1b8c44b5067525a5d266b6e99799d8097da76a29
+v0.3.5 b111277f464cf66fa34b67819a83ea683e0f64df
+v0.4.0.2 da52d065a4c4f52d6017a537f3a80326f5af8bdc
+v0.4.1 4474b9b3718653aeb0aee26422caefb90460cc0e
+v0.4.2 28c0797d30175043bbfa31307b63aab4197cf996
+v0.4.4 9a9ff839a9b70cb2601d7696af743f5652395389
+v0.4.5 37285038d3f91fa1b49159c4a35a8383168e644f
+v0.4.6 c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1
+v0.4.7 ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9
+v0.5 de33bc46452b1046387ee2b3a03191b2c71135fb
+v0.5.1 8f22f52235c48eada586795ac57edb22688e4d08
+v0.5.2 0129a8a4ff8da5314e8e4d4d3336e89667ff6d54
+v0.5.3 285c52eb693265a0a530543e9ca0aeb593a2a55e
+v0.5.4 a422246dc30bce11e970514f13f7c110f4470cc3
+v0.5.5 8f3539a27b28851153454eb737da9624cccaed2d
+v0.5.6 860a48765d1016ba226fb2c64aea35a45fe40e4a
+v0.5.7 9ce3045f139ee29bf0eea5ec59dfe7df9c3d2c51
+v0.5.8 2cee2e05ba0f3824fdbb8b957958800fa99071a1
+v0.5.9 ad145da3bcded0fe75306df02061d07a633963c3
+v0.5.10 ff4c1557d8358f158f4358788ff18ef93d2470ef
+v0.5.11 d1866845df423c6855e2b365ff230cf7d89a420b
+v0.5.12 f6e8ef27546e9a406724841e75f8df71cc4c2c97
+v0.5.13 dd34ccfe288ebda8e2568cf550994ae890379f45
+v0.6 60a20537c5f3fcc7f11946d81aebc8f90c08c117
+v0.6.1 cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22
+v0.6.2 095ef5f9f60fca1b269aabcc3cfd322b17b9e674
+v0.6.3 d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3
+v0.6.4 58f376002e0c654fbc2de127765fa297cf694a33
+v0.6.5 70d17cd69b80e7627dcad8fd5b6438f2309ac307
+v0.6.6 d4eb5a6eafdd4803d09e6f3963918216cca5a81f
+v0.7 a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3
+v0.7.1 1e1de64a1e59476b7b3d3558b92149246d5c3292
+v0.7.2 59bd58c940673b4c4a6a86f299022614bcf42b22
+v0.7.3 d7acd8b68f8131e0e714400cf124a3e228638643
+v0.7.4 e93649882c5e914eec4a8b8b593dc0587e497033
+v0.7.5 8abc0afe464626a461d2c7e192c912eeebeccc65
+v0.7.6 69aacd9d179fe6dd3110253a099c38b34cff7899
+v0.7.7 f1caed7299cdba5e45635d8377da6cc1e5fd7072
+v0.7.8 2189fe8741b635d8394d55dee28959425cfd5ad0
+v0.7.9 86dc54b836a973f132ed26db9f5a60b29f9b25b2
+v0.7.10 47a42ff432db8a2494e922ca5e767e58020f0446
+v0.7.11 f46ed718ba8d6bb4d42cd7b0f0150c406017c373
+v0.8 d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc
+v0.8.1 cd6f26664cb8d32a13847d6648567c47c580e248
+v0.8.2 7cc0999aebfe63b6bb6dd83c1dff62c3915c4820
+v0.8.3 986500fe2f10870018f1fba1e5465476b8915977
+v0.8.4 f0d0b82b8d6f5f450952113e234d0a5a49e80c48
+v0.8.5 f5d9ddc6c2a2802a61e5ce590c6b6688bf736a69
+v0.8.6 75904ed7452c6cbfb2a70cd379a899d8a75b97c2
+v0.8.7 4f9e568019492126e236da85b5ca0a059f292900
+v0.8.8 a949a49d5efaaf2d881143d23e9fb5ff9e28e88c
+v0.8.9 a034926264cd1025c69d6ceb3fe444965f960b75
+v0.8.10 59accdd814398b884167c0f41dbf46dcccf0c29c
+v0.8.11 d630ec9092f039cb5e51c5dd6d92fc47b91407e4
+v0.8.12 57871c99031cf597ffa0d819057ac1401e129f32
+v0.8.13 e43e6e876513450d235124fcb711f1724ed9814c
+v0.8.14 71901c94dbbaa2f9f156a80c14cc7ea65219fc7c
+v0.8.15 5a7c2f90361e72e9c35e88abd2e11acdc4aba354
+v0.8.16 f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e
+v0.9.0 58b84fb401a804967aa0dd5ee66fafa90194fd30
+v0.9.1 89f68bf0cbf53a81b0553d3816fdbe522f941fa1
+v0.9.2 c7d3b28b93faa6c955fb24bc64031512ee985ee9
+v0.9.3 8704c1ff952b5c85a44f50143bbd1a4f7b4887e2
+v0.9.4 fa310d9e2542cf497d92f007baec8fd751ffa99c
+v0.9.5 3b1e9017c560499786d8a0e46aaaeea65037acac
+v0.9.6 0b4d08fab8e35a264bc7383e878538f8083c33e5
+v0.9.7 f8cb04e4a8b032b72a909f68b808a50936184bee

scripts/lint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+cd $DIR/../
+
+# TODO: bring back rest of pre-commit checks:
+# https://github.com/commaai/openpilot/blob/4b11c9e914707df9def598616995be2a5d355a6a/.pre-commit-config.yaml#L2
+
+ruff check .