Merge branch 'master' into navigationd-init

This reverts commit 3bbb33f6bd.

.gitattributes

@@ -11,4 +11,13 @@
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
+third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
+third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,8 @@
+---
+name: Enhancement
+about: For openpilot enhancement suggestions
+title: ''
+labels: 'enhancement'
+assignees: ''
+---
+

.github/workflows/auto-cache/action.yaml

@@ -0,0 +1,58 @@
+name: 'automatically cache based on current runner'
+
+inputs:
+  path:
+    description: 'path to cache'
+    required: true
+  key:
+    description: 'key'
+    required: true
+  restore-keys:
+    description: 'restore-keys'
+    required: true
+  save:
+    description: 'whether to save the cache'
+    default: 'true'
+    required: false
+outputs:
+  cache-hit:
+    description: 'cache hit occurred'
+    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
+               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
+               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: setup namespace cache
+      id: ns-cache
+      if: ${{ contains(runner.name, 'nsc') }}
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        path: ${{ inputs.path }}
+
+    - name: setup github cache
+      id: gha-cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
+      uses: 'actions/cache@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: setup github cache
+      id: gha-cache-ro
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
+      uses: 'actions/cache/restore@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.path }}
+        sudo chmod -R 777 ${{ inputs.path }}
+        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -12,12 +12,12 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v6
+    - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -55,13 +55,13 @@ jobs:
               repo: context.repo.repo,
               issue_number: prNumber
             });
-
+            
             const hasDevC3Label = labels.some(label => label.name === process.env.PR_LABEL);
             const hasTrustLabel = labels.some(label => label.name === process.env.TRUST_FORK_PR_LABEL);
-
+            
             console.log(`PR #${prNumber} has ${process.env.PR_LABEL} label: ${hasDevC3Label}`);
             console.log(`PR #${prNumber} has ${process.env.TRUST_FORK_PR_LABEL} label: ${hasTrustLabel}`);
-
+            
             core.setOutput('has-dev', hasDevC3Label ? 'true' : 'false');
             core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
 
@@ -81,7 +81,7 @@ jobs:
             });
 
             console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
-
+            
             // Add a comment to the PR
             await github.rest.issues.createComment({
               owner: context.repo.owner,

.github/workflows/badges.yaml

@@ -5,7 +5,9 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: sunnypilot-base
+  DOCKER_REGISTRY: ghcr.io/sunnypilot
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
   badges:
@@ -15,13 +17,13 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
-        python3 selfdrive/ui/translations/create_badges.py
+        ${{ env.RUN }} "python3 selfdrive/ui/translations/create_badges.py"
 
         rm .gitattributes
 

.github/workflows/build-all-tinygrad-models.yaml

@@ -34,10 +34,10 @@ jobs:
           echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
           echo "tinygrad_ref is $ref"
 
-      - name: Checkout docs repo (sunnypilot-models, gh-pages)
+      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -120,7 +120,6 @@ jobs:
     with:
       upstream_branch: ${{ matrix.model.ref }}
       custom_name: ${{ matrix.model.display_name }}
-      is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
     secrets: inherit
@@ -141,7 +140,7 @@ jobs:
         run: |
           echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
           built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
+            ls output | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
           )
           jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
             'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
@@ -158,7 +157,6 @@ jobs:
     with:
       upstream_branch: ${{ matrix.model.ref }}
       custom_name: ${{ matrix.model.display_name }}
-      is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
       artifact_suffix: -retry
@@ -170,7 +168,6 @@ jobs:
     if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       max-parallel: 1
       matrix:
         model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
@@ -204,7 +201,7 @@ jobs:
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

.github/workflows/build-single-tinygrad-model.yaml

@@ -24,11 +24,6 @@ on:
         required: false
         type: string
         default: ''
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
       bypass_push:
         description: 'Bypass pushing to GitLab for build-all'
         required: false
@@ -44,11 +39,6 @@ on:
         description: 'Custom name for the model (no date, only name)'
         required: false
         type: string
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
       recompiled_dir:
         description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
         required: true
@@ -92,7 +82,7 @@ jobs:
     with:
       upstream_branch: ${{ inputs.upstream_branch }}
       custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
-      is_20hz: ${{ inputs.is_20hz }}
+      is_20hz: true
       artifact_suffix: ${{ inputs.artifact_suffix }}
     secrets: inherit
 
@@ -129,7 +119,7 @@ jobs:
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

.github/workflows/cereal_validation.yaml

@@ -20,46 +20,57 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CI: 1
+  PYTHONWARNINGS: error
+  BASE_IMAGE: openpilot-base
+  BUILD: selfdrive/test/docker_build.sh base
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
+  generate_cereal_artifact:
+    name: Generate cereal validation artifacts
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
+      - name: Generate the log file
+        run: |
+          ${{ env.RUN }} "cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema_instances.bin" && \
+          ls -la
+          ls -la cereal/messaging/tests
+      - name: 'Prepare artifact'
+        run: |
+          mkdir -p "cereal/messaging/tests/cereal_validations"
+          cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py"
+          cp schema_instances.bin "cereal/messaging/tests/cereal_validations/schema_instances.bin"
+      - name: 'Upload Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: cereal_validations
+          path: cereal/messaging/tests/cereal_validations
+
   validate_cereal_with_upstream:
     name: Validate cereal with Upstream
     runs-on: ubuntu-24.04
+    needs: generate_cereal_artifact
     steps:
-      - name: Checkout sunnypilot cereal
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v6
-        with:
-          sparse-checkout: cereal
-
-      - name: Init sunnypilot opendbc submodule
-        run: git submodule update --init --depth 1 opendbc_repo
-
-      - name: Checkout upstream openpilot cereal
-        uses: actions/checkout@v6
         with:
           repository: 'commaai/openpilot'
-          path: upstream_openpilot
+          submodules: true
-          sparse-checkout: cereal
           ref: "refs/heads/master"
-
+      - uses: ./.github/workflows/setup-with-retry
-      - name: Init upstream opendbc submodule
+      - name: Build openpilot
-        working-directory: upstream_openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
-        run: git submodule update --init --depth 1 opendbc_repo
+      - name: Download build artifacts
-
+        uses: actions/download-artifact@v4
-      - name: Install uv
+        with:
-        run: pip install uv
+          name: cereal_validations
-
+          path: cereal/messaging/tests/cereal_validations
-      - name: Generate sunnypilot schema
+      - name: 'Run the validation'
         run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          chmod +x cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py
-          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
+          ${{ env.RUN }} "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f cereal/messaging/tests/cereal_validations/schema_instances.bin"
-            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
-            -g -f /tmp/sp_schema.json --cereal-dir cereal
-
-      - name: Validate against upstream
-        run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
-          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
-            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
-            -r -f /tmp/sp_schema.json --cereal-dir upstream_openpilot/cereal

.github/workflows/ci_weekly_report.yaml

@@ -0,0 +1,101 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
+jobs:
+  setup:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
+    steps:
+      - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
+        run: |
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.run_number }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
+              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
+                case "failure":
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
+                case "canceled":
+                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
+              }
+            });
+            return JSON.stringify({"jobs": report});
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
+
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -0,0 +1,17 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    uses: sunnypilot/sunnypilot/.github/workflows/tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,21 @@
+name: 'compile openpilot'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/diff_report.yaml

@@ -1,45 +0,0 @@
-name: diff report
-
-on:
-  pull_request_target:
-    types: [opened, synchronize, reopened]
-
-jobs:
-  comment:
-    name: comment
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-    - name: Wait for process replay
-      id: wait
-      continue-on-error: true
-      uses: lewagon/wait-on-check-action@v1.3.4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        check-name: process replay
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        allowed-conclusions: success,failure
-        wait-interval: 20
-    - name: Download diff
-      if: steps.wait.outcome == 'success'
-      uses: dawidd6/action-download-artifact@v6
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: tests.yaml
-        workflow_conclusion: ''
-        pr: ${{ github.event.number }}
-        name: diff_report_${{ github.event.number }}
-        path: .
-        allow_forks: true
-    - name: Comment on PR
-      if: steps.wait.outcome == 'success'
-      uses: thollander/actions-comment-pull-request@v2
-      with:
-        filePath: diff_report.txt
-        comment_tag: diff_report
-        pr_number: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -22,19 +22,19 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
     # Build
     - name: Build docs
       run: |
-        git lfs pull
+        # TODO: can we install just the "docs" dependency group without the normal deps?
-        pip install zensical
+        pip install mkdocs
-        python scripts/docs.py build
+        mkdocs build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,44 +5,7 @@ on:
     types: [created, edited]
 
 jobs:
-  cleanup-branches:
+  # TODO: gc old branches in a separate job in this workflow
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Delete stale Jenkins branches
-        uses: actions/github-script@v8
-        with:
-          script: |
-            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
-            const prefixes = ['tmp-jenkins', '__jenkins'];
-
-            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              per_page: 100,
-            })) {
-              for (const branch of response.data) {
-                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
-
-                const { data: commit } = await github.rest.repos.getCommit({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: branch.commit.sha,
-                });
-
-                const commitDate = new Date(commit.commit.committer.date).getTime();
-                if (commitDate < cutoff) {
-                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
-                  await github.rest.git.deleteRef({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                    ref: `heads/${branch.name}`,
-                  });
-                }
-              }
-            }
-
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
@@ -52,7 +15,7 @@ jobs:
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v8
+      uses: actions/github-script@v7
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +35,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -86,7 +49,7 @@ jobs:
 
     - name: Delete trigger comment
       if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v8
+      uses: actions/github-script@v7
       with:
         script: |
           await github.rest.issues.deleteComment({

.github/workflows/model_review.yaml

@@ -16,23 +16,23 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
-        with:
-          submodules: true
       - name: Checkout master
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           ref: master
           path: base
       - run: git lfs pull
       - run: cd base && git lfs pull
 
+      - run: pip install onnx
+
       - name: scripts/reporter.py
         id: report
         run: |
           echo "content<<EOF" >> $GITHUB_OUTPUT
           echo "## Model Review" >> $GITHUB_OUTPUT
-          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Post model report comment

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh
+  BUILD: release/ci/docker_build_sp.sh prebuilt
 
 jobs:
   build_prebuilt:
@@ -28,8 +28,8 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
+        check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - run: git lfs pull

.github/workflows/raylib_ui_preview.yaml

@@ -0,0 +1,175 @@
+name: "raylib ui preview"
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened, edited]
+    branches:
+      - 'master'
+    paths:
+      - 'selfdrive/assets/**'
+      - 'selfdrive/ui/**'
+      - 'system/ui/**'
+  workflow_dispatch:
+
+env:
+  UI_JOB_NAME: "Create raylib UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-raylib-ui"
+
+jobs:
+  preview:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    name: preview
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Waiting for ui generation to start
+        run: sleep 30
+
+      - name: Waiting for ui generation to end
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ env.SHA }}
+          check-name: ${{ env.UI_JOB_NAME }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allowed-conclusions: success
+          wait-interval: 20
+
+      - name: Getting workflow run ID
+        id: get_run_id
+        run: |
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+
+      - name: Getting proposed ui
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          run_id: ${{ steps.get_run_id.outputs.run_id }}
+          search_artifacts: true
+          name: raylib-report-1-${{ env.REPORT_NAME }}
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui_raylib
+          ref: openpilot_master_ui_raylib
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git checkout --orphan=new_master_ui_raylib
+          git rm -rf *
+          git branch -D openpilot_master_ui_raylib
+          git branch -m openpilot_master_ui_raylib
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "raylib screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui_raylib --force
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: >-
+          sudo apt-get update && sudo apt-get install -y imagemagick
+
+          scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device')
+          A=($scenes)
+
+          DIFF=""
+          TABLE="<details><summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
+
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+            # Check if the master file exists
+            if [ ! -f "${{ github.workspace }}/master_ui_raylib/${A[$i]}.png" ]; then
+              # This is a new file in PR UI that doesn't exist in master
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              mv ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            else
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
+            fi
+          done
+
+          TABLE="${TABLE}</table></details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
+
+      - name: Saving proposed ui
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
+          git commit -m "raylib screenshots for PR #${{ github.event.number }}"
+          git push origin ${{ env.BRANCH_NAME }} --force
+
+      - name: Comment Screenshots on PR
+        if: github.event_name == 'pull_request_target'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_screenshots_raylib **${{ github.run_id }}**)_ -->
+            ## raylib UI Preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_screenshots_raylib
+          pr_number: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -7,12 +7,20 @@ on:
 jobs:
   build___nightly:
     name: build __nightly
+    env:
+      ImageOS: ubuntu24
+    container:
+      image: ghcr.io/sunnypilot/sunnypilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     permissions:
       checks: read
       contents: write
     steps:
+    - name: Install wait-on-check-action dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
       uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
@@ -21,11 +29,14 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build __nightly'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
+        check-regexp: ^((?!.*(build prebuilt).*).)*$
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-    - run: ./tools/op.sh setup
+    - name: Pull LFS
+      run: |
+        git config --global --add safe.directory '*'
+        git lfs pull
     - name: Push __nightly
       run: BRANCH=__nightly release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -6,54 +6,62 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: sunnypilot-base
+  BUILD: release/ci/docker_build_sp.sh base
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
-  package_updates:
+  update_translations:
-    name: package_updates
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     steps:
-    - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Update translations
+        run: |
+          ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish"
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
+        with:
+          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
+          commit-message: "Update translations"
+          title: "[bot] Update translations"
+          body: "Automatic PR from repo-maintenance -> update_translations"
+          branch: "update-translations"
+          base: "master"
+          delete-branch: true
+          labels: bot
+
+  package_updates:
+    name: package_updates
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/sunnypilot/sunnypilot-base:latest
+    if: github.repository == 'sunnypilot/sunnypilot'
+    steps:
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
     - name: uv lock
-      run: uv lock --upgrade
+      if: github.repository == 'commaai/openpilot'
-    - name: uv pip tree
-      id: pip_tree
       run: |
-        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
+        python3 -m ensurepip --upgrade
-        uv pip tree >> $GITHUB_OUTPUT
+        pip3 install uv
-        echo 'EOF' >> $GITHUB_OUTPUT
+        uv lock --upgrade
-    - name: venv size
-      id: venv_size
-      run: |
-        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
-        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
-        echo "" >> $GITHUB_OUTPUT
-        echo "Top 10 by size:" >> $GITHUB_OUTPUT
-        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
-          | grep -v '\.dist-info' \
-          | grep -v '__pycache__' \
-          | sort -rh \
-          | head -10 \
-          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
     - name: bump submodules
       run: |
-        git config submodule.msgq.update none
+        git config --global --add safe.directory '*'
-        git config submodule.rednose_repo.update none
-        git config submodule.teleoprtc_repo.update none
         git config submodule.tinygrad.update none
         git submodule update --remote
         git add .
     - name: update car docs
       run: |
+        export PYTHONPATH="$PWD"
+        scons -j$(nproc) --minimal opendbc_repo
         python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:
         author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
         token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
@@ -62,16 +70,5 @@ jobs:
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: |
+        body: 'Automatic PR from repo-maintenance -> package_updates'
-          Automatic PR from repo-maintenance -> package_updates
-
-          ```
-          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
-          ${{ steps.venv_size.outputs.VENV_SIZE }}
-          ```
-
-          ```
-          $ uv pip tree
-          ${{ steps.pip_tree.outputs.PIP_TREE }}
-          ```
         labels: bot

.github/workflows/setup-with-retry/action.yaml

@@ -0,0 +1,52 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: false
+    default: ''
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
+
+outputs:
+  duration:
+    description: 'Duration of the setup process in seconds'
+    value: ${{ steps.get_duration.outputs.duration }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: start_time
+      shell: bash
+      run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        is_retried: true
+    - id: get_duration
+      shell: bash
+      run: |
+        END_TIME=$(date +%s)
+        DURATION=$((END_TIME - START_TIME))
+        echo "Total duration: $DURATION seconds"
+        echo "duration=$DURATION" >> $GITHUB_OUTPUT

.github/workflows/setup/action.yaml

@@ -0,0 +1,56 @@
+name: 'openpilot env setup'
+
+inputs:
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    required: false
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
+    - shell: bash
+      name: No retries!
+      run: |
+        if [ "${{ github.run_attempt }}" -gt ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association == 'NONE' && 2 || 1}} ]; then
+          echo -e "\033[0;31m##################################################"
+          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
+          echo -e "\033[0;31m##################################################\033[0m"
+          exit 1
+        fi
+
+    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
+    - shell: bash
+      run: git lfs pull
+
+    # build cache
+    - id: date
+      shell: bash
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - shell: bash
+      run: echo "$CACHE_COMMIT_DATE"
+    - id: scons-cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    # build our docker image
+    - shell: bash
+      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -13,7 +13,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 
@@ -34,7 +34,7 @@ jobs:
   stale_drafts:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 

.github/workflows/sunnypilot-build-model.yaml

@@ -164,63 +164,18 @@ jobs:
           source /etc/profile
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
+          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}"
 
-          COMPILE_MODELD="${{ github.workspace }}/sunnypilot/modeld_v2/compile_modeld.py"
+          # Loop through all .onnx files
-          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
-          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
-          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
-
-          # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
-            echo "Generating metadata: $onnx_file"
+            base_name=$(basename "$onnx_file" .onnx)
-            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            output_file="${{ env.MODELS_DIR }}/${base_name}_tinygrad.pkl"
+
+            echo "Compiling: $onnx_file -> $output_file"
+            QCOM=1 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file"
+            QCOM=1 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
-          # Detect model type and build compile args
-          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
-          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
-          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
-          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
-          SUPERCOMBO_ONNX="${{ env.MODELS_DIR }}/supercombo.onnx"
-
-          MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
-          if [ -f "$VISION_ONNX" ]; then
-            ONNX_ARGS="--vision-onnx $VISION_ONNX"
-            if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_multi_policy
-              ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
-            elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_multi_policy
-              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
-            elif [ -f "$POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_policy
-              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
-            fi
-          elif [ -f "$SUPERCOMBO_ONNX" ]; then
-            MODEL_TYPE=supercombo
-            ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
-          fi
-
-          if [ -n "$MODEL_TYPE" ]; then
-            echo "Detected: $MODEL_TYPE -> driving_tinygrad.pkl"
-            env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
-              --model-type $MODEL_TYPE \
-              --model-size $MODEL_SIZE \
-              --camera-resolutions $CAMERA_RES \
-              $ONNX_ARGS \
-              --output "${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
-          fi
-
-      - name: Validate Model Outputs
-        run: |
-          source /etc/profile
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
-            --validate-only \
-            --model-dir "${{ env.MODELS_DIR }}"
-
       - name: Prepare Output
         run: |
           sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -229,9 +184,8 @@ jobs:
           # Copy the model files
           rsync -avm \
             --include='*.dlc' \
+            --include='*.thneed' \
             --include='*.pkl' \
-            --include='*.chunk*' \
-            --include='*.chunkmanifest' \
             --include='*.onnx' \
             --exclude='*' \
             --delete-excluded \

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -6,10 +6,10 @@ env:
   CI_DIR: ${{ github.workspace }}/release/ci
   SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
-
+  
   # Branch configurations
   STAGING_SOURCE_BRANCH: 'master'
-
+  
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
@@ -75,7 +75,7 @@ jobs:
             cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
             echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
             echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
-
+          
             is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
             echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
 
@@ -85,7 +85,7 @@ jobs:
           fi
           echo "build=$BUILD" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
-
+  
   validate_tests:
     runs-on: ubuntu-24.04
     needs: [ prepare_strategy ]
@@ -119,7 +119,7 @@ jobs:
       needs.prepare_strategy.result == 'success' &&
       (needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
       (!contains(github.event_name, 'pull_request') ||
-      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) 
       }}
     steps:
       - uses: actions/checkout@v4
@@ -134,7 +134,7 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
@@ -148,7 +148,7 @@ jobs:
           echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
           echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
           echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
-
+          
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
@@ -180,13 +180,6 @@ jobs:
           ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
           cd $BUILD_DIR
           sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
-          echo "Building sunnypilot's modeld_v2..."
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
-          echo "Building sunnypilot's locationd..."
-          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
-          echo "Building openpilot's locationd..."
-          scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
-          echo "Building rest of sunnypilot"
           scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
@@ -198,27 +191,37 @@ jobs:
           sudo rm -rf ${OUTPUT_DIR}
           mkdir -p ${OUTPUT_DIR}
           rsync -am${RUNNER_DEBUG:+v} \
+            --include='**/panda/board/' \
+            --include='**/panda/board/obj' \
+            --include='**/panda/board/obj/panda.bin.signed' \
+            --include='**/panda/board/obj/panda_h7.bin.signed' \
+            --include='**/panda/board/obj/bootstub.panda.bin' \
+            --include='**/panda/board/obj/bootstub.panda_h7.bin' \
             --exclude='.sconsign.dblite' \
             --exclude='*.a' \
             --exclude='*.o' \
             --exclude='*.os' \
             --exclude='*.pyc' \
             --exclude='moc_*' \
-            --exclude='__pycache__' \
+            --exclude='*.cc' \
             --exclude='Jenkinsfile' \
+            --exclude='supercombo.onnx' \
+            --exclude='**/panda/board/*' \
+            --exclude='**/panda/board/obj/**' \
+            --exclude='**/panda/certs/' \
+            --exclude='**/panda/crypto/' \
             --exclude='**/release/' \
             --exclude='**/.github/' \
             --exclude='**/selfdrive/ui/replay/' \
             --exclude='**/__pycache__/' \
+            --exclude='**/selfdrive/ui/*.h' \
+            --exclude='**/selfdrive/ui/**/*.h' \
+            --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' \
             --exclude='${{env.SCONS_CACHE_DIR}}' \
             --exclude='**/.git/' \
             --exclude='**/SConstruct' \
             --exclude='**/SConscript' \
             --exclude='**/.venv/' \
-            --exclude='selfdrive/modeld/models/*.onnx*' \
-            --exclude='sunnypilot/modeld*/models/*.onnx*' \
-            --exclude='third_party/*x86*' \
-            --exclude='third_party/*Darwin*' \
             --delete-excluded \
             --chown=comma:comma \
             ${BUILD_DIR}/ ${OUTPUT_DIR}/
@@ -238,8 +241,8 @@ jobs:
         if: always()
         run: |
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
-
+  
-
+  
   publish:
     concurrency:
       # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
@@ -290,7 +293,7 @@ jobs:
           echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
           echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
           echo "3. Update as needed (JSON array with no spaces)"
-
+      
       - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
         if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
         run: |
@@ -299,7 +302,7 @@ jobs:
             git push -f origin ${TAG}
 
   notify:
-    needs:
+    needs: 
     - prepare_strategy
     - build
     - publish
@@ -328,7 +331,7 @@ jobs:
           ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
           EOF
           )
-
+          
           {
             echo 'content<<EOFMARKER'
             echo "$MESSAGE"

.github/workflows/sunnypilot-master-dev-prep.yaml

@@ -49,7 +49,6 @@ jobs:
         with:
           fetch-depth: 0  # Fetch all history for all branches
           token: ${{ secrets.GITHUB_TOKEN }}
-          persist-credentials: false
 
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
@@ -174,38 +173,11 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Restore workflows from source
-        run: |
-          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-          SOURCE_BRANCH="${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-
-          # Ensure we are on the target branch
-          git checkout $TARGET_BRANCH
-
-          echo "Restoring .github/workflows from $SOURCE_BRANCH"
-          git checkout origin/$SOURCE_BRANCH -- .github/workflows
-
-          if ! git diff --cached --quiet; then
-            echo "Workflows differ. Committing restoration."
-            git commit -m "chore: restore .github/workflows from $SOURCE_BRANCH"
-          else
-            echo "Workflows match $SOURCE_BRANCH."
-          fi
-
-      - uses: actions/create-github-app-token@v2
-        id: ci-token
-        with:
-          app-id: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_ID }}
-          private-key: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_PRIVATE_KEY }}
-
       - name: Push changes if there are diffs
-        id: push-changes
+        id: push-changes  # Add an id so we can reference this step
         run: |
           TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
 
-          # Use the App Token to set the remote URL with authentication
-          git remote set-url origin "https://x-access-token:${{ steps.ci-token.outputs.token }}@github.com/${{ github.repository }}.git"
-
           # Fetch the latest from remote
           git fetch origin $TARGET_BRANCH
 
@@ -216,7 +188,7 @@ jobs:
             exit 0
           fi
 
-          # Push with the authenticated origin
+          # If we get here, there are diffs, so push
           if ! git push origin $TARGET_BRANCH --force; then
             echo "Failed to push changes to $TARGET_BRANCH"
             exit 1
@@ -241,3 +213,10 @@ jobs:
           gh run watch "$RUN_ID"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Trigger prebuilt workflow
+        if: success() && steps.push-changes.outputs.has_changes == 'true'
+        run: |
+          gh workflow run sunnypilot-build-prebuilt.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yaml

@@ -18,8 +18,16 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CI: 1
+  PYTHONWARNINGS: error
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: sunnypilot-base
+  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
+  MAPBOX_TOKEN_CI: ${{ secrets.MAPBOX_TOKEN_CI }}
+
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: release/ci/docker_build_sp.sh base
+
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e MAPBOX_TOKEN_CI=$MAPBOX_TOKEN_CI -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
   PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
 
 jobs:
@@ -29,13 +37,12 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     env:
       STRIPPED_DIR: /tmp/releasepilot
-      PYTHONPATH: /tmp/releasepilot
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Getting LFS files
@@ -47,15 +54,17 @@ jobs:
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot and run checks
-      timeout-minutes: 30
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      working-directory: ${{ env.STRIPPED_DIR }}
+      run: |
-      run: python3 system/manager/build.py
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "python3 system/manager/build.py"
     - name: Run tests
       timeout-minutes: 1
-      working-directory: ${{ env.STRIPPED_DIR }}
+      run: |
-      run: release/check-dirty.sh
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "release/check-dirty.sh"
     - name: Check submodules
       if: github.repository == 'sunnypilot/sunnypilot'
       timeout-minutes: 3
@@ -77,20 +86,73 @@ jobs:
         fi
         release/check-submodules.sh
 
+  build:
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup docker push
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'sunnypilot/sunnypilot'
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
+    - uses: ./.github/workflows/setup-with-retry
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: 30
+
   build_mac:
     name: build macOS
     runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Remove Homebrew from environment
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-      run: |
+    - name: Homebrew cache
-        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
+      uses: ./.github/workflows/auto-cache
-        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
+      if: false # disabling the cache for now because it is breaking macos builds...
-    - run: ./tools/op.sh setup
+      with:
+        save: false # No need save here if we manually save it later conditionally
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          brew-macos-${{ env.CACHE_COMMIT_DATE }}
+          brew-macos
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+      env:
+        PYTHONWARNINGS: default  # package install has DeprecationWarnings
+        HOMEBREW_DISPLAY_INSTALL_TIMES: 1
+    - name: Save Homebrew cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+    - run: git lfs pull
+    - name: Getting scons cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        save: false # No need save here if we manually save it later conditionally
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}-macos
     - name: Building openpilot
-      run: scons
+      run: . .venv/bin/activate && scons -j$(nproc)
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   static_analysis:
     name: static analysis
@@ -98,16 +160,18 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
+    env:
+      PYTHONWARNINGS: default
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Static analysis
       timeout-minutes: 1
-      run: scripts/lint/lint.sh
+      run: ${{ env.RUN }} "scripts/lint/lint.sh"
 
   unit_tests:
     name: unit tests
@@ -115,22 +179,24 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
+      id: setup-step
     - name: Build openpilot
-      run: scons
+      run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 999 }}
       run: |
-        source selfdrive/test/setup_xvfb.sh
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-        # Pre-compile Python bytecode so each pytest worker doesn't need to
+                        # Pre-compile Python bytecode so each pytest worker doesn't need to
-        $PYTEST --collect-only -m 'not slow' -qq
+                        $PYTEST --collect-only -m 'not slow' -qq && \
-        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
+                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
+                        chmod -R 777 /tmp/comma_download_cache"
 
   process_replay:
     name: process replay
@@ -139,71 +205,48 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
+      id: setup-step
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
     - name: Build openpilot
-      run: scons
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
-      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
+      run: |
-      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
+        ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache"
     - name: Print diff
       id: print-diff
       if: always()
       run: cat selfdrive/test/process_replay/diff.txt
-    - name: Print diff report
+    - uses: actions/upload-artifact@v4
-      if: always()
-      run: cat selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v6
       if: always()
       continue-on-error: true
       with:
         name: process_replay_diff.txt
         path: selfdrive/test/process_replay/diff.txt
-    - name: Upload diff report
+    - name: Upload reference logs
-      uses: actions/upload-artifact@v6
+      if: false # TODO: move this to github instead of azure
-      if: always() && github.event_name == 'pull_request'
-      continue-on-error: true
-      with:
-        name: diff_report_${{ github.event.number }}
-        path: selfdrive/test/process_replay/diff_report.txt
-    - name: Checkout ci-artifacts
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v4
-      with:
-        repository: commaai/ci-artifacts
-        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-        path: ${{ github.workspace }}/ci-artifacts
-    - name: Prepare refs
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      working-directory: ${{ github.workspace }}/ci-artifacts
       run: |
-        git config user.name "GitHub Actions Bot"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-        git config user.email "<>"
-        git fetch origin process-replay || true
-        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
-        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
-        echo "${{ github.sha }}" > ref_commit
-        git add .
-        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
-    - name: Push refs
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force
     - name: Run regen
       if: false
       timeout-minutes: 4
-      env:
+      run: |
-        ONNXCPU: 1
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-      run: $PYTEST selfdrive/test/process_replay/test_regen.py
+                        chmod -R 777 /tmp/comma_download_cache"
 
   simulator_driving:
     name: simulator driving
@@ -211,44 +254,47 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     if: false  # FIXME: Started to timeout recently
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
+      id: setup-step
     - name: Build openpilot
-      run: scons
-    - name: Driving test
-      timeout-minutes: 2
       run: |
-        source selfdrive/test/setup_xvfb.sh
+        ${{ env.RUN }} "scons -j$(nproc)"
-        pytest -s tools/sim/tests/test_metadrive_bridge.py
+    - name: Driving test
+      timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
+      run: |
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        source selfdrive/test/setup_vsound.sh && \
+                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
 
-  create_ui_report:
+  create_raylib_ui_report:
-    name: Create UI Report
+    name: Create raylib UI Report
     runs-on: ${{
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         with:
           submodules: true
-      - run: ./tools/op.sh setup
+      - uses: ./.github/workflows/setup-with-retry
       - name: Build openpilot
-        run: scons
+        run: ${{ env.RUN }} "scons -j$(nproc)"
-      - name: Create UI Report
+      - name: Create raylib UI Report
-        run: |
+        run: >
-          source selfdrive/test/setup_xvfb.sh
+          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-          python3 selfdrive/ui/tests/diff/replay.py
+                          source selfdrive/test/setup_xvfb.sh &&
-          python3 selfdrive/ui/tests/diff/replay.py --big
+                          python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py"
-      - name: Upload UI Report
+      - name: Upload Raylib UI Report
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
-          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/diff/report
+          path: selfdrive/ui/tests/test_ui/raylib_report/screenshots

.github/workflows/ui_preview.yaml

@@ -1,175 +0,0 @@
-name: "ui preview"
-on:
-  push:
-    branches:
-      - master
-  pull_request_target:
-    types: [assigned, opened, synchronize, reopened, edited]
-    branches:
-      - 'master'
-    paths:
-      - 'selfdrive/assets/**'
-      - 'selfdrive/ui/**'
-      - 'system/ui/**'
-  workflow_dispatch:
-
-env:
-  UI_JOB_NAME: "Create UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
-  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
-
-  # variant:video_prefix:master_branch
-  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
-
-jobs:
-  preview:
-    if: github.repository == 'sunnypilot/sunnypilot'
-    name: preview
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-
-      - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@v1.3.4
-        with:
-          ref: ${{ env.SHA }}
-          check-name: ${{ env.UI_JOB_NAME }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allowed-conclusions: success
-          wait-interval: 20
-
-      - name: Getting workflow run ID
-        id: get_run_id
-        run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
-
-      - name: Getting proposed ui
-        uses: dawidd6/action-download-artifact@v6
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ steps.get_run_id.outputs.run_id }}
-          search_artifacts: true
-          name: ui-report-1-${{ env.REPORT_NAME }}
-          path: ${{ github.workspace }}/pr_ui
-
-      - name: Getting mici master ui
-        uses: actions/checkout@v6
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_mici
-          ref: openpilot_master_ui_mici_raylib
-
-      - name: Getting big master ui
-        uses: actions/checkout@v6
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_big
-          ref: openpilot_master_ui_big_raylib
-
-      - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video branch <<< "$variant"
-            master_dir="${{ github.workspace }}/master_${name}"
-            cd "$master_dir"
-            git checkout --orphan=new_branch
-            git rm -rf *
-            git branch -D "$branch"
-            git branch -m "$branch"
-            git config user.name "GitHub Actions Bot"
-            git config user.email "<>"
-            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
-            git add .
-            git commit -m "${name} video for commit ${{ env.SHA }}"
-            git push origin "$branch" --force
-          done
-
-      - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
-
-      - name: Finding diffs
-        if: github.event_name == 'pull_request_target'
-        id: find_diff
-        run: |
-          export PYTHONPATH=${{ github.workspace }}
-          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
-
-          COMMENT=""
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video _ <<< "$variant"
-            diff_name="${name}_diff"
-
-            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
-            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
-
-            diff_exit_code=0
-            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
-              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
-              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
-              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
-
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
-
-            REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
-            if [ $diff_exit_code -eq 0 ]; then
-              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
-            else
-              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
-            fi
-          done
-
-          {
-            echo "COMMENT<<EOF"
-            echo "$COMMENT"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Saving proposed ui
-        if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_mici
-        run: |
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          git checkout --orphan=${{ env.BRANCH_NAME }}
-          git rm -rf *
-          mv ${{ github.workspace }}/pr_ui/* .
-          git add .
-          git commit -m "ui videos for PR #${{ github.event.number }}"
-          git push origin ${{ env.BRANCH_NAME }} --force
-
-          # Append diff reports to report files branch
-          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
-          for variant in $VARIANTS; do
-            IFS=':' read -r name _ _ <<< "$variant"
-            diff_name="${name}_diff"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
-            git add "${diff_name}_pr_${{ github.event.number }}.html"
-          done
-          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
-          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-
-      - name: Comment on PR
-        if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          message: |
-            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
-            ## UI Preview
-            ${{ steps.find_diff.outputs.COMMENT }}
-          comment_tag: run_id_ui_preview
-          pr_number: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -13,13 +13,13 @@ venv/
 a.out
 .hypothesis
 .cache/
-bin/
+
+/docs_site/
 
 *.mp4
 *.dylib
 *.DSYM
 *.d
-*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -39,15 +39,11 @@ bin/
 *.mo
 *_pyx.cpp
 *.stats
-*.pkl
-*.pkl*
 config.json
+clcache
 compile_commands.json
 compare_runtime*.html
-selfdrive/modeld/models/tg_input_devices.json
 
-# build artifacts
-docs_site/
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
@@ -60,36 +56,48 @@ system/camerad/test/ae_gray_test
 .coverage*
 coverage.xml
 htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/models/*.pkl
+sunnypilot/modeld*/thneed/compile
+sunnypilot/modeld*/models/*.thneed
+sunnypilot/modeld*/models/*.pkl
 
 # openpilot log files
 *.bz2
 *.zst
-*.rlog
 
 build/
 
 !**/.gitkeep
 
+poetry.toml
+Pipfile
 
 ### VisualStudioCode ###
-*.vsix
-.history
-.ionide
 .vscode/*
-.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# agents
+# Local History for Visual Studio Code
-.claude/
+.history/
-.context/
+
-PLAN.md
+# Built Visual Studio Code Extensions
-TASK.md
+*.vsix
-CLAUDE.md
+
-SKILL.md
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
 
 ### JetBrains ###
 !.idea/customTargets.xml

.gitmodules

@@ -15,7 +15,7 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/sunnypilot/tinygrad.git
+  url = https://github.com/commaai/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
   path = sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.python-version

@@ -1 +0,0 @@
-3.12.13

.vscode/launch.json

@@ -52,9 +52,6 @@
       "type": "lldb",
       "request": "attach",
       "pid": "${command:pickMyProcess}",
-      "sourceMap": {
-        ".": "${workspaceFolder}/opendbc/safety"
-      },
       "initCommands": [
         "script import time; time.sleep(3)"
       ]

.vscode/settings.json

@@ -21,6 +21,7 @@
     "common/**",
     "selfdrive/**",
     "system/**",
+    "third_party/**",
     "tools/**",
   ]
 }

CHANGELOG.md

@@ -1,193 +1,6 @@
-sunnypilot Version 2026.002.000 (2026-xx-xx)
+sunnypilot Version 2025.003.000 (20xx-xx-xx)
 ========================
 
-sunnypilot Version 2026.001.000 (2026-05-06)
-========================
-* What's Changed (sunnypilot/sunnypilot)
-  * Complete rewrite of the user interface from Qt C++ to Raylib Python
-  * comma four support
-  * ui: sunnypilot toggle style by @nayan8teen
-  * ui: fix scroll panel mouse wheel behavior by @nayan8teen
-  * ui: sunnypilot panels by @nayan8teen
-  * sunnylink: centralize key pair handling in sunnylink registration by @devtekve
-  * ui: reimplement sunnypilot branding with Raylib by @sunnyhaibin
-  * ui: Platform Selector by @Discountchubbs
-  * ui: vehicle brand settings by @Discountchubbs
-  * ui: sunnylink client-side implementation by @nayan8teen
-  * ui: `NetworkUISP` by @Discountchubbs
-  * ui: add sunnypilot font by @nayan8teen
-  * ui: sunnypilot sponsor tier color mapping by @sunnyhaibin
-  * ui: sunnylink panel by @nayan8teen
-  * ui: Models panel by @Discountchubbs
-  * ui: software panel by @Discountchubbs
-  * modeld_v2: support planplus outputs by @Discountchubbs
-  * ui: OSM panel by @Discountchubbs
-  * ui: Developer panel extension by @Discountchubbs
-  * sunnylink: Vehicle Selector support by @sunnyhaibin
-  * [TIZI/TICI] ui: Developer Metrics by @rav4kumar
-  * [comma 4] ui: sunnylink panel by @nayan8teen
-  * ui: lateral-only and longitudinal-only UI statuses support by @royjr
-  * sunnylink: elliptic curve keys support and improve key path handling by @nayan8teen
-  * sunnylink: block remote modification of SSH key parameters by @zikeji
-  * [TIZI/TICI] ui: rainbow path by @rav4kumar
-  * [TIZI/TICI] ui: chevron metrics by @rav4kumar
-  * ui: include MADS enabled state to `engaged` check by @sunnyhaibin
-  * Toyota: Enforce Factory Longitudinal Control by @sunnyhaibin
-  * ui: fix malformed dongle ID display on the PC if dongleID is not set by @dzid26
-  * SL: Re enable and validate ingestion of swaglogs by @devtekve
-  * modeld_v2: planplus model tuning by @Discountchubbs
-  * ui: fix Always Offroad button visibility by @nayan8teen
-  * Reimplement sunnypilot Terms of Service & sunnylink Consent Screens by @sunnyhaibin
-  * [TIZI/TICI] ui: update dmoji position and Developer UI adjustments by @rav4kumar
-  * modeld: configurable camera offset by @Discountchubbs
-  * [TIZI/TICI] ui: sunnylink status on sidebar by @Copilot
-  * ui: Global Brightness Override by @nayan8teen
-  * ui: Customizable Interactive Timeout by @sunnyhaibin
-  * sunnylink: add units to param metadata by @nayan8teen
-  * ui: Customizable Onroad Brightness by @sunnyhaibin
-  * [TIZI/TICI] ui: Steering panel by @nayan8teen
-  * [TIZI/TICI] ui: Rocket Fuel by @rav4kumar
-  * [TIZI/TICI] ui: MICI style turn signals by @rav4kumar
-  * [TIZI/TICI] ui: MICI style blindspot indicators by @sunnyhaibin
-  * [MICI] ui: display blindspot indicators when available by @rav4kumar
-  * [TIZI/TICI] ui: Road Name by @rav4kumar
-  * [TIZI/TICI] ui: Blue "Exit Always Offroad" button by @dzid26
-  * [TIZI/TICI] ui: Speed Limit by @rav4kumar
-  * Reapply "latcontrol_torque: lower kp and lower friction threshold (commaai/openpilot#36619)" by @sunnyhaibin
-  * [TIZI/TICI] ui: steering arc by @royjr
-  * [TIZI/TICI] ui: Smart Cruise Control elements by @sunnyhaibin
-  * [TIZI/TICI] ui: Green Light and Lead Departure elements by @sunnyhaibin
-  * [TIZI/TICI] ui: standstill timer by @sunnyhaibin
-  * [MICI] ui: driving models selector by @Discountchubbs
-  * [TIZI/TICI] ui: Hide vEgo and True vEgo by @sunnyhaibin
-  * [TIZI/TICI] ui: Visuals panel by @nayan8teen
-  * Device: Retain QuickBoot state after op switch by @nayan8teen
-  * [TIZI/TICI] ui: Trips panel by @sunnyhaibin
-  * [TIZI/TICI] ui: dynamic ICBM status by @sunnyhaibin
-  * [TIZI/TICI] ui: Cruise panel by @sunnyhaibin
-  * ui: better wake mode support by @nayan8teen
-  * Pause Lateral Control with Blinker: Post-Blinker Delay by @CHaucke89
-  * SCC-V: Use p97 for predicted lateral accel by @yasu-oh
-  * Controls: Support for Torque Lateral Control v0 Tune by @sunnyhaibin
-  * [TIZI/TICI] ui: ensure null checks for `CarParams` and `CarParamsSP` by @sunnyhaibin
-  * [TIZI/TICI] ui: use `vCruiseCluster` and `vEgoCluster` for SLA `preActive` by @sunnyhaibin
-  * Fix display of values when using use_float_scaling by @CHaucke89
-  * models: fix default & index "0" by @nayan8teen
-  * [TIZI/TICI] visuals: Improved speed limit by @angaz
-  * ICBM: ensure button timers update on disable to clear stale presses by @jamesmikesell
-  * [TIZI/TICI] ui: simplify Smart Cruise Control text rendering by @sunnyhaibin
-  * controlsd: fix steer_limited_by_safety not updating under MADS by @zephleggett
-  * soundd: trigger timeout warning during MADS lateral-only by @zephleggett
-  * pandad: flasher for Rivian long upgrade module by @lukasloetkolben
-  * modeld_v2: tinygrad transformation warp by @Discountchubbs
-  * tools: block `manage_sunnylinkd` in sim startup script by @sunnyhaibin
-  * [MICI] ui: need superclass `_render` in `HudRendererSP` by @sunnyhaibin
-  * [TIZI/TICI] ui: Speed Limit Assist active status by @sunnyhaibin
-  * ui: reimplement "Screen Off" option to Onroad Brightness by @sunnyhaibin
-  * ui: don't hide steering wheel when blindspot disabled by @royjr
-  * ui: Speed Limit Assist `preActive` improvements  by @sunnyhaibin
-  * ui: consolidate Speed Limit Assist `preActive` status rendering by @sunnyhaibin
-  * [MICI] ui: Speed Limit Assist `preActive` status by @sunnyhaibin
-  * sunnypilot modeld: remove thneed modeld by @Discountchubbs
-  * modeld_v2: decouple planplus scaling from accel by @Discountchubbs
-  * sunnylink: Handle exceptions in `getParamsAllKeysV1` to log crashes by @devtekve
-  * [TIZI/TICI] ui: Developer UI cleanup by @sunnyhaibin
-  * [TIZI/TICI] ui: dynamic alert size by @nayan8teen
-  * i18n(fr): Add French translations by @didlawowo
-  * Toyota: Stop and Go Hack (Alpha) by @sunnyhaibin
-  * ui: `AlertFadeAnimator` for longitudinal-related statuses by @sunnyhaibin
-  * pandad: gate unsupported pandas before flashing by @sunnyhaibin
-  * Rivian: Flash xnor's Longitudinal Upgrade Kit prior supported panda check by @lukasloetkolben
-  * [TIZI/TICI] ui: add back gate steering arc behind toggle by @sunnyhaibin
-  * ui: gate Onroad Brightness Delay on readiness by @sunnyhaibin
-  * ui: add new timer options for Onroad Brightness Delay by @sunnyhaibin
-  * [TIZI/TICI] ui: branch switcher is always available by @sunnyhaibin
-  * pandad: always prioritize internal panda by @sunnyhaibin
-  * sunnylinkd: fetch compressed params schema by @sunnyhaibin
-  * sunnypilot locationd: remove unused car_ekf filter by @sunnyhaibin
-  * modeld_v2: update deprecated temporalPose ref by @sunnyhaibin
-  * NNLC: restore pre-v1 PID gains in torque extension by @mmmorks
-  * MADS safety: enable heartbeat and lateral controls mismatch checks by @sunnyhaibin
-  * [MICI] ui: models panel enhancements by @nayan8teen
-  * [TIZI/TICI] ui: fix unintended selection while scrolling in TreeOptionDialog by @TheSecurityDev
-  * tools: script for video concatenation by @Discountchubbs
-  * tools: profile memory usage by @Discountchubbs
-  * [TIZI/TICI] ui: remove per-frame param sync by @sunnyhaibin
-  * [MICI] ui: always offroad by @nayan8teen
-  * controls: always default Torque Lateral Control to v0 Tune by @sunnyhaibin
-  * Revert "controls: always default Torque Lateral Control to v0 Tune" by @sunnyhaibin
-  * Reapply "controls: always default Torque Lateral Control to v0 Tune" (#1806) by @sunnyhaibin
-  * [MICI] ui: add sunnylink info & connectivity check by @nayan8teen
-  * sunnylink: Remove unused API endpoint by @devtekve
-  * DM: wheel touch enforcement in MADS by @sunnyhaibin
-  * torque: show static override values in Dev UI & gate `useParams` on custom torque tune by @sunnyhaibin
-  * MADS: suppress espActive event when long is not engaged by @sunnyhaibin
-  * sunnylink: SDUI by @sunnyhaibin
-  * [MICI] ui: align upstream changes with sunnypilot settings buttons by @nayan8teen
-  * ui: fix cellular toggles by @AmyJeanes
-  * sunnylink: switch athena domain by @devtekve
-  * Platform List: dynamically migrate CarPlatformBundle by @sunnyhaibin
-* What's Changed (sunnypilot/opendbc)
-  * Honda: DBC for Accord 9th Generation by @mvl-boston
-  * FCA: update tire stiffness values for `RAM_HD` by @dparring
-  * Honda: Nidec hybrid baseline brake support by @mvl-boston
-  * Subaru Global Gen2: bump steering limits and update tuning by @sunnyhaibin
-  * Toyota: Enforce Stock Longitudinal Control by @rav4kumar
-  * Nissan: use MADS enabled status for LKAS HUD logic by @downquark7
-  * Reapply "Lateral: lower friction threshold (#2915)" (#378) by @sunnyhaibin
-  * HKG: add KIA_FORTE_2019_NON_SCC fingerprint by @royjr
-  * Nissan: Parse cruise control buttons by @downquark7
-  * Rivian: Add stalk down ACC behavior to match stock Rivian by @lukasloetkolben
-  * Tesla: remove `TESLA_MODEL_X` from `dashcamOnly` by @ssysm
-  * Hyundai Longitudinal: refactor tuning by @Discountchubbs
-  * Tesla: add fingerprint for Model 3 Performance HW4 by @sunnyhaibin
-  * Toyota: do not disable radar when smartDSU or CAN Filter detected by @sunnyhaibin
-  * Honda: add missing `GasInterceptor` messages to Taiwan Odyssey DBC by @mvl-boston
-  * GM: remove `CHEVROLET_EQUINOX_NON_ACC_3RD_GEN` from `dashcamOnly` by @sunnyhaibin
-  * GM: remove `CHEVROLET_BOLT_NON_ACC_2ND_GEN` from `dashcamOnly` by @sunnyhaibin
-  * Hyundai Longitudinal: deprecate ramp update for dynamic tune by @Discountchubbs
-  * Rivian: long upgrade messages on bus 1 by @lukasloetkolben
-  * Toyota: Stop and Go Hack (Alpha) by @sunnyhaibin
-  * Toyota: gate Smart DSU behind Alpha Longitudinal by @sunnyhaibin
-  * Toyota: Gas Interceptor always set `standstill_req` by @sunnyhaibin
-  * MADS safety: dedicated `controls_allowed_lateral` by @sunnyhaibin
-  * Platform List: include community supported platforms by @sunnyhaibin
-* New Contributors (sunnypilot/sunnypilot)
-  * @TheSecurityDev made their first contribution in "ui: fix sidebar scroll in UI screenshots"
-  * @zikeji made their first contribution in "sunnylink: block remote modification of SSH key parameters"
-  * @Candy0707 made their first contribution in "[TIZI/TICI] ui: Fix misaligned turn signals and blindspot indicators with sidebar"
-  * @CHaucke89 made their first contribution in "Pause Lateral Control with Blinker: Post-Blinker Delay"
-  * @yasu-oh made their first contribution in "SCC-V: Use p97 for predicted lateral accel"
-  * @angaz made their first contribution in "[TIZI/TICI] visuals: Improved speed limit"
-  * @jamesmikesell made their first contribution in "ICBM: ensure button timers update on disable to clear stale presses"
-  * @zephleggett made their first contribution in "controlsd: fix steer_limited_by_safety not updating under MADS"
-  * @lukasloetkolben made their first contribution in "pandad: flasher for Rivian long upgrade module"
-  * @didlawowo made their first contribution in "i18n(fr): Add French translations"
-  * @mmmorks made their first contribution in "NNLC: restore pre-v1 PID gains in torque extension"
-* New Contributors (sunnypilot/opendbc)
-  * @AmyJeanes made their first contribution in "Tesla: Fix stock LKAS being blocked when MADS is enabled"
-  * @mvl-boston made their first contribution in "Honda: Update Clarity brake to renamed DBC message name"
-  * @dzid26 made their first contribution in "Tesla: Parse speed limit from CAN"
-  * @firestar5683 made their first contribution in "GM: Non-ACC platforms with steering only support"
-  * @downquark7 made their first contribution in "Nissan: use MADS enabled status for LKAS HUD logic"
-  * @royjr made their first contribution in "HKG: add KIA_FORTE_2019_NON_SCC fingerprint"
-  * @ssysm made their first contribution in "Tesla: remove `TESLA_MODEL_X` from `dashcamOnly`"
-* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2025.002.000...v2026.001.000
-************************
-* Synced with commaai's openpilot (v0.11.1)
-  * master commit c001f3c9b490a80e69539f0af6022f6e07ceb721 (April 16, 2026)
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-* Improved inter-process communication memory efficiency
-* comma four support
-
 sunnypilot Version 2025.002.000 (2025-11-06)
 ========================
 * What's Changed (sunnypilot/sunnypilot)

Dockerfile.openpilot

@@ -1,38 +1,14 @@
-FROM ubuntu:24.04
+FROM ghcr.io/commaai/openpilot-base:latest
 
 ENV PYTHONUNBUFFERED=1
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV OPENPILOT_PATH=/home/batman/openpilot
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales && \
-    rm -rf /var/lib/apt/lists/*
 
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-ENV OPENPILOT_PATH=/home/$USER/openpilot
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY --chown=$USER . ${OPENPILOT_PATH}/
+COPY . ${OPENPILOT_PATH}/
 
-ENV UV_BIN="/home/$USER/.local/bin/"
+ENV UV_BIN="/home/batman/.local/bin/"
-ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
+ENV PATH="$UV_BIN:$PATH"
-ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
+RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc)
-RUN tools/setup_dependencies.sh && \
-    sudo rm -rf /var/lib/apt/lists/*
-
-USER root
-RUN git config --global --add safe.directory '*'

Dockerfile.openpilot_base

@@ -0,0 +1,82 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED=1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX=1
+
+RUN dbus-uuidgen > /etc/machine-id
+RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /home/$USER
+COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /home/$USER && \
+    tools/install_python_dependencies.sh && \
+    rm -rf tools/ pyproject.toml uv.lock .cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Dockerfile.sunnypilot

@@ -0,0 +1,12 @@
+FROM ghcr.io/sunnypilot/sunnypilot-base:latest
+
+ENV PYTHONUNBUFFERED=1
+
+ENV OPENPILOT_PATH=/home/batman/openpilot
+
+RUN mkdir -p ${OPENPILOT_PATH}
+WORKDIR ${OPENPILOT_PATH}
+
+COPY . ${OPENPILOT_PATH}/
+
+RUN scons --cache-readonly -j$(nproc)

Dockerfile.sunnypilot_base

@@ -0,0 +1,83 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED=1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+    rsync \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX=1
+
+RUN dbus-uuidgen > /etc/machine-id
+RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /home/$USER
+COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /home/$USER && \
+    tools/install_python_dependencies.sh && \
+    rm -rf tools/ pyproject.toml uv.lock .cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -22,7 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-#export LOGPRINT=debug # this has gotten too spammy...
+export LOGPRINT=debug
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -166,8 +166,8 @@ node {
   env.GIT_BRANCH = checkout(scm).GIT_BRANCH
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
-  def excludeBranches = ['__nightly', 'devel', 'devel-staging',
+  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tizi', 'release-tizi-staging', 'release-mici', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
+                         'release-tici', 'release-tizi', 'release-tizi-staging', 'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
   if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
@@ -179,7 +179,7 @@ node {
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/release/build_release.sh"),
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
@@ -210,23 +210,30 @@ node {
       'HW + Unit Tests': {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
           step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
+      'loopback': {
+        deviceStage("loopback", "tizi-loopback", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+        ])
+      },
       'camerad OX03C10': {
         deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'sensord': {
@@ -244,9 +251,12 @@ node {
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
           step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          // TODO: enable once new AGNOS is available
+          // step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),
+          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
         ])
       },
 

README.md

@@ -11,10 +11,66 @@ Join the official sunnypilot community forum to stay up to date with all the lat
 https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
 
 ## 🚘 Running on a dedicated device in a car
-First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
+* A supported device to run this software
+    * a [comma three](https://comma.ai/shop/products/three) or a [C3X](https://comma.ai/shop/comma-3x)
+* This software
+* One of [the 325+ supported cars](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford, and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
+* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
+
+Detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
 
 ## Installation
-Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
+Please refer to [Recommended Branches](#recommended-branches) to find your preferred/supported branch. This guide will assume you want to install the latest `staging` branch.
+
+### If you want to use our newest branches (our rewrite)
+> [!TIP]
+>You can see the rewrite state on our [rewrite project board](https://github.com/orgs/sunnypilot/projects/2), and to install the new branches, you can use the following links
+
+* sunnypilot not installed or you installed a version before 0.8.17?
+  1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
+  2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
+  3. Input the installation URL per [Recommended Branches](#recommended-branches). Example: ```https://staging.sunnypilot.ai```.
+  4. Complete the rest of the installation following the onscreen instructions.
+
+* sunnypilot already installed and you installed a version after 0.8.17?
+  1. On the comma three/3X, go to `Settings` ▶️ `Software`.
+  2. At the `Download` option, press `CHECK`. This will fetch the list of latest branches from sunnypilot.
+  3. At the `Target Branch` option, press `SELECT` to open the Target Branch selector.
+  4. Scroll to select the desired branch per  Recommended Branches (see below). Example: `staging`
+
+### Recommended Branches
+|      Branch     |                 Installation URL              |
+|:---------------:|:---------------------------------------------:|
+| `release`       | `https://release.sunnypilot.ai`               |
+| `staging`       | `https://staging.sunnypilot.ai`               |
+|   `dev`         | `https://dev.sunnypilot.ai`                   |
+| `custom-branch` | `https://install.sunnypilot.ai/{branch_name}` |
+
+> [!TIP]
+> You can use sunnypilot/targetbranch as an install URL. Example: 'sunnypilot/staging'.
+
+> [!NOTE]
+> Do you require further assistance with software installation? Join the [sunnypilot community forum](https://community.sunnypilot.ai/new-topic?category=general/qa) and create a topic in the General/Q&A Category channel.
+
+
+<details>
+
+<summary>Older legacy branches</summary>
+
+### If you want to use our older legacy branches (*not recommended*)
+
+> [**IMPORTANT**]
+> It is recommended to [re-flash AGNOS](https://flash.comma.ai/) if you intend to downgrade from the new branches.
+> You can still restore the latest sunnylink backup made on the old branches.
+
+|    Branch    |         Installation URL         |
+|:------------:|:--------------------------------:|
+| `release-c3` | https://release-c3.sunnypilot.ai |
+| `staging-c3` | https://staging-c3.sunnypilot.ai |
+|   `dev-c3`   | https://dev-c3.sunnypilot.ai     |
+
+</details>
+
 
 ## 🎆 Pull Requests
 We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.

RELEASES.md

@@ -1,33 +1,3 @@
-Version 0.11.2 (2026-06-15)
-========================
-
-
-Version 0.11.1 (2026-05-18)
-========================
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Improved thermal policy for comma four
-* Acura MDX 2022-24 support thanks to mvl-boston!
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-Version 0.11.0 (2026-03-17)
-========================
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-
-Version 0.10.3 (2025-12-17)
-========================
-* New driving model #36249
-  * New temporal policy architecture
-  * New on-policy training physics noise model
-* New driver monitoring model #36409
-  * Trained on a new dataset, including comma four data
-* Improved inter-process communication memory efficiency
-
 Version 0.10.2 (2025-11-19)
 ========================
 * comma four support

SConstruct

@@ -4,34 +4,33 @@ import sys
 import sysconfig
 import platform
 import shlex
-import importlib
 import numpy as np
 
 import SCons.Errors
-from SCons.Defaults import _stripixes
-
-TICI = os.path.isfile('/TICI')
 
 SCons.Warnings.warningAsException(True)
 
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2))))
+SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
 
+AddOption('--kaitai', action='store_true', help='Regenerate kaitai struct parsers')
+AddOption('--asan', action='store_true', help='turn on ASAN')
+AddOption('--ubsan', action='store_true', help='turn on UBSan')
+AddOption('--mutation', action='store_true', help='generate mutation-ready code')
 AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
-AddOption('--verbose', action='store_true', default=False, help='show full build commands')
-release = not os.path.exists(File('#.gitattributes').abspath) # file absent on release branch, see release_files.py
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=(not TICI and not release),
+          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
 # Detect platform
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and TICI:
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and os.path.isfile('/TICI'):
   arch = "larch64"
 assert arch in [
   "larch64",  # linux tici arm64
@@ -40,69 +39,23 @@ assert arch in [
   "Darwin",   # macOS arm64 (x86 not supported)
 ]
 
-pkg_names = ['acados', 'bzip2', 'capnproto', 'catch2', 'eigen', 'ffmpeg', 'json11', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
-pkgs = [importlib.import_module(name) for name in pkg_names]
-acados = pkgs[pkg_names.index('acados')]
-acados_include_dirs = [
-  acados.INCLUDE_DIR,
-  os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
-  os.path.join(acados.INCLUDE_DIR, "hpipm", "include"),
-]
-
-
-# ***** enforce a whitelist of system libraries *****
-# this prevents silently relying on a 3rd party package,
-# e.g. apt-installed libusb. all libraries should either
-# be distributed with all Linux distros and macOS, or
-# vendored in commaai/dependencies.
-allowed_system_libs = {
-  "EGL", "GLESv2", "GL",
-  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
-  "dl", "drm", "gbm", "m", "pthread",
-}
-
-def _resolve_lib(env, name):
-  for d in env.Flatten(env.get('LIBPATH', [])):
-    p = Dir(str(d)).abspath
-    for ext in ('.a', '.so', '.dylib'):
-      f = File(os.path.join(p, f'lib{name}{ext}'))
-      if f.exists() or f.has_builder():
-        return name
-  if name in allowed_system_libs:
-    return name
-  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
-
-def _libflags(target, source, env, for_signature):
-  libs = []
-  lp = env.subst('$LIBLITERALPREFIX')
-  for lib in env.Flatten(env.get('LIBS', [])):
-    if isinstance(lib, str):
-      if os.sep in lib or lib.startswith('#'):
-        libs.append(File(lib))
-      elif lib.startswith('-') or (lp and lib.startswith(lp)):
-        libs.append(lib)
-      else:
-        libs.append(_resolve_lib(env, lib))
-    else:
-      libs.append(lib)
-  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
-                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
-
 env = Environment(
   ENV={
     "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath,
+    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": acados.DIR,
+    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
+    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": acados.TERA_PATH
+    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
   },
+  CC='clang',
+  CXX='clang++',
   CCFLAGS=[
     "-g",
     "-fPIC",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
+    "-Wshadow",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
@@ -114,15 +67,24 @@ env = Environment(
   CPPPATH=[
     "#",
     "#msgq",
-    acados_include_dirs,
+    "#third_party",
-    [x.INCLUDE_DIR for x in pkgs],
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/acados/include",
+    "#third_party/acados/include/blasfeo/include",
+    "#third_party/acados/include/hpipm/include",
+    "#third_party/catch2/include",
+    "#third_party/libyuv/include",
+    "#third_party/snpe/include",
   ],
   LIBPATH=[
     "#common",
     "#msgq_repo",
+    "#third_party",
     "#selfdrive/pandad",
     "#rednose/helpers",
-    [x.LIB_DIR for x in pkgs],
+    f"#third_party/libyuv/{arch}/lib",
+    f"#third_party/acados/{arch}/lib",
   ],
   RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
@@ -131,25 +93,53 @@ env = Environment(
   tools=["default", "cython", "compilation_db", "rednose_filter"],
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
-if arch != "larch64":
-  env['_LIBFLAGS'] = _libflags
 
 # Arch-specific flags and paths
 if arch == "larch64":
-  env["CC"] = "clang"
+  env.Append(CPPPATH=["#third_party/opencl/include"])
-  env["CXX"] = "clang++"
   env.Append(LIBPATH=[
+    "/usr/local/lib",
+    "/system/vendor/lib64",
     "/usr/lib/aarch64-linux-gnu",
+    "#third_party/snpe/larch64",
   ])
   arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
   env.Append(CCFLAGS=arch_flags)
   env.Append(CXXFLAGS=arch_flags)
 elif arch == "Darwin":
   env.Append(LIBPATH=[
+    f"{brew_prefix}/lib",
+    f"{brew_prefix}/opt/openssl@3.0/lib",
+    f"{brew_prefix}/opt/llvm/lib/c++",
     "/System/Library/Frameworks/OpenGL.framework/Libraries",
   ])
   env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
   env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CPPPATH=[
+    f"{brew_prefix}/include",
+    f"{brew_prefix}/opt/openssl@3.0/include",
+  ])
+else:
+  env.Append(LIBPATH=[
+    "/usr/lib",
+    "/usr/local/lib",
+  ])
+
+  if arch == "x86_64":
+    env.Append(LIBPATH=[
+      f"#third_party/snpe/{arch}"
+    ])
+    env.Append(RPATH=[
+      Dir(f"#third_party/snpe/{arch}").abspath,
+    ])
+
+# Sanitizers and extra CCFLAGS from CLI
+if GetOption('asan'):
+  env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
+  env.Append(LINKFLAGS=["-fsanitize=address"])
+elif GetOption('ubsan'):
+  env.Append(CCFLAGS=["-fsanitize=undefined"])
+  env.Append(LINKFLAGS=["-fsanitize=undefined"])
 
 _extra_cc = shlex.split(GetOption('ccflags') or '')
 if _extra_cc:
@@ -159,26 +149,21 @@ if _extra_cc:
 if arch != "Darwin":
   env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
 
-# Shorter build output: show brief descriptions instead of full commands.
+# progress output
-# Full command lines are still printed on failure by scons.
+node_interval = 5
-if not GetOption('verbose'):
+node_count = 0
-  for action, short in (
+def progress_function(node):
-    ("CC",     "CC"),
+  global node_count
-    ("CXX",    "CXX"),
+  node_count += node_interval
-    ("LINK",   "LINK"),
+  sys.stderr.write("progress: %d\n" % node_count)
-    ("SHCC",   "CC"),
+if os.environ.get('SCONS_PROGRESS'):
-    ("SHCXX",  "CXX"),
+  Progress(progress_function, interval=node_interval)
-    ("SHLINK", "LINK"),
-    ("AR",     "AR"),
-    ("RANLIB", "RANLIB"),
-    ("AS",     "AS"),
-  ):
-    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
 
 # ********** Cython build environment **********
+py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
-envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
+envCython["CPPPATH"] += [py_include, np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
@@ -190,24 +175,14 @@ else:
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch', 'acados', 'release')
+Export('env', 'arch')
 
 # Setup cache dir
 default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
 cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
-cache_size_limit = 4e9 if "CI" in os.environ else 2e9
 CacheDir(cache_dir)
 Clean(["."], cache_dir)
 
-def prune_cache_dir(target=None, source=None, env=None):
-  cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime)
-  cache_size = sum(os.path.getsize(f) for f in cache_files)
-  for f in cache_files:
-    if cache_size < cache_size_limit:
-      break
-    cache_size -= os.path.getsize(f)
-    os.unlink(f)
-
 # ********** start building stuff **********
 
 # Build common module
@@ -221,6 +196,7 @@ Export('common')
 env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
 
 SConscript(['cereal/SConscript'])
 
@@ -237,65 +213,24 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
+  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
 
-# Build selfdrive
+# Build openpilot
-SConscript([
+SConscript(['third_party/SConscript'])
-  'selfdrive/pandad/SConscript',
+
-  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
+SConscript(['selfdrive/SConscript'])
-  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'selfdrive/locationd/SConscript',
-  'selfdrive/modeld/SConscript',
-  'selfdrive/ui/SConscript',
-])
 
 SConscript(['sunnypilot/SConscript'])
 
-# Build desktop-only tools
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
-if GetOption('extras') and arch != "larch64":
+  SConscript(['tools/replay/SConscript'])
-  SConscript([
+  if arch != "larch64":
-    'tools/replay/SConscript',
+    SConscript(['tools/cabana/SConscript'])
-    'tools/cabana/SConscript',
-    'tools/jotpluggler/SConscript',
-  ])
 
 
 env.CompilationDatabase('compile_commands.json')
-
-# progress output
-def count_scons_nodes(nodes):
-  seen = set()
-  stack = list(nodes)
-
-  while stack:
-    node = stack.pop().disambiguate()
-    if node in seen:
-      continue
-    seen.add(node)
-    executor = node.get_executor()
-    if executor is not None:
-      stack += executor.get_all_prerequisites() + executor.get_all_children()
-
-  return len(seen)
-
-progress_interval = 5
-progress_count = 0
-progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
-
-def progress_function(node):
-  global progress_count
-  if progress_count >= progress_total:
-    return
-  progress_count = min(progress_count + progress_interval, progress_total)
-  progress = round(100. * progress_count / progress_total, 1)
-  sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
-  if progress == 100. and sys.stderr.isatty():
-    sys.stderr.write("\n")
-  sys.stderr.flush()
-
-Progress(progress_function, interval=progress_interval)
-AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)

cereal/SConscript

@@ -4,7 +4,7 @@ cereal_dir = Dir('.')
 gen_dir = Dir('gen')
 
 # Build cereal
-schema_files = ['log.capnp', 'car.capnp', 'deprecated.capnp', 'custom.capnp']
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
 env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
             schema_files,
             f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
@@ -13,7 +13,7 @@ cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
 socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 

cereal/custom.capnp

@@ -153,8 +153,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
       navigation @1;
       vision @2;
       policy @3;
-      offPolicy @4;
-      onPolicy @5;
     }
   }
 

cereal/legacy.capnp

@@ -3,7 +3,7 @@ $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
 
-# deprecated.capnp: a home for deprecated structs
+# legacy.capnp: a home for deprecated structs
 
 struct LogRotate @0x9811e1f38f62f2d1 {
   segmentNum @0 :Int32;
@@ -571,219 +571,4 @@ struct LidarPts @0xe3d6685d4e9d8f7a {
   pkt @4 :Data;
 }
 
-struct LiveTracksDEPRECATED @0xb16f60103159415a {
-  trackId @0 :Int32;
-  dRel @1 :Float32;
-  yRel @2 :Float32;
-  vRel @3 :Float32;
-  aRel @4 :Float32;
-  timeStamp @5 :Float32;
-  status @6 :Float32;
-  currentTime @7 :Float32;
-  stationary @8 :Bool;
-  oncoming @9 :Bool;
-}
 
-struct LiveMpcData @0x92a5e332a85f32a0 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  psi @2 :List(Float32);
-  curvature @3 :List(Float32);
-  qpIterations @4 :UInt32;
-  calculationTime @5 :UInt64;
-  cost @6 :Float64;
-}
-
-struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
-  xEgo @0 :List(Float32);
-  vEgo @1 :List(Float32);
-  aEgo @2 :List(Float32);
-  xLead @3 :List(Float32);
-  vLead @4 :List(Float32);
-  aLead @5 :List(Float32);
-  aLeadTau @6 :Float32;    # lead accel time constant
-  qpIterations @7 :UInt32;
-  mpcId @8 :UInt32;
-  calculationTime @9 :UInt64;
-  cost @10 :Float64;
-}
-
-struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
-  frameId @0 :UInt32;
-  modelExecutionTime @14 :Float32;
-  dspExecutionTime @16 :Float32;
-  rawPredictions @15 :Data;
-
-  faceOrientation @3 :List(Float32);
-  facePosition @4 :List(Float32);
-  faceProb @5 :Float32;
-  leftEyeProb @6 :Float32;
-  rightEyeProb @7 :Float32;
-  leftBlinkProb @8 :Float32;
-  rightBlinkProb @9 :Float32;
-  faceOrientationStd @11 :List(Float32);
-  facePositionStd @12 :List(Float32);
-  sunglassesProb @13 :Float32;
-  poorVision @17 :Float32;
-  partialFace @18 :Float32;
-  distractedPose @19 :Float32;
-  distractedEyes @20 :Float32;
-  eyesOnRoad @21 :Float32;
-  phoneUse @22 :Float32;
-  occludedProb @23 :Float32;
-
-  readyProb @24 :List(Float32);
-  notReadyProb @25 :List(Float32);
-
-  irPwrDEPRECATED @10 :Float32;
-  descriptorDEPRECATED @1 :List(Float32);
-  stdDEPRECATED @2 :Float32;
-}
-
-struct NavModelData @0xac3de5c437be057a {
-  frameId @0 :UInt32;
-  locationMonoTime @6 :UInt64;
-  modelExecutionTime @1 :Float32;
-  dspExecutionTime @2 :Float32;
-  features @3 :List(Float32);
-  # predicted future position
-  position @4 :XYData;
-  desirePrediction @5 :List(Float32);
-
-  # All SI units and in device frame
-  struct XYData @0xbe09e615b2507e26 {
-    x @0 :List(Float32);
-    y @1 :List(Float32);
-    xStd @2 :List(Float32);
-    yStd @3 :List(Float32);
-  }
-}
-
-struct AndroidBuildInfo @0xfe2919d5c21f426c {
-  board @0 :Text;
-  bootloader @1 :Text;
-  brand @2 :Text;
-  device @3 :Text;
-  display @4 :Text;
-  fingerprint @5 :Text;
-  hardware @6 :Text;
-  host @7 :Text;
-  id @8 :Text;
-  manufacturer @9 :Text;
-  model @10 :Text;
-  product @11 :Text;
-  radioVersion @12 :Text;
-  serial @13 :Text;
-  supportedAbis @14 :List(Text);
-  tags @15 :Text;
-  time @16 :Int64;
-  type @17 :Text;
-  user @18 :Text;
-
-  versionCodename @19 :Text;
-  versionRelease @20 :Text;
-  versionSdk @21 :Int32;
-  versionSecurityPatch @22 :Text;
-}
-
-struct AndroidSensor @0x9b513b93a887dbcd {
-  id @0 :Int32;
-  name @1 :Text;
-  vendor @2 :Text;
-  version @3 :Int32;
-  handle @4 :Int32;
-  type @5 :Int32;
-  maxRange @6 :Float32;
-  resolution @7 :Float32;
-  power @8 :Float32;
-  minDelay @9 :Int32;
-  fifoReservedEventCount @10 :UInt32;
-  fifoMaxEventCount @11 :UInt32;
-  stringType @12 :Text;
-  maxDelay @13 :Int32;
-}
-
-struct IosBuildInfo @0xd97e3b28239f5580 {
-  appVersion @0 :Text;
-  appBuild @1 :UInt32;
-  osVersion @2 :Text;
-  deviceModel @3 :Text;
-}
-
-enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
-  unknown @0;
-  neo @1;
-  chffrAndroid @2;
-  front @3;
-}
-
-struct AndroidCaptureResult @0xbcc3efbac41d2048 {
-  sensitivity @0 :Int32;
-  frameDuration @1 :Int64;
-  exposureTime @2 :Int64;
-  rollingShutterSkew @3 :UInt64;
-  colorCorrectionTransform @4 :List(Int32);
-  colorCorrectionGains @5 :List(Float32);
-  displayRotation @6 :Int8;
-}
-
-enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
-  none @0;
-  client @1;
-  cdp @2;
-  dcp @3;
-}
-
-struct LateralINDIState @0x939463348632375e {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  steeringRateDeg @2 :Float32;
-  steeringAccelDeg @3 :Float32;
-  rateSetPoint @4 :Float32;
-  accelSetPoint @5 :Float32;
-  accelError @6 :Float32;
-  delayedOutput @7 :Float32;
-  delta @8 :Float32;
-  output @9 :Float32;
-  saturated @10 :Bool;
-  steeringAngleDesiredDeg @11 :Float32;
-  steeringRateDesiredDeg @12 :Float32;
-}
-
-struct LateralLQRState @0x9024e2d790c82ade {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  i @2 :Float32;
-  output @3 :Float32;
-  lqrOutput @4 :Float32;
-  saturated @5 :Bool;
-  steeringAngleDesiredDeg @6 :Float32;
-}
-
-struct LateralCurvatureState @0xad9d8095c06f7c61 {
-  active @0 :Bool;
-  actualCurvature @1 :Float32;
-  desiredCurvature @2 :Float32;
-  error @3 :Float32;
-  p @4 :Float32;
-  i @5 :Float32;
-  f @6 :Float32;
-  output @7 :Float32;
-  saturated @8 :Bool;
-}
-
-struct LateralPlannerSolution @0x84caeca5a6b4acfe {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  yaw @2 :List(Float32);
-  yawRate @3 :List(Float32);
-  xStd @4 :List(Float32);
-  yStd @5 :List(Float32);
-  yawStd @6 :List(Float32);
-  yawRateStd @7 :List(Float32);
-}
-
-struct GpsTrajectory @0x8cfeb072f5301000 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-}

cereal/messaging/__init__.py

@@ -1,8 +1,10 @@
 # must be built with scons
-from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
 import msgq
+
 import os
 import capnp
 import time
@@ -11,25 +13,11 @@ from typing import Optional, List, Union, Dict
 
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def pub_sock(endpoint: str) -> PubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.pub_sock(endpoint, segment_size)
-
-
-def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
-                       timeout=timeout, segment_size=segment_size)
-
-
 def reset_context():
   msgq.context = Context()
 
@@ -259,11 +247,11 @@ class PubMaster:
     self.sock[s].send(dat)
 
   def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
-    try:
+    for _ in range(int(timeout*(1./dt))):
-      self.sock[s].wait_for_readers(timeout=timeout, interval=dt)
+      if self.sock[s].all_readers_updated():
-      return True
+        return True
-    except TimeoutError:
+      time.sleep(dt)
-      return False
+    return False
 
   def all_readers_updated(self, s: str) -> bool:
-    return self.sock[s].all_readers_updated()
+    return self.sock[s].all_readers_updated()  # type: ignore

cereal/messaging/bridge.cc

@@ -25,16 +25,15 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<BridgeZmqPoller>();
+  auto poller = std::make_unique<ZMQPoller>();
-  auto pub_context = std::make_unique<Context>();
+  auto pub_context = std::make_unique<MSGQContext>();
-  auto sub_context = std::make_unique<BridgeZmqContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
-  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
+  std::map<SubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new PubSocket();
+    auto pub_sock = new MSGQPubSocket();
-    auto sub_sock = new BridgeZmqSubSocket();
+    auto sub_sock = new ZMQSubSocket();
-    size_t queue_size = services.at(endpoint).queue_size;
+    pub_sock->connect(pub_context.get(), endpoint);
-    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/bridge_zmq.cc

@@ -1,170 +0,0 @@
-#include "cereal/messaging/bridge_zmq.h"
-
-#include <cassert>
-#include <cstring>
-#include <unistd.h>
-
-static size_t fnv1a_hash(const std::string &str) {
-  const size_t fnv_prime = 0x100000001b3;
-  size_t hash_value = 0xcbf29ce484222325;
-  for (char c : str) {
-    hash_value ^= (unsigned char)c;
-    hash_value *= fnv_prime;
-  }
-  return hash_value;
-}
-
-// FIXME: This is a hack to get the port number from the socket name, might have collisions.
-static int get_port(std::string endpoint) {
-  size_t hash_value = fnv1a_hash(endpoint);
-  int start_port = 8023;
-  int max_port = 65535;
-  return start_port + (hash_value % (max_port - start_port));
-}
-
-BridgeZmqContext::BridgeZmqContext() {
-  context = zmq_ctx_new();
-}
-
-BridgeZmqContext::~BridgeZmqContext() {
-  if (context != nullptr) {
-    zmq_ctx_term(context);
-  }
-}
-
-void BridgeZmqMessage::init(size_t sz) {
-  size = sz;
-  data = new char[size];
-}
-
-void BridgeZmqMessage::init(char *d, size_t sz) {
-  size = sz;
-  data = new char[size];
-  memcpy(data, d, size);
-}
-
-void BridgeZmqMessage::close() {
-  if (size > 0) {
-    delete[] data;
-  }
-  data = nullptr;
-  size = 0;
-}
-
-BridgeZmqMessage::~BridgeZmqMessage() {
-  close();
-}
-
-int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
-
-  if (conflate) {
-    int arg = 1;
-    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
-  }
-
-  int reconnect_ivl = 500;
-  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
-
-  full_endpoint = "tcp://" + address + ":";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  return zmq_connect(sock, full_endpoint.c_str());
-}
-
-void BridgeZmqSubSocket::setTimeout(int timeout) {
-  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
-}
-
-Message *BridgeZmqSubSocket::receive(bool non_blocking) {
-  zmq_msg_t msg;
-  assert(zmq_msg_init(&msg) == 0);
-
-  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
-  int rc = zmq_msg_recv(&msg, sock, flags);
-
-  Message *ret = nullptr;
-  if (rc >= 0) {
-    ret = new BridgeZmqMessage;
-    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
-  }
-
-  zmq_msg_close(&msg);
-  return ret;
-}
-
-BridgeZmqSubSocket::~BridgeZmqSubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  full_endpoint = "tcp://*:";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
-  pid = getpid();
-
-  return zmq_bind(sock, full_endpoint.c_str());
-}
-
-int BridgeZmqPubSocket::sendMessage(Message *message) {
-  assert(pid == getpid());
-  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
-}
-
-int BridgeZmqPubSocket::send(char *data, size_t size) {
-  assert(pid == getpid());
-  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
-}
-
-BridgeZmqPubSocket::~BridgeZmqPubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
-  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
-  polls[num_polls].socket = socket->getRawSocket();
-  polls[num_polls].events = ZMQ_POLLIN;
-
-  sockets.push_back(socket);
-  num_polls++;
-}
-
-std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
-  std::vector<BridgeZmqSubSocket *> ret;
-
-  int rc = zmq_poll(polls, num_polls, timeout);
-  if (rc < 0) {
-    return ret;
-  }
-
-  for (size_t i = 0; i < num_polls; i++) {
-    if (polls[i].revents) {
-      ret.push_back(sockets[i]);
-    }
-  }
-
-  return ret;
-}

cereal/messaging/bridge_zmq.h

@@ -1,72 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <string>
-#include <vector>
-
-#include <zmq.h>
-
-#include "msgq/ipc.h"
-
-class BridgeZmqContext {
-public:
-  BridgeZmqContext();
-  void *getRawContext() { return context; }
-  ~BridgeZmqContext();
-
-private:
-  void *context = nullptr;
-};
-
-class BridgeZmqMessage : public Message {
-public:
-  void init(size_t size);
-  void init(char *data, size_t size);
-  void close();
-  size_t getSize() { return size; }
-  char *getData() { return data; }
-  ~BridgeZmqMessage();
-
-private:
-  char *data = nullptr;
-  size_t size = 0;
-};
-
-class BridgeZmqSubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
-  void setTimeout(int timeout);
-  Message *receive(bool non_blocking = false);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqSubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-};
-
-class BridgeZmqPubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
-  int sendMessage(Message *message);
-  int send(char *data, size_t size);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqPubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-  int pid = -1;
-};
-
-class BridgeZmqPoller {
-public:
-  void registerSocket(BridgeZmqSubSocket *socket);
-  std::vector<BridgeZmqSubSocket *> poll(int timeout);
-
-private:
-  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
-  std::vector<BridgeZmqSubSocket *> sockets;
-  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
-  size_t num_polls = 0;
-};

cereal/messaging/msgq_to_zmq.cc

@@ -2,7 +2,6 @@
 
 #include <cassert>
 
-#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -22,14 +21,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
+  zmq_context = std::make_unique<ZMQContext>();
-  msgq_context = std::make_unique<Context>();
+  msgq_context = std::make_unique<MSGQContext>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -49,7 +48,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -72,7 +71,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -109,8 +108,7 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -130,7 +128,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,8 +7,9 @@
 #include <string>
 #include <vector>
 
+#define private public
 #include "msgq/impl_msgq.h"
-#include "cereal/messaging/bridge_zmq.h"
+#include "msgq/impl_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -21,16 +22,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<Context> msgq_context;
+  std::unique_ptr<MSGQContext> msgq_context;
-  std::unique_ptr<BridgeZmqContext> zmq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,8 +187,7 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    service serv = services.at(std::string(name));
+    PubSocket *socket = PubSocket::create(message_context.context(), name);
-    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/messaging/tests/test_messaging.py

@@ -5,7 +5,7 @@ import numbers
 import random
 import threading
 import time
-from openpilot.common.parameterized import parameterized
+from parameterized import parameterized
 import pytest
 
 from cereal import log, car
@@ -30,7 +30,7 @@ def zmq_sleep(t=1):
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "steeringTorque", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:

cereal/messaging/tests/test_services.py

@@ -1,7 +1,7 @@
 import os
 import tempfile
 from typing import Dict
-from openpilot.common.parameterized import parameterized
+from parameterized import parameterized
 
 import cereal.services as services
 from cereal.services import SERVICE_LIST

cereal/messaging/tests/validate_sp_cereal_upstream.py

@@ -1,256 +1,222 @@
 #!/usr/bin/env python3
-"""Schema-level cereal compat check between sunnypilot and upstream openpilot.
-
-Rules (per struct matched across sides by typeId):
-  R1  shared ordinal must reference the same type.
-  R2  sunnypilot-only ordinal in a union -> FAIL (unknown discriminant upstream).
-  R3  sunnypilot-only ordinal on a regular field -> OK (additive struct evolution).
-  R4  upstream-only ordinal -> OK.
-  R5  sunnypilot-only struct referenced via an upstream-shared field -> FAIL.
-"""
-
-from __future__ import annotations
-
 import argparse
-import json
-import os
 import sys
-from typing import Any
+from typing import Any, List, Tuple
 
-NO_DISCRIMINANT = 0xFFFF
+DEBUG = False
 
 
-def hex_id(value: int) -> str:
+def print_debug(string: str) -> None:
-  return f"0x{value:016x}"
+  if DEBUG:
+    print(string)
 
 
-def encode_type(type_node: Any) -> dict:
+def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
-  which = type_node.which()
+  """
-  if which == "struct":
+  Create a new instance of a schema type, handling different field types.
-    return {"kind": "struct", "typeId": hex_id(type_node.struct.typeId)}
+
-  if which == "enum":
+  Args:
-    return {"kind": "enum", "typeId": hex_id(type_node.enum.typeId)}
+      struct: The Cap'n Proto schema structure
-  if which == "interface":
+      prop: A tuple containing the field name and field metadata
-    return {"kind": "interface", "typeId": hex_id(type_node.interface.typeId)}
+
-  if which == "list":
+  Returns:
-    return {"kind": "list", "element": encode_type(type_node.list.elementType)}
+      A new initialized schema instance
-  if which == "anyPointer":
+  """
-    return {"kind": "anyPointer"}
+  struct_instance = struct.new_message()
-  return {"kind": which}
+  field_name, field_metadata = prop
+
+  try:
+    field_type = field_metadata.proto.slot.type.which()
+
+    # Initialize different types of fields
+    if field_type in ('list', 'text', 'data'):
+      struct_instance.init(field_name, 1)
+      print_debug(f"Initialized list/text/data field: {field_name}")
+    elif field_type in ('struct', 'object'):
+      struct_instance.init(field_name)
+      print_debug(f"Initialized struct/object field: {field_name}")
+
+    return struct_instance
+
+  except Exception as e:
+    print(f"Error creating instance for {field_name}: {e}")
+    return None
 
 
-def encode_field(name: str, field: Any) -> dict:
+def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
-  proto = field.proto
+  """
-  ordinal = proto.ordinal.explicit if proto.ordinal.which() == "explicit" else None
+  Retrieve all fields from a given schema structure.
-  discriminant = proto.discriminantValue if proto.discriminantValue != NO_DISCRIMINANT else None
 
-  if proto.which() == "group":
+  Args:
-    type_desc = {"kind": "group", "typeId": hex_id(proto.group.typeId)}
+      schema_struct: The Cap'n Proto schema structure
-  else:
-    type_desc = encode_type(proto.slot.type)
 
-  return {
+  Returns:
-    "name": name,
+      A list of field names and their metadata
-    "ordinal": ordinal,
+  """
-    "discriminant": discriminant,
+  try:
-    "type": type_desc,
+    # Get all fields from the schema
-  }
+    schema_fields = list(schema_struct.schema.fields.items())
+
+    print_debug("Discovered schema fields:")
+    for field_name, field_metadata in schema_fields:
+      print_debug(f"- {field_name}")
+
+    return schema_fields
+
+  except Exception as e:
+    print(f"Error retrieving schema fields: {e}")
+    return []
 
 
-def encode_struct(schema: Any) -> dict:
+def generate_schema_instances(schema_struct: Any) -> List[Any]:
-  node = schema.node
+  """
-  return {
+  Generate instances for all fields in a given schema.
-    "typeId": hex_id(node.id),
+
-    "displayName": node.displayName,
+  Args:
-    "hasUnion": node.struct.discriminantCount > 0,
+      schema_struct: The Cap'n Proto schema structure
-    "fields": [encode_field(name, field) for name, field in schema.fields.items()],
+
-  }
+  Returns:
+      A list of schema instances
+  """
+  schema_fields = get_schema_fields(schema_struct)
+  instances = []
+
+  for field_prop in schema_fields:
+    try:
+      instance = create_schema_instance(schema_struct, field_prop)
+      if instance is not None:
+        instances.append(instance)
+    except Exception as e:
+      print(f"Skipping field due to error: {e}")
+
+  print(f"Generated {len(instances)} schema instances")
+  return instances
 
 
-def _child_struct_schema(field: Any) -> Any:
+def persist_instances(instances: List[Any], filename: str) -> None:
-  proto = field.proto
+  """
-  if proto.which() == "group":
+  Write schema instances to a binary file.
-    return field.schema
+
-  type_node = proto.slot.type
+  Args:
-  which = type_node.which()
+      instances: List of schema instances
-  if which == "struct":
+      filename: Output file path
-    return field.schema
+  """
-  if which == "list":
+  try:
-    container = field.schema
+    with open(filename, 'wb') as f:
-    element_type = type_node.list.elementType
+      for instance in instances:
-    while element_type.which() == "list":
+        f.write(instance.to_bytes())
-      container = container.elementType
+
-      element_type = element_type.list.elementType
+    print(f"Successfully wrote {len(instances)} instances to {filename}")
-    if element_type.which() == "struct":
+
-      return container.elementType
+  except Exception as e:
-  return None
+    print(f"Error persisting instances: {e}")
+    sys.exit(1)
 
 
-def collect_schema(root: Any) -> dict[str, dict]:
+def read_instances(filename: str, schema_type: Any) -> List[Any]:
-  structs: dict[str, dict] = {}
+  """
-  stack = [root]
+  Read schema instances from a binary file.
-  while stack:
+
-    schema = stack.pop()
+  Args:
-    type_id = hex_id(schema.node.id)
+      filename: Input file path
-    if type_id in structs:
+      schema_type: The schema type to use for reading
-      continue
+
-    structs[type_id] = encode_struct(schema)
+  Returns:
-    for _name, field in schema.fields.items():
+      A list of read schema instances
-      try:
+  """
-        child = _child_struct_schema(field)
+  try:
-      except Exception:
+    with open(filename, 'rb') as f:
-        child = None
+      data = f.read()
-      if child is not None:
+
-        stack.append(child)
+    instances = list(schema_type.read_multiple_bytes(data))
-  return structs
+
+    print(f"Read {len(instances)} instances from {filename}")
+    return instances
+
+  except Exception as e:
+    print(f"Error reading instances: {e}")
+    sys.exit(1)
 
 
-def load_log(cereal_dir: str) -> Any:
+def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
-  import capnp
+  """
-  cereal_dir = os.path.abspath(cereal_dir)
+  Compare original and read-back instances to detect potential breaking changes.
-  capnp.remove_import_hook()
-  return capnp.load(os.path.join(cereal_dir, "log.capnp"), imports=[cereal_dir])
 
+  Args:
+      original_instances: List of originally generated instances
+      read_instances: List of instances read back from file
 
-def dump_schema(cereal_dir: str, path: str) -> None:
+  Returns:
-  log = load_log(cereal_dir)
+      Boolean indicating whether schemas appear compatible
-  payload = {
+  """
-    "root": hex_id(log.Event.schema.node.id),
+  if len(original_instances) != len(read_instances):
-    "structs": collect_schema(log.Event.schema),
+    print("❌ Schema Compatibility Warning: Instance count mismatch")
-  }
-  with open(path, "w", encoding="utf-8") as handle:
-    json.dump(payload, handle, indent=2, sort_keys=True)
-  print(f"wrote schema dump with {len(payload['structs'])} structs to {path}")
-
-
-def types_equal(a: dict, b: dict) -> bool:
-  if a.get("kind") != b.get("kind"):
     return False
-  kind = a["kind"]
+
-  if kind in ("struct", "enum", "interface", "group"):
+  compatible = True
-    return a.get("typeId") == b.get("typeId")
+  for struct in read_instances:
-  if kind == "list":
+    try:
-    return types_equal(a["element"], b["element"])
+      getattr(struct, struct.which())  # Attempting to access the field to validate readability
-  return True
+    except Exception as e:
+      print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
+      compatible = False
+
+  return compatible
 
 
-def type_repr(t: dict) -> str:
+def main():
-  kind = t.get("kind", "?")
+  """
-  if kind in ("struct", "enum", "interface", "group"):
+  CLI entry point for schema compatibility testing.
-    return f"{kind}({t.get('typeId')})"
+  """
-  if kind == "list":
+  # Setup argument parser
-    return f"list<{type_repr(t['element'])}>"
-  return kind
-
-
-def field_is_union_variant(field: dict) -> bool:
-  return field.get("discriminant") is not None
-
-
-def index_fields_by_ordinal(struct: dict) -> dict[int, dict]:
-  indexed: dict[int, dict] = {}
-  for field in struct["fields"]:
-    ordinal = field.get("ordinal")
-    if ordinal is None:
-      continue
-    indexed[ordinal] = field
-  return indexed
-
-
-def compare(sunnypilot_dump: dict, upstream_dump: dict) -> list[str]:
-  violations: list[str] = []
-  sunnypilot_structs: dict[str, dict] = sunnypilot_dump["structs"]
-  upstream_structs: dict[str, dict] = upstream_dump["structs"]
-
-  sunnypilot_struct_referenced_from_shared: set[str] = set()
-
-  for type_id, sunnypilot_struct in sunnypilot_structs.items():
-    upstream_struct = upstream_structs.get(type_id)
-    if upstream_struct is None:
-      continue
-
-    sunnypilot_fields = index_fields_by_ordinal(sunnypilot_struct)
-    upstream_fields = index_fields_by_ordinal(upstream_struct)
-    display = sunnypilot_struct["displayName"]
-
-    for ordinal, sunnypilot_field in sunnypilot_fields.items():
-      upstream_field = upstream_fields.get(ordinal)
-      if upstream_field is None:
-        if field_is_union_variant(sunnypilot_field):
-          violations.append(
-            f"[R2] {display} @{ordinal} ('{sunnypilot_field['name']}', {type_repr(sunnypilot_field['type'])}): "
-            f"union variant not present upstream. upstream cannot parse this discriminant."
-          )
-        continue
-
-      if not types_equal(sunnypilot_field["type"], upstream_field["type"]):
-        violations.append(
-          f"[R1] {display} @{ordinal}: type mismatch. "
-          f"sunnypilot='{sunnypilot_field['name']}' {type_repr(sunnypilot_field['type'])} vs "
-          f"upstream='{upstream_field['name']}' {type_repr(upstream_field['type'])}."
-        )
-        continue
-
-      cursor = sunnypilot_field["type"]
-      while cursor.get("kind") == "list":
-        cursor = cursor["element"]
-      if cursor.get("kind") in ("struct", "group", "interface") and cursor.get("typeId"):
-        sunnypilot_struct_referenced_from_shared.add(cursor["typeId"])
-
-  for type_id, sunnypilot_struct in sunnypilot_structs.items():
-    if type_id in upstream_structs:
-      continue
-    if type_id in sunnypilot_struct_referenced_from_shared:
-      violations.append(
-        f"[R5] struct {sunnypilot_struct['displayName']} ({type_id}) exists only on sunnypilot "
-        f"but is referenced from an upstream-shared field. upstream cannot resolve this type."
-      )
-
-  return violations
-
-
-def load_peer(path: str) -> dict:
-  with open(path, "r", encoding="utf-8") as handle:
-    return json.load(handle)
-
-
-def run_read(cereal_dir: str, peer_path: str) -> int:
-  log = load_log(cereal_dir)
-  peer_dump = load_peer(peer_path)
-  local_dump = {
-    "root": hex_id(log.Event.schema.node.id),
-    "structs": collect_schema(log.Event.schema),
-  }
-  violations = compare(sunnypilot_dump=peer_dump, upstream_dump=local_dump)
-
-  if not violations:
-    print("cereal compat OK: upstream openpilot can parse sunnypilot routes "
-          "(no leaked structs, no ordinal collisions).")
-    return 0
-
-  print(f"cereal compat FAIL: upstream openpilot would misparse sunnypilot routes "
-        f"({len(violations)} violation(s)):")
-  for v in violations:
-    print(f"  {v}")
-  return 1
-
-
-def main() -> int:
   parser = argparse.ArgumentParser(
-    description="sunnypilot <-> upstream cereal compatibility validator (schema-level)."
+    description='Cap\'n Proto Schema Compatibility Testing Tool',
+    epilog='Test schema compatibility by generating and reading back instances.'
   )
-  mode = parser.add_mutually_exclusive_group(required=True)
+
-  mode.add_argument("-g", "--generate", action="store_true", help="dump local schema to JSON")
+  # Add mutually exclusive group for generation or reading mode
-  mode.add_argument("-r", "--read", action="store_true", help="load peer JSON and diff against local")
+  mode_group = parser.add_mutually_exclusive_group(required=True)
-  parser.add_argument("-f", "--file", default="schema.json", help="JSON file path (default: schema.json)")
+  mode_group.add_argument('-g', '--generate', action='store_true',
-  parser.add_argument("--cereal-dir", required=True, help="path to cereal directory containing log.capnp")
+                          help='Generate schema instances')
+  mode_group.add_argument('-r', '--read', action='store_true',
+                          help='Read and validate schema instances')
+
+  # Common arguments
+  parser.add_argument('-f', '--file',
+                      default='schema_instances.bin',
+                      help='Output/input binary file (default: schema_instances.bin)')
+
+  # Parse arguments
   args = parser.parse_args()
 
+  # Import the schema dynamically 
+  try:
+    from cereal import log
+    schema_type = log.Event
+  except ImportError:
+    print("Error: Unable to import schema. Ensure 'cereal' is installed.")
+    sys.exit(1)
+
+  # Execute based on mode
   if args.generate:
-    dump_schema(args.cereal_dir, args.file)
+    print("🔧 Generating Schema Instances")
-    return 0
+    instances = generate_schema_instances(schema_type)
-  return run_read(args.cereal_dir, args.file)
+    persist_instances(instances, args.file)
+    print("✅ Instance generation complete")
+
+  elif args.read:
+    print("🔍 Reading and Validating Schema Instances")
+    generated_instances = generate_schema_instances(schema_type)
+    read_back_instances = read_instances(args.file, schema_type)
+
+    # Compare schemas
+    if compare_schemas(generated_instances, read_back_instances):
+      print("✅ Schema Compatibility: No breaking changes detected")
+      sys.exit(0)
+    else:
+      print("❌ Potential Schema Breaking Changes Detected")
+      sys.exit(1)
 
 
 if __name__ == "__main__":
-  sys.exit(main())
+  main()

cereal/services.py

@@ -1,43 +1,39 @@
 #!/usr/bin/env python3
-from enum import IntEnum
 from typing import Optional
 
 
-# TODO: this should be automatically determined using the capnp schema
-class QueueSize(IntEnum):
-  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
-  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
-  SMALL = 250 * 1024          # 250KB - most services
-
-
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
-               queue_size: QueueSize = QueueSize.SMALL):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
-    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
+  "accelerometer2": (True, 100., 100),
+  "magnetometer": (True, 25.),
+  "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
+  "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
   "touch": (True, 20., 1),
-  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
+  "controlsState": (True, 100., 10),
   "selfdriveState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
+  "sendcan": (True, 100., 139),
-  "logMessage": (True, 0., None, QueueSize.BIG),
+  "logMessage": (True, 0.),
-  "errorLogMessage": (True, 0., 1, QueueSize.BIG),
+  "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
   "liveDelay": (True, 4., 1),
@@ -46,13 +42,13 @@ _services: dict[str, tuple] = {
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
-  "lateralManeuverPlan": (True, 20.),
   "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15, QueueSize.BIG),
+  "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
   "qcomGnss": (True, 2.),
+  "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
   "livePose": (True, 20., 4),
@@ -69,22 +65,22 @@ _services: dict[str, tuple] = {
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., None, QueueSize.BIG),
+  "modelV2": (True, 20.),
   "managerState": (True, 2., 1),
+  "uploaderState": (True, 0., 1),
+  "navInstruction": (True, 1., 10),
+  "navRoute": (True, 0.),
+  "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
   "userBookmark": (True, 0., 1),
   "soundPressure": (True, 10., 10),
   "rawAudioData": (False, 20.),
   "bookmarkButton": (True, 0., 1),
   "audioFeedback": (True, 0., 1),
-  "roadEncodeData": (False, 20., None, QueueSize.BIG),
-  "driverEncodeData": (False, 20., None, QueueSize.BIG),
-  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
 
   # sunnypilot
-  "modelManagerSP": (False, 1., 1, QueueSize.BIG),
+  "modelManagerSP": (False, 1., 1),
-  "backupManagerSP": (False, 1., 1, QueueSize.BIG),
+  "backupManagerSP": (False, 1., 1),
   "selfdriveStateSP": (True, 100., 10),
   "longitudinalPlanSP": (True, 20., 10),
   "onroadEventsSP": (True, 1., 1),
@@ -92,20 +88,26 @@ _services: dict[str, tuple] = {
   "carControlSP": (True, 100., 10),
   "carStateSP": (True, 100., 10),
   "liveMapDataSP": (True, 1., 1),
-  "modelDataV2SP": (True, 20., None, QueueSize.BIG),
+  "modelDataV2SP": (True, 20.),
   "liveLocationKalman": (True, 20.),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamWideRoadEncodeData": (False, 20.),
-  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamRoadEncodeData": (False, 20.),
-  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamDriverEncodeData": (False, 20.),
   "customReservedRawData0": (True, 0.),
+  "customReservedRawData1": (True, 0.),
+  "customReservedRawData2": (True, 0.),
 }
 SERVICE_LIST = {name: Service(*vals) for
                 idx, (name, vals) in enumerate(_services.items())}
@@ -120,13 +122,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
+  h += "struct service { std::string name; bool should_log; float frequency; int decimation; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
+    h += '  { "%s", {"%s", %s, %f, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation, v.queue_size)
+         (k, k, should_log, v.frequency, decimation)
   h += "};\n"
 
   h += "#endif\n"

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -1,10 +1,11 @@
-Import('env', 'envCython')
+Import('env', 'envCython', 'arch')
 
 common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
   'ratekeeper.cc',
+  'clutil.cc',
 ]
 
 _common = env.Library('common', common_libs, LIBS="json11")
@@ -18,6 +19,11 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-common_python = [params_python]
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
 
 Export('common_python')

common/api/__init__.py

@@ -18,9 +18,9 @@ class Api:
     return self.service.get_token(payload_extra, expiry_hours)
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
-  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
+  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, **params)
 
 
-def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+def get_key_pair():
   return CommaConnectApi(None).get_key_pair()

common/api/base.py

@@ -6,9 +6,9 @@ from datetime import datetime, timedelta, UTC
 from openpilot.system.hardware.hw import Paths
 from openpilot.system.version import get_version
 
-# name: jwt signature algorithm
+       # name : jwt signature algorithm
-KEYS = {"id_rsa": "RS256",
+KEYS = {"id_rsa" : "RS256",
-        "id_ecdsa": "ES256"}
+        "id_ecdsa" : "ES256"}
 
 
 class BaseApi:
@@ -51,7 +51,7 @@ class BaseApi:
     ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
     return ascii_encoded_text.decode()
 
-  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, json=None, **params):
     headers = {}
     if access_token is not None:
       headers['Authorization'] = "JWT " + access_token
@@ -59,12 +59,10 @@ class BaseApi:
     version = self.remove_non_ascii_chars(get_version())
     headers['User-Agent'] = self.user_agent + version
 
-    # TODO: add session to Api
+    return requests.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
-    req = requests if session is None else session
-    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
 
   @staticmethod
-  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+  def get_key_pair():
     for key in KEYS:
       if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
         with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:

common/clutil.cc

@@ -0,0 +1,98 @@
+#include "common/clutil.h"
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+
+#include "common/util.h"
+#include "common/swaglog.h"
+
+namespace {  // helper functions
+
+template <typename Func, typename Id, typename Name>
+std::string get_info(Func get_info_func, Id id, Name param_name) {
+  size_t size = 0;
+  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
+  std::string info(size, '\0');
+  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
+  return info;
+}
+inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
+inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
+
+void cl_print_info(cl_platform_id platform, cl_device_id device) {
+  size_t work_group_size = 0;
+  cl_device_type device_type = 0;
+  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
+  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
+  const char *type_str = "Other...";
+  switch (device_type) {
+    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
+    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
+    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
+  }
+
+  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
+  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
+  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
+  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
+  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
+  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
+  LOGD("max work group size: %zu", work_group_size);
+  LOGD("type = %d, %s", (int)device_type, type_str);
+}
+
+void cl_print_build_errors(cl_program program, cl_device_id device) {
+  cl_build_status status;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
+  size_t log_size;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
+  std::string log(log_size, '\0');
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
+
+  LOGE("build failed; status=%d, log: %s", status, log.c_str());
+}
+
+}  // namespace
+
+cl_device_id cl_get_device_id(cl_device_type device_type) {
+  cl_uint num_platforms = 0;
+  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
+  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
+  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
+
+  for (size_t i = 0; i < num_platforms; ++i) {
+    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
+
+    // Get first device
+    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
+      cl_print_info(platform_ids[i], device_id);
+      return device_id;
+    }
+  }
+  LOGE("No valid openCL platform found");
+  assert(0);
+  return nullptr;
+}
+
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
+void cl_release_context(cl_context context) {
+  clReleaseContext(context);
+}
+
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
+  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
+}
+
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
+  const char *csrc = src.c_str();
+  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
+  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
+    cl_print_build_errors(prg, device_id);
+    assert(0);
+  }
+  return prg;
+}

common/clutil.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#include <string>
+
+#define CL_CHECK(_expr)          \
+  do {                           \
+    assert(CL_SUCCESS == (_expr)); \
+  } while (0)
+
+#define CL_CHECK_ERR(_expr)           \
+  ({                                  \
+    cl_int err = CL_INVALID_VALUE;    \
+    __typeof__(_expr) _ret = _expr;   \
+    assert(_ret&& err == CL_SUCCESS); \
+    _ret;                             \
+  })
+
+cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
+void cl_release_context(cl_context context);
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/file_chunker.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import math
-import os
-from pathlib import Path
-
-CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
-
-def get_chunk_name(name, idx, num_chunks):
-  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
-
-def get_manifest_path(name):
-  return f"{name}.chunkmanifest"
-
-def _chunk_paths(path, num_chunks):
-  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
-
-def get_chunk_targets(path, file_size):
-  num_chunks = math.ceil(file_size / CHUNK_SIZE)
-  return _chunk_paths(path, num_chunks)
-
-def chunk_file(path, targets):
-  manifest_path, *chunk_paths = targets
-  with open(path, 'rb') as f:
-    data = f.read()
-  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
-  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
-  for i, chunk_path in enumerate(chunk_paths):
-    with open(chunk_path, 'wb') as f:
-      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
-  Path(manifest_path).write_text(str(len(chunk_paths)))
-  os.remove(path)
-
-def get_existing_chunks(path):
-  if os.path.isfile(path):
-    return [path]
-  if os.path.isfile(manifest := get_manifest_path(path)):
-    num_chunks = int(Path(manifest).read_text().strip())
-    return _chunk_paths(path, num_chunks)
-  raise FileNotFoundError(path)
-
-def read_file_chunked(path):
-  manifest_path = get_manifest_path(path)
-  if os.path.isfile(manifest_path):
-    num_chunks = int(Path(manifest_path).read_text().strip())
-    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
-  if os.path.isfile(path):
-    return Path(path).read_bytes()
-  raise FileNotFoundError(path)
-
-
-if __name__ == "__main__":
-  path = sys.argv[1]
-  chunk_paths = get_chunk_targets(path, os.path.getsize(path))
-  chunk_file(path, chunk_paths)

common/filter_simple.py

@@ -28,7 +28,7 @@ class BounceFilter(FirstOrderFilter):
     scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
     self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
     self.velocity.update(0.0)
-    if abs(self.velocity.x) < 1e-3:
+    if abs(self.velocity.x) < 1e-5:
       self.velocity.x = 0.0
     self.x += self.velocity.x
     return self.x

common/git.py

@@ -4,27 +4,27 @@ from openpilot.common.utils import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str | None = None) -> str:
+def get_short_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str | None = None) -> str:
+def get_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str | None = None) -> str:
+def get_origin(cwd: str = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
@@ -34,7 +34,7 @@ def get_origin(cwd: str | None = None) -> str:
 
 
 @cache
-def get_normalized_origin(cwd: str | None = None) -> str:
+def get_normalized_origin(cwd: str = None) -> str:
   return get_origin(cwd) \
     .replace("git@", "", 1) \
     .replace(".git", "", 1) \

common/i2c.py

@@ -1,81 +0,0 @@
-import os
-import fcntl
-import ctypes
-
-# I2C constants from /usr/include/linux/i2c-dev.h
-I2C_SLAVE = 0x0703
-I2C_SLAVE_FORCE = 0x0706
-I2C_SMBUS = 0x0720
-
-# SMBus transfer types
-I2C_SMBUS_READ = 1
-I2C_SMBUS_WRITE = 0
-I2C_SMBUS_BYTE_DATA = 2
-I2C_SMBUS_I2C_BLOCK_DATA = 8
-
-I2C_SMBUS_BLOCK_MAX = 32
-
-
-class _I2cSmbusData(ctypes.Union):
-  _fields_ = [
-    ("byte", ctypes.c_uint8),
-    ("word", ctypes.c_uint16),
-    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
-  ]
-
-
-class _I2cSmbusIoctlData(ctypes.Structure):
-  _fields_ = [
-    ("read_write", ctypes.c_uint8),
-    ("command", ctypes.c_uint8),
-    ("size", ctypes.c_uint32),
-    ("data", ctypes.POINTER(_I2cSmbusData)),
-  ]
-
-
-class SMBus:
-  def __init__(self, bus: int):
-    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
-
-  def __enter__(self) -> 'SMBus':
-    return self
-
-  def __exit__(self, *args) -> None:
-    self.close()
-
-  def close(self) -> None:
-    if hasattr(self, '_fd') and self._fd >= 0:
-      os.close(self._fd)
-      self._fd = -1
-
-  def _set_address(self, addr: int, force: bool = False) -> None:
-    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
-    fcntl.ioctl(self._fd, ioctl_arg, addr)
-
-  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
-    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
-    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
-
-  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
-    return int(data.byte)
-
-  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    data.byte = value & 0xFF
-    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
-
-  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
-    self._set_address(addr, force)
-    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
-      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
-
-    data = _I2cSmbusData()
-    data.block[0] = length
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
-    read_len = int(data.block[0]) or length
-    read_len = min(read_len, length)
-    return [int(b) for b in data.block[1 : read_len + 1]]

common/mat.h

@@ -0,0 +1,85 @@
+#pragma once
+
+typedef struct vec3 {
+  float v[3];
+} vec3;
+
+typedef struct vec4 {
+  float v[4];
+} vec4;
+
+typedef struct mat3 {
+  float v[3*3];
+} mat3;
+
+typedef struct mat4 {
+  float v[4*4];
+} mat4;
+
+static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
+  mat3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      float v = 0.0;
+      for (int k=0; k<3; k++) {
+        v += a.v[r*3+k] * b.v[k*3+c];
+      }
+      ret.v[r*3+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
+  vec3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      ret.v[r] += a.v[r*3+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+static inline mat4 matmul(const mat4 &a, const mat4 &b) {
+  mat4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      float v = 0.0;
+      for (int k=0; k<4; k++) {
+        v += a.v[r*4+k] * b.v[k*4+c];
+      }
+      ret.v[r*4+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
+  vec4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      ret.v[r] += a.v[r*4+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+// scales the input and output space of a transformation matrix
+// that assumes pixel-center origin.
+static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
+  // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
+
+  mat3 transform_out = {{
+    1.0f/s, 0.0f, 0.5f,
+    0.0f, 1.0f/s, 0.5f,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  mat3 transform_in = {{
+    s,  0.0f, -0.5f*s,
+    0.0f, s, -0.5f*s,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  return matmul3(transform_in, matmul3(in, transform_out));
+}

common/model.h

@@ -0,0 +1 @@
+#define DEFAULT_MODEL "The Cool People (Default)"

common/parameterized.py

@@ -1,54 +0,0 @@
-import re
-import sys
-import pytest
-import inspect
-
-
-def _to_safe_name(s):
-  return re.sub(r"[^a-zA-Z0-9_]+", "_", str(s)).strip("_")
-
-
-class parameterized:
-  @staticmethod
-  def expand(cases):
-    cases = list(cases)
-
-    if not cases:
-      return lambda func: pytest.mark.skip("no parameterized cases")(func)
-
-    def decorator(func):
-      params = [p for p in inspect.signature(func).parameters if p != 'self']
-      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
-      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
-      expand_params = params[: len(normalized[0])]
-      if len(expand_params) == 1:
-        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
-      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
-
-    return decorator
-
-
-def parameterized_class(attrs, input_list=None):
-  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
-    params_list = attrs
-  else:
-    assert input_list is not None
-    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
-    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
-
-  def decorator(cls):
-    globs = sys._getframe(1).f_globals
-    for i, params in enumerate(params_list):
-      # append sanitized string param values so pytest -k can filter by them
-      suffix = "_".join(filter(None, (_to_safe_name(v) for v in params.values() if isinstance(v, str))))
-      name = f"{cls.__name__}_{i}" + (f"_{suffix}" if suffix else "")
-      new_cls = type(name, (cls,), dict(params))
-      new_cls.__module__ = cls.__module__
-      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
-      globs[name] = new_cls
-    # Don't collect the un-parametrised base, but return it so outer decorators
-    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
-    cls.__test__ = False
-    return cls
-
-  return decorator

common/params.py

@@ -14,6 +14,6 @@ if __name__ == "__main__":
   if len(sys.argv) == 3:
     val = sys.argv[2]
     print(f"SET: {key} = {val}")
-    params.put(key, val, block=True)
+    params.put(key, val)
   elif len(sys.argv) == 2:
     print(f"GET: {key} = {params.get(key)}")

common/params_keys.h

@@ -71,7 +71,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"LastGPSPosition", {PERSISTENT, STRING}},
     {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
-    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
@@ -80,10 +79,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
     {"LiveParameters", {PERSISTENT, JSON}},
     {"LiveParametersV2", {PERSISTENT, BYTES}},
-    {"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
     {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
     {"LocationFilterInitialState", {PERSISTENT, BYTES}},
-    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
     {"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
@@ -104,6 +101,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
     {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
     {"PrimeType", {PERSISTENT, INT}},
     {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
     {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -115,7 +114,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
     {"TermsVersion", {PERSISTENT, STRING}},
-    {"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TrainingVersion", {PERSISTENT, STRING}},
     {"UbloxAvailable", {PERSISTENT, BOOL}},
     {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
@@ -131,25 +129,20 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
     {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //
     {"ApiCache_DriveStats", {PERSISTENT, JSON}},
     {"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
-    {"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}},  // seconds
     {"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}},  // MPH or km/h
     {"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
     {"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"CarList", {PERSISTENT, JSON}},
     {"CarParamsSP", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
     {"CarParamsSPCache", {CLEAR_ON_MANAGER_START, BYTES}},
     {"CarParamsSPPersistent", {PERSISTENT, BYTES}},
     {"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
     {"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
-    {"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
     {"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
     {"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
@@ -159,7 +152,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
     {"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
-    {"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
     {"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
     {"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
@@ -172,14 +164,12 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
     {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
     {"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
+    {"OnroadScreenOffControl", {PERSISTENT | BACKUP, BOOL}},
     {"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
-    {"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
     {"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -194,18 +184,22 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     // Model Manager params
     {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
     {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
+    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT, "0"}},
     {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
     {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
     {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
 
+    // Navigation params
+    {"MapboxToken", {PERSISTENT | BACKUP, STRING}},
+    {"MapboxSettings", {CLEAR_ON_MANAGER_START, JSON}},
+    {"MapboxRoute", {CLEAR_ON_MANAGER_START, STRING}},
+
     // Neural Network Lateral Control
     {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     // sunnylink params
     {"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
     {"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"ParamsVersion", {PERSISTENT, INT}},
     {"SunnylinkCache_Roles", {PERSISTENT, STRING}},
     {"SunnylinkCache_Users", {PERSISTENT, STRING}},
     {"SunnylinkDongleId", {PERSISTENT, STRING}},
@@ -222,20 +216,16 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     {"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     // sunnypilot model params
-    {"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
     {"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
     {"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
     {"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
-    {"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
 
     // mapd
     {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
@@ -256,7 +246,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OsmStateTitle", {PERSISTENT, STRING}},
     {"OsmWayTest", {PERSISTENT, STRING}},
     {"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RoadNameToggle", {PERSISTENT, STRING}},
 
     // Speed Limit
     {"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
@@ -274,7 +264,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
     {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
     {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
-    {"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
     {"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
     {"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},

common/params_pyx.pyx

@@ -142,28 +142,33 @@ cdef class Params:
     cdef ParamKeyType t = self.p.getKeyType(k)
     return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
 
-  def put(self, key, dat, bool block = False):
+  def put(self, key, dat):
     """
-    Warning: block=True blocks until the param is written to disk!
+    Warning: This function blocks until the param is written to disk!
     In very rare cases this can take over a second, and your code will hang.
-    Use block=False in time sensitive code, but in general try to avoid
+    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
-    writing params as much as possible.
+    in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
     cdef string dat_bytes = self._put_cast(key, dat)
     with nogil:
-      if block:
+      self.p.put(k, dat_bytes)
-        self.p.put(k, dat_bytes)
-      else:
-        self.p.putNonBlocking(k, dat_bytes)
 
-  def put_bool(self, key, bool val, bool block = False):
+  def put_bool(self, key, bool val):
     cdef string k = self.check_key(key)
     with nogil:
-      if block:
+      self.p.putBool(k, val)
-        self.p.putBool(k, val)
+
-      else:
+  def put_nonblocking(self, key, dat):
-        self.p.putBoolNonBlocking(k, val)
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = self._put_cast(key, dat)
+    with nogil:
+      self.p.putNonBlocking(k, dat_bytes)
+
+  def put_bool_nonblocking(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBoolNonBlocking(k, val)
 
   def remove(self, key):
     cdef string k = self.check_key(key)

common/pid.py

@@ -3,9 +3,15 @@ from numbers import Number
 
 class PIDController:
   def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
-    self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
+    self._k_p = k_p
-    self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
+    self._k_i = k_i
-    self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
+    self._k_d = k_d
+    if isinstance(self._k_p, Number):
+      self._k_p = [[0], [self._k_p]]
+    if isinstance(self._k_i, Number):
+      self._k_i = [[0], [self._k_i]]
+    if isinstance(self._k_d, Number):
+      self._k_d = [[0], [self._k_d]]
 
     self.set_limits(pos_limit, neg_limit)
 

common/prefix.h

@@ -13,11 +13,7 @@ public:
     if (prefix.empty()) {
       prefix = util::random_string(15);
     }
-#ifdef __APPLE__
+    msgq_path = Path::shm_path() + "/" + prefix;
-    msgq_path = "/tmp/msgq_" + prefix;
-#else
-    msgq_path = "/dev/shm/msgq_" + prefix;
-#endif
     bool ret = util::create_directories(msgq_path, 0777);
     assert(ret);
     setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
@@ -27,14 +23,14 @@ public:
     auto param_path = Params().getParamPath();
     if (util::file_exists(param_path)) {
       std::string real_path = util::readlink(param_path);
-      util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
+      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
       unlink(param_path.c_str());
     }
     if (getenv("COMMA_CACHE") == nullptr) {
-      util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
+      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
     }
-    util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
+    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
-    util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
+    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
     unsetenv("OPENPILOT_PREFIX");
   }
 

common/prefix.py

@@ -1,5 +1,4 @@
 import os
-import platform
 import shutil
 import uuid
 
@@ -10,10 +9,9 @@ from openpilot.system.hardware.hw import Paths
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
-  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+  def __init__(self, prefix: str = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
     self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
+    self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
-    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
     self.create_dirs_on_enter = create_dirs_on_enter
     self.clean_dirs_on_exit = clean_dirs_on_exit
     self.shared_download_cache = shared_download_cache

common/ratekeeper.cc

@@ -6,9 +6,9 @@
 #include "common/timing.h"
 #include "common/util.h"
 
-RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
-    : name(name_),
+    : name(name),
-      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
   interval = 1 / rate;
   last_monitor_time = seconds_since_boot();
   next_frame_time = last_monitor_time + interval;

common/realtime.py

@@ -6,7 +6,7 @@ import time
 
 from setproctitle import getproctitle
 
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
 from openpilot.system.hardware import PC
 
 
@@ -28,11 +28,6 @@ class Priority:
   CTRL_HIGH = 53
 
 
-def drop_realtime() -> None:
-  if sys.platform == 'linux' and not PC:
-    os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
-
-
 def set_core_affinity(cores: list[int]) -> None:
   if sys.platform == 'linux' and not PC:
     os.sched_setaffinity(0, cores)

common/swaglog.cc

@@ -11,7 +11,7 @@
 
 #include <zmq.h>
 #include <stdarg.h>
-#include "json11/json11.hpp"
+#include "third_party/json11/json11.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"
 

common/tests/test_file_helpers.py

@@ -1,7 +1,7 @@
 import os
 from uuid import uuid4
 
-from openpilot.common.utils import atomic_write
+from openpilot.common.utils import atomic_write_in_dir
 
 
 class TestFileHelpers:
@@ -15,5 +15,5 @@ class TestFileHelpers:
       assert f.read() == "test"
     os.remove(path)
 
-  def test_atomic_write(self):
+  def test_atomic_write_in_dir(self):
-    self.run_atomic_write_func(atomic_write)
+    self.run_atomic_write_func(atomic_write_in_dir)

common/tests/test_params.py

@@ -12,17 +12,17 @@ class TestParams:
     self.params = Params()
 
   def test_params_put_and_get(self):
-    self.params.put("DongleId", "cb38263377b873ee", block=True)
+    self.params.put("DongleId", "cb38263377b873ee")
     assert self.params.get("DongleId") == "cb38263377b873ee"
 
   def test_params_non_ascii(self):
     st = b"\xe1\x90\xff"
-    self.params.put("CarParams", st, block=True)
+    self.params.put("CarParams", st)
     assert self.params.get("CarParams") == st
 
   def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", b"test", block=True)
+    self.params.put("CarParams", b"test")
-    self.params.put("DongleId", "cb38263377b873ee", block=True)
+    self.params.put("DongleId", "cb38263377b873ee")
     assert self.params.get("CarParams") == b"test"
 
     undefined_param = self.params.get_param_path(uuid.uuid4().hex)
@@ -36,15 +36,15 @@ class TestParams:
     assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
-    self.params.put("DongleId", "bob", block=True)
+    self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", 123, block=True)
+    self.params.put("AthenadPid", 123)
     assert self.params.get("DongleId") == "bob"
     assert self.params.get("AthenadPid") == 123
 
   def test_params_get_block(self):
     def _delayed_writer():
       time.sleep(0.1)
-      self.params.put("CarParams", b"test", block=True)
+      self.params.put("CarParams", b"test")
     threading.Thread(target=_delayed_writer).start()
     assert self.params.get("CarParams") is None
     assert self.params.get("CarParams", block=True) == b"test"
@@ -57,10 +57,10 @@ class TestParams:
       self.params.get_bool("swag")
 
     with pytest.raises(UnknownKeyName):
-      self.params.put("swag", "abc", block=True)
+      self.params.put("swag", "abc")
 
     with pytest.raises(UnknownKeyName):
-      self.params.put_bool("swag", True, block=True)
+      self.params.put_bool("swag", True)
 
   def test_remove_not_there(self):
     assert self.params.get("CarParams") is None
@@ -71,23 +71,23 @@ class TestParams:
     self.params.remove("IsMetric")
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put_bool("IsMetric", True, block=True)
+    self.params.put_bool("IsMetric", True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put_bool("IsMetric", False, block=True)
+    self.params.put_bool("IsMetric", False)
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", True, block=True)
+    self.params.put("IsMetric", True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", False, block=True)
+    self.params.put("IsMetric", False)
     assert not self.params.get_bool("IsMetric")
 
   def test_put_non_blocking_with_get_block(self):
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put("CarParams", b"test")
+      Params().put_nonblocking("CarParams", b"test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
@@ -96,7 +96,7 @@ class TestParams:
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_bool("CarParams", True)
+      Params().put_bool_nonblocking("CarParams", True)
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"1"
@@ -123,19 +123,19 @@ class TestParams:
 
   def test_params_get_type(self):
     # json
-    self.params.put("ApiCache_FirehoseStats", {"a": 0}, block=True)
+    self.params.put("ApiCache_FirehoseStats", {"a": 0})
     assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
 
     # int
-    self.params.put("BootCount", 1441, block=True)
+    self.params.put("BootCount", 1441)
     assert self.params.get("BootCount") == 1441
 
     # bool
-    self.params.put("AdbEnabled", True, block=True)
+    self.params.put("AdbEnabled", True)
     assert self.params.get("AdbEnabled")
     assert isinstance(self.params.get("AdbEnabled"), bool)
 
     # time
     now = datetime.datetime.now(datetime.UTC)
-    self.params.put("InstallDate", now, block=True)
+    self.params.put("InstallDate", now)
     assert self.params.get("InstallDate") == now

common/tests/test_swaglog.cc

@@ -7,7 +7,7 @@
 #include "common/util.h"
 #include "common/version.h"
 #include "system/hardware/hw.h"
-#include "json11/json11.hpp"
+#include "third_party/json11/json11.hpp"
 
 #include "sunnypilot/common/version.h"
 

common/tests/test_util.cc

@@ -36,7 +36,7 @@ TEST_CASE("util::read_file") {
     REQUIRE(util::read_file(filename).empty());
 
     std::string content = random_bytes(64 * 1024);
-    REQUIRE(write(fd, content.c_str(), content.size()) == (ssize_t)content.size());
+    write(fd, content.c_str(), content.size());
     std::string ret = util::read_file(filename);
     bool equal = (ret == content);
     REQUIRE(equal);
@@ -114,12 +114,12 @@ TEST_CASE("util::safe_fwrite") {
 }
 
 TEST_CASE("util::create_directories") {
-  REQUIRE(system("rm /tmp/test_create_directories -rf") == 0);
+  system("rm /tmp/test_create_directories -rf");
   std::string dir = "/tmp/test_create_directories/a/b/c/d/e/f";
 
-  auto check_dir_permissions = [](const std::string &path, mode_t mode) -> bool {
+  auto check_dir_permissions = [](const std::string &dir, mode_t mode) -> bool {
     struct stat st = {};
-    return stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
+    return stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
   };
 
   SECTION("create_directories") {
@@ -132,7 +132,7 @@ TEST_CASE("util::create_directories") {
   }
   SECTION("a file exists with the same name") {
     REQUIRE(util::create_directories(dir, 0755));
-    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT, 0644);
+    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT);
     REQUIRE(f != -1);
     close(f);
     REQUIRE(util::create_directories(dir + "/file", 0755) == false);

common/time_helpers.py

@@ -2,7 +2,6 @@ import datetime
 from pathlib import Path
 
 MIN_DATE = datetime.datetime(year=2025, month=2, day=21)
-MAX_DATE = datetime.datetime(year=2035, month=1, day=1)
 
 def min_date():
   # on systemd systems, the default time is the systemd build time
@@ -13,4 +12,4 @@ def min_date():
   return MIN_DATE
 
 def system_time_valid():
-  return min_date() < datetime.datetime.now() < MAX_DATE
+  return datetime.datetime.now() > min_date()

common/transformations/.gitignore

@@ -0,0 +1,2 @@
+transformations
+transformations.cpp

common/transformations/SConscript

@@ -0,0 +1,5 @@
+Import('env', 'envCython')
+
+transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc'])
+transformations_python = envCython.Program('transformations.so', 'transformations.pyx')
+Export('transformations', 'transformations_python')

common/transformations/coordinates.cc

@@ -1,6 +1,6 @@
 #define _USE_MATH_DEFINES
 
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 #include <iostream>
 #include <cmath>

common/transformations/orientation.cc

@@ -4,8 +4,8 @@
 #include <cmath>
 #include <eigen3/Eigen/Dense>
 
-#include "sunnypilot/common/transformations/orientation.hpp"
+#include "common/transformations/orientation.hpp"
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 Eigen::Quaterniond ensure_unique(const Eigen::Quaterniond &quat) {
   if (quat.w() > 0){
@@ -141,3 +141,4 @@ Eigen::Vector3d ned_euler_from_ecef(const ECEF &ecef_init, const Eigen::Vector3d
 
   return {phi, theta, psi};
 }
+

common/transformations/orientation.hpp

@@ -1,6 +1,6 @@
 #pragma once
 #include <eigen3/Eigen/Dense>
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 
 Eigen::Quaterniond ensure_unique(const Eigen::Quaterniond &quat);

common/transformations/tests/test_coordinates.py

@@ -102,36 +102,3 @@ class TestNED:
     np.testing.assert_allclose(converter.ned2ecef(ned_offsets_batch),
                                                            ecef_positions_offset_batch,
                                                            rtol=1e-9, atol=1e-7)
-
-  def test_errors(self):
-    # Test wrong shape/type for geodetic2ecef
-    # numpy_wrap raises IndexError for scalar input
-    with np.testing.assert_raises(IndexError):
-      coord.geodetic2ecef(1.0)
-
-    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
-      coord.geodetic2ecef([0, 0])
-
-    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
-      coord.geodetic2ecef([0, 0, 0, 0])
-
-    with np.testing.assert_raises(TypeError):
-      coord.geodetic2ecef(['a', 'b', 'c'])
-
-    # Test LocalCoord constructor errors
-    with np.testing.assert_raises(ValueError):
-      coord.LocalCoord.from_geodetic([0, 0])
-
-    with np.testing.assert_raises(ValueError):
-      coord.LocalCoord.from_geodetic(1)
-
-    with np.testing.assert_raises(TypeError):
-      coord.LocalCoord.from_geodetic(['a', 'b', 'c'])
-
-    # Test wrong shape/type for ecef2geodetic
-    with np.testing.assert_raises(ValueError):
-      coord.ecef2geodetic([1, 2])
-    with np.testing.assert_raises(ValueError):
-      coord.ecef2geodetic([1, 2, 3, 4])
-    with np.testing.assert_raises(IndexError):
-      coord.ecef2geodetic(1.0)

common/transformations/tests/test_orientation.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pytest
 
 from openpilot.common.transformations.orientation import euler2quat, quat2euler, euler2rot, rot2euler, \
                                                rot2quat, quat2rot, \
@@ -60,32 +59,3 @@ class TestOrientation:
       np.testing.assert_allclose(ned_eulers[i], ned_euler_from_ecef(ecef_positions[i], eulers[i]), rtol=1e-7)
       #np.testing.assert_allclose(eulers[i], ecef_euler_from_ned(ecef_positions[i], ned_eulers[i]), rtol=1e-7)
     # np.testing.assert_allclose(ned_eulers, ned_euler_from_ecef(ecef_positions, eulers), rtol=1e-7)
-
-  def test_inputs(self):
-    with pytest.raises(ValueError):
-      euler2quat([1, 2])
-
-    with pytest.raises(ValueError):
-      quat2rot([1, 2, 3])
-
-    with pytest.raises(IndexError):
-      rot2quat(np.zeros((2, 2)))
-
-  def test_euler_rot_consistency(self):
-    rpy = [0.1, 0.2, 0.3]
-    R = euler2rot(rpy)
-
-    # R -> q -> R
-    q = rot2quat(R)
-    R_new = quat2rot(q)
-    np.testing.assert_allclose(R, R_new, atol=1e-15)
-
-    # q -> R -> Euler (quat2euler) -> R
-    rpy_new = quat2euler(q)
-    R_new2 = euler2rot(rpy_new)
-    np.testing.assert_allclose(R, R_new2, atol=1e-15)
-
-    # R -> Euler (rot2euler) -> R
-    rpy_from_rot = rot2euler(R)
-    R_new3 = euler2rot(rpy_from_rot)
-    np.testing.assert_allclose(R, R_new3, atol=1e-15)

common/transformations/transformations.pxd

@@ -0,0 +1,72 @@
+# cython: language_level=3
+from libcpp cimport bool
+
+cdef extern from "orientation.cc":
+  pass
+
+cdef extern from "orientation.hpp":
+  cdef cppclass Quaternion "Eigen::Quaterniond":
+    Quaternion()
+    Quaternion(double, double, double, double)
+    double w()
+    double x()
+    double y()
+    double z()
+
+  cdef cppclass Vector3 "Eigen::Vector3d":
+    Vector3()
+    Vector3(double, double, double)
+    double operator()(int)
+
+  cdef cppclass Matrix3 "Eigen::Matrix3d":
+    Matrix3()
+    Matrix3(double*)
+
+    double operator()(int, int)
+
+  Quaternion euler2quat(const Vector3 &)
+  Vector3 quat2euler(const Quaternion &)
+  Matrix3 quat2rot(const Quaternion &)
+  Quaternion rot2quat(const Matrix3 &)
+  Vector3 rot2euler(const Matrix3 &)
+  Matrix3 euler2rot(const Vector3 &)
+  Matrix3 rot_matrix(double, double, double)
+  Vector3 ecef_euler_from_ned(const ECEF &, const Vector3 &)
+  Vector3 ned_euler_from_ecef(const ECEF &, const Vector3 &)
+
+
+cdef extern from "coordinates.cc":
+  cdef struct ECEF:
+    double x
+    double y
+    double z
+
+  cdef struct NED:
+    double n
+    double e
+    double d
+
+  cdef struct Geodetic:
+    double lat
+    double lon
+    double alt
+    bool radians
+
+  ECEF geodetic2ecef(const Geodetic &)
+  Geodetic ecef2geodetic(const ECEF &)
+
+  cdef cppclass LocalCoord_c "LocalCoord":
+    Matrix3 ned2ecef_matrix
+    Matrix3 ecef2ned_matrix
+
+    LocalCoord_c(const Geodetic &, const ECEF &)
+    LocalCoord_c(const Geodetic &)
+    LocalCoord_c(const ECEF &)
+
+    NED ecef2ned(const ECEF &)
+    ECEF ned2ecef(const NED &)
+    NED geodetic2ned(const Geodetic &)
+    Geodetic ned2geodetic(const NED &)
+
+cdef extern from "coordinates.hpp":
+  pass

common/transformations/transformations.py

@@ -1,342 +0,0 @@
-import numpy as np
-
-
-# Constants
-a = 6378137.0
-b = 6356752.3142
-esq = 6.69437999014e-3
-e1sq = 6.73949674228e-3
-
-
-def geodetic2ecef_single(g):
-  """
-  Convert geodetic coordinates (latitude, longitude, altitude) to ECEF.
-  """
-  try:
-    if len(g) != 3:
-      raise ValueError("Geodetic must be size 3")
-  except TypeError:
-    raise ValueError("Geodetic must be a sequence of length 3") from None
-
-  lat, lon, alt = g
-  lat = np.radians(lat)
-  lon = np.radians(lon)
-  xi = np.sqrt(1.0 - esq * np.sin(lat)**2)
-  x = (a / xi + alt) * np.cos(lat) * np.cos(lon)
-  y = (a / xi + alt) * np.cos(lat) * np.sin(lon)
-  z = (a / xi * (1.0 - esq) + alt) * np.sin(lat)
-  return np.array([x, y, z])
-
-
-def ecef2geodetic_single(e):
-  """
-  Convert ECEF to geodetic coordinates using Ferrari's solution.
-  """
-  x, y, z = e
-  r = np.sqrt(x**2 + y**2)
-  Esq = a**2 - b**2
-  F = 54 * b**2 * z**2
-  G = r**2 + (1 - esq) * z**2 - esq * Esq
-  C = (esq**2 * F * r**2) / (G**3)
-  S = np.cbrt(1 + C + np.sqrt(C**2 + 2 * C))
-  P = F / (3 * (S + 1 / S + 1)**2 * G**2)
-  Q = np.sqrt(1 + 2 * esq**2 * P)
-  r_0 = -(P * esq * r) / (1 + Q) + np.sqrt(0.5 * a**2 * (1 + 1.0 / Q) - P * (1 - esq) * z**2 / (Q * (1 + Q)) - 0.5 * P * r**2)
-  U = np.sqrt((r - esq * r_0)**2 + z**2)
-  V = np.sqrt((r - esq * r_0)**2 + (1 - esq) * z**2)
-  Z_0 = b**2 * z / (a * V)
-  h = U * (1 - b**2 / (a * V))
-  lat = np.arctan((z + e1sq * Z_0) / r)
-  lon = np.arctan2(y, x)
-  return np.array([np.degrees(lat), np.degrees(lon), h])
-
-
-def euler2quat_single(euler):
-  """
-  Convert Euler angles (roll, pitch, yaw) to a quaternion.
-  Rotation order: Z-Y-X (yaw, pitch, roll).
-  """
-  phi, theta, psi = euler
-
-  c_phi, s_phi = np.cos(phi / 2), np.sin(phi / 2)
-  c_theta, s_theta = np.cos(theta / 2), np.sin(theta / 2)
-  c_psi, s_psi = np.cos(psi / 2), np.sin(psi / 2)
-
-  w = c_phi * c_theta * c_psi + s_phi * s_theta * s_psi
-  x = s_phi * c_theta * c_psi - c_phi * s_theta * s_psi
-  y = c_phi * s_theta * c_psi + s_phi * c_theta * s_psi
-  z = c_phi * c_theta * s_psi - s_phi * s_theta * c_psi
-
-  if w < 0:
-    return np.array([-w, -x, -y, -z])
-  return np.array([w, x, y, z])
-
-
-def quat2euler_single(q):
-  """
-  Convert a quaternion to Euler angles (roll, pitch, yaw).
-  """
-  w, x, y, z = q
-  gamma = np.arctan2(2 * (w * x + y * z), 1 - 2 * (x**2 + y**2))
-  sin_arg = 2 * (w * y - z * x)
-  sin_arg = np.clip(sin_arg, -1.0, 1.0)
-  theta = np.arcsin(sin_arg)
-  psi = np.arctan2(2 * (w * z + x * y), 1 - 2 * (y**2 + z**2))
-  return np.array([gamma, theta, psi])
-
-
-def quat2rot_single(q):
-  """
-  Convert a quaternion to a 3x3 rotation matrix.
-  """
-  w, x, y, z = q
-  xx, yy, zz = x * x, y * y, z * z
-  xy, xz, yz = x * y, x * z, y * z
-  wx, wy, wz = w * x, w * y, w * z
-
-  mat = np.array([
-    [1 - 2 * (yy + zz), 2 * (xy - wz), 2 * (xz + wy)],
-    [2 * (xy + wz), 1 - 2 * (xx + zz), 2 * (yz - wx)],
-    [2 * (xz - wy), 2 * (yz + wx), 1 - 2 * (xx + yy)]
-  ])
-  return mat
-
-
-def rot2quat_single(rot):
-  """
-  Convert a 3x3 rotation matrix to a quaternion.
-  """
-  trace = np.trace(rot)
-  if trace > 0:
-    s = 0.5 / np.sqrt(trace + 1.0)
-    w = 0.25 / s
-    x = (rot[2, 1] - rot[1, 2]) * s
-    y = (rot[0, 2] - rot[2, 0]) * s
-    z = (rot[1, 0] - rot[0, 1]) * s
-  else:
-    if rot[0, 0] > rot[1, 1] and rot[0, 0] > rot[2, 2]:
-      s = 2.0 * np.sqrt(1.0 + rot[0, 0] - rot[1, 1] - rot[2, 2])
-      w = (rot[2, 1] - rot[1, 2]) / s
-      x = 0.25 * s
-      y = (rot[0, 1] + rot[1, 0]) / s
-      z = (rot[0, 2] + rot[2, 0]) / s
-    elif rot[1, 1] > rot[2, 2]:
-      s = 2.0 * np.sqrt(1.0 + rot[1, 1] - rot[0, 0] - rot[2, 2])
-      w = (rot[0, 2] - rot[2, 0]) / s
-      x = (rot[0, 1] + rot[1, 0]) / s
-      y = 0.25 * s
-      z = (rot[1, 2] + rot[2, 1]) / s
-    else:
-      s = 2.0 * np.sqrt(1.0 + rot[2, 2] - rot[0, 0] - rot[1, 1])
-      w = (rot[1, 0] - rot[0, 1]) / s
-      x = (rot[0, 2] + rot[2, 0]) / s
-      y = (rot[1, 2] + rot[2, 1]) / s
-      z = 0.25 * s
-
-  if w < 0:
-    return np.array([-w, -x, -y, -z])
-  return np.array([w, x, y, z])
-
-
-def euler2rot_single(euler):
-  """
-  Convert Euler angles (roll, pitch, yaw) to a 3x3 rotation matrix.
-  Rotation order: Z-Y-X (yaw, pitch, roll).
-  """
-  phi, theta, psi = euler
-
-  cx, sx = np.cos(phi), np.sin(phi)
-  cy, sy = np.cos(theta), np.sin(theta)
-  cz, sz = np.cos(psi), np.sin(psi)
-
-  Rx = np.array([[1, 0, 0], [0, cx, -sx], [0, sx, cx]])
-  Ry = np.array([[cy, 0, sy], [0, 1, 0], [-sy, 0, cy]])
-  Rz = np.array([[cz, -sz, 0], [sz, cz, 0], [0, 0, 1]])
-
-  return Rz @ Ry @ Rx
-
-
-def rot2euler_single(rot):
-  """
-  Convert a 3x3 rotation matrix to Euler angles (roll, pitch, yaw).
-  """
-  return quat2euler_single(rot2quat_single(rot))
-
-
-def rot_matrix(roll, pitch, yaw):
-  """
-  Create a 3x3 rotation matrix from roll, pitch, and yaw angles.
-  """
-  return euler2rot_single([roll, pitch, yaw])
-
-
-def axis_angle_to_rot(axis, angle):
-  """
-  Convert an axis-angle representation to a 3x3 rotation matrix.
-  """
-  c = np.cos(angle / 2)
-  s = np.sin(angle / 2)
-  q = np.array([c, s*axis[0], s*axis[1], s*axis[2]])
-  return quat2rot_single(q)
-
-
-class LocalCoord:
-  """
-  A class to handle conversions between ECEF and local NED coordinates.
-  """
-  def __init__(self, geodetic=None, ecef=None):
-    """
-    Initialize LocalCoord with either geodetic or ECEF coordinates.
-    """
-    if geodetic is not None:
-      self.init_ecef = geodetic2ecef_single(geodetic)
-      lat, lon, _ = geodetic
-    elif ecef is not None:
-      self.init_ecef = np.array(ecef)
-      lat, lon, _ = ecef2geodetic_single(ecef)
-    else:
-      raise ValueError("Must provide geodetic or ecef")
-
-    lat = np.radians(lat)
-    lon = np.radians(lon)
-
-    self.ned2ecef_matrix = np.array([
-      [-np.sin(lat) * np.cos(lon), -np.sin(lon), -np.cos(lat) * np.cos(lon)],
-      [-np.sin(lat) * np.sin(lon), np.cos(lon), -np.cos(lat) * np.sin(lon)],
-      [np.cos(lat), 0, -np.sin(lat)]
-    ])
-    self.ecef2ned_matrix = self.ned2ecef_matrix.T
-
-  @classmethod
-  def from_geodetic(cls, geodetic):
-    """
-    Create a LocalCoord instance from geodetic coordinates.
-    """
-    return cls(geodetic=geodetic)
-
-  @classmethod
-  def from_ecef(cls, ecef):
-    """
-    Create a LocalCoord instance from ECEF coordinates.
-    """
-    return cls(ecef=ecef)
-
-  def ecef2ned_single(self, ecef):
-    """
-    Convert a single ECEF point to NED coordinates relative to the origin.
-    """
-    return self.ecef2ned_matrix @ (ecef - self.init_ecef)
-
-  def ned2ecef_single(self, ned):
-    """
-    Convert a single NED point to ECEF coordinates.
-    """
-    return self.ned2ecef_matrix @ ned + self.init_ecef
-
-  def geodetic2ned_single(self, geodetic):
-    """
-    Convert a single geodetic point to NED coordinates.
-    """
-    ecef = geodetic2ecef_single(geodetic)
-    return self.ecef2ned_single(ecef)
-
-  def ned2geodetic_single(self, ned):
-    """
-    Convert a single NED point to geodetic coordinates.
-    """
-    ecef = self.ned2ecef_single(ned)
-    return ecef2geodetic_single(ecef)
-
-  @property
-  def ned_from_ecef_matrix(self):
-    """
-    Returns the rotation matrix from ECEF to NED coordinates.
-    """
-    return self.ecef2ned_matrix
-
-  @property
-  def ecef_from_ned_matrix(self):
-    """
-    Returns the rotation matrix from NED to ECEF coordinates.
-    """
-    return self.ned2ecef_matrix
-
-
-def ecef_euler_from_ned_single(ecef_init, ned_pose):
-  """
-  Convert NED Euler angles (roll, pitch, yaw) at a given ECEF origin
-  to equivalent ECEF Euler angles.
-  """
-  converter = LocalCoord(ecef=ecef_init)
-  zero = np.array(ecef_init)
-
-  x0 = converter.ned2ecef_single([1, 0, 0]) - zero
-  y0 = converter.ned2ecef_single([0, 1, 0]) - zero
-  z0 = converter.ned2ecef_single([0, 0, 1]) - zero
-
-  phi, theta, psi = ned_pose
-
-  x1 = axis_angle_to_rot(z0, psi) @ x0
-  y1 = axis_angle_to_rot(z0, psi) @ y0
-  z1 = axis_angle_to_rot(z0, psi) @ z0
-
-  x2 = axis_angle_to_rot(y1, theta) @ x1
-  y2 = axis_angle_to_rot(y1, theta) @ y1
-  z2 = axis_angle_to_rot(y1, theta) @ z1
-
-  x3 = axis_angle_to_rot(x2, phi) @ x2
-  y3 = axis_angle_to_rot(x2, phi) @ y2
-
-  x0 = np.array([1.0, 0, 0])
-  y0 = np.array([0, 1.0, 0])
-  z0 = np.array([0, 0, 1.0])
-
-  psi_out = np.arctan2(np.dot(x3, y0), np.dot(x3, x0))
-  theta_out = np.arctan2(-np.dot(x3, z0), np.sqrt(np.dot(x3, x0)**2 + np.dot(x3, y0)**2))
-
-  y2 = axis_angle_to_rot(z0, psi_out) @ y0
-  z2 = axis_angle_to_rot(y2, theta_out) @ z0
-
-  phi_out = np.arctan2(np.dot(y3, z2), np.dot(y3, y2))
-
-  return np.array([phi_out, theta_out, psi_out])
-
-
-def ned_euler_from_ecef_single(ecef_init, ecef_pose):
-  """
-  Convert ECEF Euler angles (roll, pitch, yaw) at a given ECEF origin
-  to equivalent NED Euler angles.
-  """
-  converter = LocalCoord(ecef=ecef_init)
-
-  x0 = np.array([1.0, 0, 0])
-  y0 = np.array([0, 1.0, 0])
-  z0 = np.array([0, 0, 1.0])
-
-  phi, theta, psi = ecef_pose
-
-  x1 = axis_angle_to_rot(z0, psi) @ x0
-  y1 = axis_angle_to_rot(z0, psi) @ y0
-  z1 = axis_angle_to_rot(z0, psi) @ z0
-
-  x2 = axis_angle_to_rot(y1, theta) @ x1
-  y2 = axis_angle_to_rot(y1, theta) @ y1
-  z2 = axis_angle_to_rot(y1, theta) @ z1
-
-  x3 = axis_angle_to_rot(x2, phi) @ x2
-  y3 = axis_angle_to_rot(x2, phi) @ y2
-
-  zero = np.array(ecef_init)
-  x0 = converter.ned2ecef_single([1, 0, 0]) - zero
-  y0 = converter.ned2ecef_single([0, 1, 0]) - zero
-  z0 = converter.ned2ecef_single([0, 0, 1]) - zero
-
-  psi_out = np.arctan2(np.dot(x3, y0), np.dot(x3, x0))
-  theta_out = np.arctan2(-np.dot(x3, z0), np.sqrt(np.dot(x3, x0)**2 + np.dot(x3, y0)**2))
-
-  y2 = axis_angle_to_rot(z0, psi_out) @ y0
-  z2 = axis_angle_to_rot(y2, theta_out) @ z0
-
-  phi_out = np.arctan2(np.dot(y3, z2), np.dot(y3, y2))
-
-  return np.array([phi_out, theta_out, psi_out])

common/transformations/transformations.pyx

@@ -0,0 +1,173 @@
+# distutils: language = c++
+# cython: language_level = 3
+from openpilot.common.transformations.transformations cimport Matrix3, Vector3, Quaternion
+from openpilot.common.transformations.transformations cimport ECEF, NED, Geodetic
+
+from openpilot.common.transformations.transformations cimport euler2quat as euler2quat_c
+from openpilot.common.transformations.transformations cimport quat2euler as quat2euler_c
+from openpilot.common.transformations.transformations cimport quat2rot as quat2rot_c
+from openpilot.common.transformations.transformations cimport rot2quat as rot2quat_c
+from openpilot.common.transformations.transformations cimport euler2rot as euler2rot_c
+from openpilot.common.transformations.transformations cimport rot2euler as rot2euler_c
+from openpilot.common.transformations.transformations cimport rot_matrix as rot_matrix_c
+from openpilot.common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from openpilot.common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from openpilot.common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
+from openpilot.common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
+from openpilot.common.transformations.transformations cimport LocalCoord_c
+
+
+import numpy as np
+cimport numpy as np
+
+cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m):
+    return np.array([
+        [m(0, 0), m(0, 1), m(0, 2)],
+        [m(1, 0), m(1, 1), m(1, 2)],
+        [m(2, 0), m(2, 1), m(2, 2)],
+    ])
+
+cdef Matrix3 numpy2matrix(np.ndarray[double, ndim=2, mode="fortran"] m):
+    assert m.shape[0] == 3
+    assert m.shape[1] == 3
+    return Matrix3(<double*>m.data)
+
+cdef ECEF list2ecef(ecef):
+    cdef ECEF e
+    e.x = ecef[0]
+    e.y = ecef[1]
+    e.z = ecef[2]
+    return e
+
+cdef NED list2ned(ned):
+    cdef NED n
+    n.n = ned[0]
+    n.e = ned[1]
+    n.d = ned[2]
+    return n
+
+cdef Geodetic list2geodetic(geodetic):
+    cdef Geodetic g
+    g.lat = geodetic[0]
+    g.lon = geodetic[1]
+    g.alt = geodetic[2]
+    return g
+
+def euler2quat_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Quaternion q = euler2quat_c(e)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def quat2euler_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Vector3 e = quat2euler_c(q)
+    return [e(0), e(1), e(2)]
+
+def quat2rot_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Matrix3 r = quat2rot_c(q)
+    return matrix2numpy(r)
+
+def rot2quat_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Quaternion q = rot2quat_c(r)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def euler2rot_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Matrix3 r = euler2rot_c(e)
+    return matrix2numpy(r)
+
+def rot2euler_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Vector3 e = rot2euler_c(r)
+    return [e(0), e(1), e(2)]
+
+def rot_matrix(roll, pitch, yaw):
+    return matrix2numpy(rot_matrix_c(roll, pitch, yaw))
+
+def ecef_euler_from_ned_single(ecef_init, ned_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ned_pose[0], ned_pose[1], ned_pose[2])
+
+    cdef Vector3 e = ecef_euler_from_ned_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def ned_euler_from_ecef_single(ecef_init, ecef_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ecef_pose[0], ecef_pose[1], ecef_pose[2])
+
+    cdef Vector3 e = ned_euler_from_ecef_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def geodetic2ecef_single(geodetic):
+    cdef Geodetic g = list2geodetic(geodetic)
+    cdef ECEF e = geodetic2ecef_c(g)
+    return [e.x, e.y, e.z]
+
+def ecef2geodetic_single(ecef):
+    cdef ECEF e = list2ecef(ecef)
+    cdef Geodetic g = ecef2geodetic_c(e)
+    return [g.lat, g.lon, g.alt]
+
+
+cdef class LocalCoord:
+    cdef LocalCoord_c * lc
+
+    def __init__(self, geodetic=None, ecef=None):
+        assert (geodetic is not None) or (ecef is not None)
+        if geodetic is not None:
+            self.lc = new LocalCoord_c(list2geodetic(geodetic))
+        elif ecef is not None:
+            self.lc = new LocalCoord_c(list2ecef(ecef))
+
+    @property
+    def ned2ecef_matrix(self):
+        return matrix2numpy(self.lc.ned2ecef_matrix)
+
+    @property
+    def ecef2ned_matrix(self):
+        return matrix2numpy(self.lc.ecef2ned_matrix)
+
+    @property
+    def ned_from_ecef_matrix(self):
+        return self.ecef2ned_matrix
+
+    @property
+    def ecef_from_ned_matrix(self):
+        return self.ned2ecef_matrix
+
+    @classmethod
+    def from_geodetic(cls, geodetic):
+        return cls(geodetic=geodetic)
+
+    @classmethod
+    def from_ecef(cls, ecef):
+        return cls(ecef=ecef)
+
+    def ecef2ned_single(self, ecef):
+        assert self.lc
+        cdef ECEF e = list2ecef(ecef)
+        cdef NED n = self.lc.ecef2ned(e)
+        return [n.n, n.e, n.d]
+
+    def ned2ecef_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef ECEF e = self.lc.ned2ecef(n)
+        return [e.x, e.y, e.z]
+
+    def geodetic2ned_single(self, geodetic):
+        assert self.lc
+        cdef Geodetic g = list2geodetic(geodetic)
+        cdef NED n = self.lc.geodetic2ned(g)
+        return [n.n, n.e, n.d]
+
+    def ned2geodetic_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef Geodetic g = self.lc.ned2geodetic(n)
+        return [g.lat, g.lon, g.alt]
+
+    def __dealloc__(self):
+        del self.lc

common/util.cc

@@ -181,9 +181,9 @@ bool file_exists(const std::string& fn) {
 }
 
 static bool createDirectory(std::string dir, mode_t mode) {
-  auto verify_dir = [](const std::string& path) -> bool {
+  auto verify_dir = [](const std::string& dir) -> bool {
     struct stat st = {};
-    return (stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR);
+    return (stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR);
   };
   // remove trailing /'s
   while (dir.size() > 1 && dir.back() == '/') {
@@ -288,7 +288,7 @@ std::string strip(const std::string &str) {
 std::string check_output(const std::string& command) {
   char buffer[128];
   std::string result;
-  std::unique_ptr<FILE, int(*)(FILE*)> pipe(popen(command.c_str(), "r"), pclose);
+  std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(command.c_str(), "r"), pclose);
 
   if (!pipe) {
     return "";
@@ -303,7 +303,7 @@ std::string check_output(const std::string& command) {
 
 bool system_time_valid() {
   // Default to August 26, 2024
-  tm min_tm = {.tm_mday = 26, .tm_mon = 7, .tm_year = 2024 - 1900};
+  tm min_tm = {.tm_year = 2024 - 1900, .tm_mon = 7, .tm_mday = 26};
   time_t min_date = mktime(&min_tm);
 
   struct stat st;

common/util.h

@@ -97,13 +97,6 @@ bool create_directories(const std::string &dir, mode_t mode);
 
 std::string check_output(const std::string& command);
 
-inline void check_system(const std::string& cmd) {
-  int ret = std::system(cmd.c_str());
-  if (ret != 0) {
-    fprintf(stderr, "system command failed (%d): %s\n", ret, cmd.c_str());
-  }
-}
-
 bool system_time_valid();
 
 inline void sleep_for(const int milliseconds) {

common/util.py

@@ -0,0 +1,46 @@
+import os
+import subprocess
+
+def sudo_write(val: str, path: str) -> None:
+  try:
+    with open(path, 'w') as f:
+      f.write(str(val))
+  except PermissionError:
+    os.system(f"sudo chmod a+w {path}")
+    try:
+      with open(path, 'w') as f:
+        f.write(str(val))
+    except PermissionError:
+      # fallback for debugfs files
+      os.system(f"sudo su -c 'echo {val} > {path}'")
+
+def sudo_read(path: str) -> str:
+  try:
+    return subprocess.check_output(f"sudo cat {path}", shell=True, encoding='utf8').strip()
+  except Exception:
+    return ""
+
+class MovingAverage:
+  def __init__(self, window_size: int):
+    self.window_size: int = window_size
+    self.buffer: list[float] = [0.0] * window_size
+    self.index: int = 0
+    self.count: int = 0
+    self.sum: float = 0.0
+
+  def add_value(self, new_value: float):
+    # Update the sum: subtract the value being replaced and add the new value
+    self.sum -= self.buffer[self.index]
+    self.buffer[self.index] = new_value
+    self.sum += new_value
+
+    # Update the index in a circular manner
+    self.index = (self.index + 1) % self.window_size
+
+    # Track the number of added values (for partial windows)
+    self.count = min(self.count + 1, self.window_size)
+
+  def get_average(self) -> float:
+    if self.count == 0:
+      return float('nan')
+    return self.sum / self.count

common/utils.py

@@ -7,82 +7,14 @@ import time
 import functools
 from subprocess import Popen, PIPE, TimeoutExpired
 import zstandard as zstd
+from openpilot.common.swaglog import cloudlog
 
 LOG_COMPRESSION_LEVEL = 10  # little benefit up to level 15. level ~17 is a small step change
 
-class Timer:
-  """Simple lap timer for profiling sequential operations."""
-
-  def __init__(self):
-    self._start = self._lap = time.monotonic()
-    self._sections = {}
-
-  def lap(self, name):
-    now = time.monotonic()
-    self._sections[name] = now - self._lap
-    self._lap = now
-
-  @property
-  def total(self):
-    return time.monotonic() - self._start
-
-  def fmt(self, duration):
-    parts = ", ".join(f"{k}={v:.2f}s" + (f" ({duration/v:.0f}x)" if k == 'render' and v > 0 else "") for k, v in self._sections.items())
-    total = self.total
-    realtime = f"{duration/total:.1f}x realtime" if total > 0 else "N/A"
-    return f"{duration}s in {total:.1f}s ({realtime}) | {parts}"
-
-def sudo_write(val: str, path: str) -> None:
-  try:
-    with open(path, 'w') as f:
-      f.write(str(val))
-  except PermissionError:
-    os.system(f"sudo chmod a+w {path}")
-    try:
-      with open(path, 'w') as f:
-        f.write(str(val))
-    except PermissionError:
-      # fallback for debugfs files
-      os.system(f"sudo su -c 'echo {val} > {path}'")
-
-
-def sudo_read(path: str) -> str:
-  try:
-    return subprocess.check_output(["sudo", "cat", "--", path], encoding='utf8').strip()
-  except Exception:
-    return ""
-
-
-class MovingAverage:
-  def __init__(self, window_size: int):
-    self.window_size: int = window_size
-    self.buffer: list[float] = [0.0] * window_size
-    self.index: int = 0
-    self.count: int = 0
-    self.sum: float = 0.0
-
-  def add_value(self, new_value: float):
-    # Update the sum: subtract the value being replaced and add the new value
-    self.sum -= self.buffer[self.index]
-    self.buffer[self.index] = new_value
-    self.sum += new_value
-
-    # Update the index in a circular manner
-    self.index = (self.index + 1) % self.window_size
-
-    # Track the number of added values (for partial windows)
-    self.count = min(self.count + 1, self.window_size)
-
-  def get_average(self) -> float:
-    if self.count == 0:
-      return float('nan')
-    return self.sum / self.count
-
 
 class CallbackReader:
   """Wraps a file, but overrides the read method to also
   call a callback function with the number of bytes read so far."""
-
   def __init__(self, f, callback, *args):
     self.f = f
     self.callback = callback
@@ -100,8 +32,8 @@ class CallbackReader:
 
 
 @contextlib.contextmanager
-def atomic_write(path: str, mode: str = 'w', buffering: int = -1, encoding: str | None = None, newline: str | None = None,
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str | None = None, newline: str | None = None,
-                 overwrite: bool = False):
+                        overwrite: bool = False):
   """Write to a file atomically using a temporary file in the same directory as the destination file."""
   dir_name = os.path.dirname(path)
 
@@ -131,11 +63,11 @@ def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.Buffered
     return compressed_stream, compressed_size
 
 
-# remove all keys that end in DEPRECATED, plus any "deprecated" group
+# remove all keys that end in DEPRECATED
 def strip_deprecated_keys(d):
   for k in list(d.keys()):
     if isinstance(k, str):
-      if k.endswith('DEPRECATED') or k == 'deprecated':
+      if k.endswith('DEPRECATED'):
         d.pop(k)
       elif isinstance(d[k], dict):
         strip_deprecated_keys(d[k])
@@ -167,92 +99,6 @@ def managed_proc(cmd: list[str], env: dict[str, str]):
       proc.kill()
 
 
-def tabulate(tabular_data, headers=(), tablefmt="simple", floatfmt="g", stralign="left", numalign=None):
-  rows = [list(row) for row in tabular_data]
-
-  def fmt(val):
-    if isinstance(val, str):
-      return val
-    if isinstance(val, (bool, int)):
-      return str(val)
-    try:
-      return format(val, floatfmt)
-    except (TypeError, ValueError):
-      return str(val)
-
-  formatted = [[fmt(c) for c in row] for row in rows]
-  hdrs = [str(h) for h in headers] if headers else None
-
-  ncols = max((len(r) for r in formatted), default=0)
-  if hdrs:
-    ncols = max(ncols, len(hdrs))
-  if ncols == 0:
-    return ""
-
-  for r in formatted:
-    r.extend([""] * (ncols - len(r)))
-  if hdrs:
-    hdrs.extend([""] * (ncols - len(hdrs)))
-
-  widths = [0] * ncols
-  if hdrs:
-    for i in range(ncols):
-      widths[i] = len(hdrs[i])
-  for row in formatted:
-    for i in range(ncols):
-      widths[i] = max(widths[i], max(len(ln) for ln in row[i].split('\n')))
-
-  def _align(s, w):
-    if stralign == "center":
-      return s.center(w)
-    return s.ljust(w)
-
-  if tablefmt == "html":
-    parts = ["<table>"]
-    if hdrs:
-      parts.append("<thead>")
-      parts.append("<tr>" + "".join(f"<th>{h}</th>" for h in hdrs) + "</tr>")
-      parts.append("</thead>")
-    parts.append("<tbody>")
-    for row in formatted:
-      parts.append("<tr>" + "".join(f"<td>{c}</td>" for c in row) + "</tr>")
-    parts.append("</tbody>")
-    parts.append("</table>")
-    return "\n".join(parts)
-
-  if tablefmt == "simple_grid":
-    def _sep(left, mid, right):
-      return left + mid.join("\u2500" * (w + 2) for w in widths) + right
-
-    top, mid_sep, bot = _sep("\u250c", "\u252c", "\u2510"), _sep("\u251c", "\u253c", "\u2524"), _sep("\u2514", "\u2534", "\u2518")
-
-    def _fmt_row(cells):
-      split = [c.split('\n') for c in cells]
-      nlines = max(len(s) for s in split)
-      for s in split:
-        s.extend([""] * (nlines - len(s)))
-      return ["\u2502" + "\u2502".join(f" {_align(split[i][li], widths[i])} " for i in range(ncols)) + "\u2502" for li in range(nlines)]
-
-    lines = [top]
-    if hdrs:
-      lines.extend(_fmt_row(hdrs))
-      lines.append(mid_sep)
-    for ri, row in enumerate(formatted):
-      lines.extend(_fmt_row(row))
-      lines.append(mid_sep if ri < len(formatted) - 1 else bot)
-    return "\n".join(lines)
-
-  # simple
-  gap = "  "
-  lines = []
-  if hdrs:
-    lines.append(gap.join(h.ljust(w) for h, w in zip(hdrs, widths, strict=True)))
-    lines.append(gap.join("-" * w for w in widths))
-  for row in formatted:
-    lines.append(gap.join(_align(row[i], widths[i]) for i in range(ncols)))
-  return "\n".join(lines)
-
-
 def retry(attempts=3, delay=1.0, ignore_failure=False):
   def decorator(func):
     @functools.wraps(func)
@@ -261,11 +107,11 @@ def retry(attempts=3, delay=1.0, ignore_failure=False):
         try:
           return func(*args, **kwargs)
         except Exception:
-          print(f"{func.__name__} failed, trying again")
+          cloudlog.exception(f"{func.__name__} failed, trying again")
           time.sleep(delay)
 
       if ignore_failure:
-        print(f"{func.__name__} failed after retry")
+        cloudlog.error(f"{func.__name__} failed after retry")
       else:
         raise Exception(f"{func.__name__} failed after retry")
     return wrapper

common/version.h

@@ -1 +1 @@
-#define COMMA_VERSION "0.11.2"
+#define COMMA_VERSION "0.10.2"

conftest.py

@@ -7,19 +7,26 @@ from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.system.manager import manager
 from openpilot.system.hardware import TICI, HARDWARE
 
-# these are heavy CI-only tests, invoked explicitly in .github/workflows/tests.yaml
+# TODO: pytest-cpp doesn't support FAIL, and we need to create test translations in sessionstart
+# pending https://github.com/pytest-dev/pytest-cpp/pull/147
 collect_ignore = [
+  "selfdrive/ui/tests/test_translations",
   "selfdrive/test/process_replay/test_processes.py",
   "selfdrive/test/process_replay/test_regen.py",
-  # tinygrad JIT has process-global state. Other test files import modeld → tinygrad,
-  # which corrupts JIT captures for test_warp.py in the same process. Run separately in CI.
-  "sunnypilot/modeld_v2/tests/test_warp.py",
 ]
 collect_ignore_glob = [
   "selfdrive/debug/*.py",
+  "selfdrive/modeld/*.py",
+  "sunnypilot/modeld*/*.py",
 ]
 
 
+def pytest_sessionstart(session):
+  # TODO: fix tests and enable test order randomization
+  if session.config.pluginmanager.hasplugin('randomly'):
+    session.config.option.randomly_reorganize = False
+
+
 @pytest.hookimpl(hookwrapper=True, trylast=True)
 def pytest_runtest_call(item):
   # ensure we run as a hook after capturemanager's
@@ -91,3 +98,15 @@ def pytest_collection_modifyitems(config, items):
       class_property_name = item.get_closest_marker('xdist_group_class_property').args[0]
       class_property_value = getattr(item.cls, class_property_name)
       item.add_marker(pytest.mark.xdist_group(class_property_value))
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_configure(config):
+  config_line = "xdist_group_class_property: group tests by a property of the class that contains them"
+  config.addinivalue_line("markers", config_line)
+
+  config_line = "nocapture: don't capture test output"
+  config.addinivalue_line("markers", config_line)
+
+  config_line = "shared_download_cache: share download cache between tests"
+  config.addinivalue_line("markers", config_line)