ref

* Revert "modeld: fix capnp memory leak (#38117)"

This reverts commit 0f17a98793.

* Revert "fix memory leak from pycapnp 2.2+ (#38001)"

This reverts commit 294cb687f6.

* downgrade

* agnos 18.4

.clang-tidy

@@ -1,19 +0,0 @@
----
-Checks: '
-bugprone-*,
--bugprone-integer-division,
--bugprone-narrowing-conversions,
-performance-*,
-clang-analyzer-*,
-misc-*,
--misc-unused-parameters,
-modernize-*,
--modernize-avoid-c-arrays,
--modernize-deprecated-headers,
--modernize-use-auto,
--modernize-use-using,
--modernize-use-nullptr,
--modernize-use-trailing-return-type,
-'
-CheckOptions:
-...

.codespellignore

@@ -3,3 +3,4 @@ REGIST
 PullRequest
 cancelled
 FOF
+NoO

.dockerignore

@@ -13,27 +13,6 @@
 *.o-*
 *.os
 *.os-*
-*.so
-*.a
 
 venv/
 .venv/
-
-notebooks
-phone
-massivemap
-neos
-installer
-chffr/app2
-chffr/backend/env
-selfdrive/nav
-selfdrive/baseui
-selfdrive/test/simulator2
-**/cache_data
-xx/plus
-xx/community
-xx/projects
-!xx/projects/eon_testing_master
-!xx/projects/map3d
-xx/ops
-xx/junk

.gitattributes

@@ -7,15 +7,8 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
+*.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
 system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
-third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
-third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
-third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
-third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/enhancement.md

@@ -1,8 +0,0 @@
----
-name: Enhancement
-about: For openpilot enhancement suggestions
-title: ''
-labels: 'enhancement'
-assignees: ''
----
-

.github/labeler.yaml

@@ -16,7 +16,7 @@ simulation:
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{selfdrive/ui/**,system/ui/**}'
+    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
 
 tools:
   - changed-files:

.github/workflows/auto-cache/action.yaml

@@ -1,58 +0,0 @@
-name: 'automatically cache based on current runner'
-
-inputs:
-  path:
-    description: 'path to cache'
-    required: true
-  key:
-    description: 'key'
-    required: true
-  restore-keys:
-    description: 'restore-keys'
-    required: true
-  save:
-    description: 'whether to save the cache'
-    default: 'true'
-    required: false
-outputs:
-  cache-hit:
-    description: 'cache hit occurred'
-    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: setup namespace cache
-      id: ns-cache
-      if: ${{ contains(runner.name, 'nsc') }}
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        path: ${{ inputs.path }}
-
-    - name: setup github cache
-      id: gha-cache
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
-      uses: 'actions/cache@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    - name: setup github cache
-      id: gha-cache-ro
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
-      uses: 'actions/cache/restore@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
-    - id: scons-cache-setup
-      shell: bash
-      run: |
-        mkdir -p ${{ inputs.path }}
-        sudo chmod -R 777 ${{ inputs.path }}
-        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -12,12 +12,12 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v5.0.0
+    - uses: actions/labeler@v6
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -40,10 +40,10 @@ jobs:
     runs-on: ubuntu-latest
     if: (github.event.pull_request.head.repo.fork && (contains(github.event_name, 'pull_request') && github.event.action == 'synchronize'))
     env:
-      PR_LABEL: 'dev-c3'
+      PR_LABEL: 'dev'
       TRUST_FORK_PR_LABEL: 'trust-fork-pr'
     steps:
-      - name: Check if PR has dev-c3 label
+      - name: Check if PR has dev label
         id: check-labels
         uses: actions/github-script@v7
         with:
@@ -55,18 +55,18 @@ jobs:
               repo: context.repo.repo,
               issue_number: prNumber
             });
-            
+
             const hasDevC3Label = labels.some(label => label.name === process.env.PR_LABEL);
             const hasTrustLabel = labels.some(label => label.name === process.env.TRUST_FORK_PR_LABEL);
-            
+
             console.log(`PR #${prNumber} has ${process.env.PR_LABEL} label: ${hasDevC3Label}`);
             console.log(`PR #${prNumber} has ${process.env.TRUST_FORK_PR_LABEL} label: ${hasTrustLabel}`);
-            
-            core.setOutput('has-dev-c3', hasDevC3Label ? 'true' : 'false');
+
+            core.setOutput('has-dev', hasDevC3Label ? 'true' : 'false');
             core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
 
       - name: Remove trust-fork-pr label if present
-        if: steps.check-labels.outputs.has-dev-c3 == 'true' && steps.check-labels.outputs.has-trust == 'true'
+        if: steps.check-labels.outputs.has-dev == 'true' && steps.check-labels.outputs.has-trust == 'true'
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -81,7 +81,7 @@ jobs:
             });
 
             console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
-            
+
             // Add a comment to the PR
             await github.rest.issues.createComment({
               owner: context.repo.owner,

.github/workflows/badges.yaml

@@ -5,9 +5,7 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: sunnypilot-base
-  DOCKER_REGISTRY: ghcr.io/sunnypilot
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
+  PYTHONPATH: ${{ github.workspace }}
 
 jobs:
   badges:
@@ -17,13 +15,13 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Push badges
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py"
+        python3 selfdrive/ui/translations/create_badges.py
 
         rm .gitattributes
 

.github/workflows/build-all-tinygrad-models.yaml

@@ -34,10 +34,10 @@ jobs:
           echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
           echo "tinygrad_ref is $ref"
 
-      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
+      - name: Checkout docs repo (sunnypilot-models, gh-pages)
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-docs
+          repository: sunnypilot/sunnypilot-models
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -74,7 +74,7 @@ jobs:
         env:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
           cd gitlab_docs
           git checkout main
           git sparse-checkout set --no-cone models/
@@ -120,6 +120,7 @@ jobs:
     with:
       upstream_branch: ${{ matrix.model.ref }}
       custom_name: ${{ matrix.model.display_name }}
+      is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
     secrets: inherit
@@ -140,7 +141,7 @@ jobs:
         run: |
           echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
           built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            ls output | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
+            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
           )
           jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
             'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
@@ -157,6 +158,7 @@ jobs:
     with:
       upstream_branch: ${{ matrix.model.ref }}
       custom_name: ${{ matrix.model.display_name }}
+      is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
       artifact_suffix: -retry
@@ -168,6 +170,7 @@ jobs:
     if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       max-parallel: 1
       matrix:
         model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
@@ -191,7 +194,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
@@ -201,7 +204,7 @@ jobs:
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-docs
+          repository: sunnypilot/sunnypilot-models
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

.github/workflows/build-single-tinygrad-model.yaml

@@ -24,6 +24,11 @@ on:
         required: false
         type: string
         default: ''
+      is_20hz:
+        description: 'Is this a 20Hz model'
+        required: false
+        type: boolean
+        default: true
       bypass_push:
         description: 'Bypass pushing to GitLab for build-all'
         required: false
@@ -39,6 +44,11 @@ on:
         description: 'Custom name for the model (no date, only name)'
         required: false
         type: string
+      is_20hz:
+        description: 'Is this a 20Hz model'
+        required: false
+        type: boolean
+        default: true
       recompiled_dir:
         description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
         required: true
@@ -82,7 +92,7 @@ jobs:
     with:
       upstream_branch: ${{ inputs.upstream_branch }}
       custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
-      is_20hz: true
+      is_20hz: ${{ inputs.is_20hz }}
       artifact_suffix: ${{ inputs.artifact_suffix }}
     secrets: inherit
 
@@ -109,7 +119,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
@@ -119,7 +129,7 @@ jobs:
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-docs
+          repository: sunnypilot/sunnypilot-models
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

.github/workflows/cereal_validation.yaml

@@ -20,57 +20,46 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  PYTHONWARNINGS: error
-  BASE_IMAGE: openpilot-base
-  BUILD: selfdrive/test/docker_build.sh base
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  CI: 1
 
 jobs:
-  generate_cereal_artifact:
-    name: Generate cereal validation artifacts
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
-      - name: Generate the log file
-        run: |
-          ${{ env.RUN }} "cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema_instances.bin" && \
-          ls -la
-          ls -la cereal/messaging/tests
-      - name: 'Prepare artifact'
-        run: |
-          mkdir -p "cereal/messaging/tests/cereal_validations"
-          cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py"
-          cp schema_instances.bin "cereal/messaging/tests/cereal_validations/schema_instances.bin"
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v4
-        with:
-          name: cereal_validations
-          path: cereal/messaging/tests/cereal_validations
-
   validate_cereal_with_upstream:
     name: Validate cereal with Upstream
     runs-on: ubuntu-24.04
-    needs: generate_cereal_artifact
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout sunnypilot cereal
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: cereal
+
+      - name: Init sunnypilot opendbc submodule
+        run: git submodule update --init --depth 1 opendbc_repo
+
+      - name: Checkout upstream openpilot cereal
+        uses: actions/checkout@v6
         with:
           repository: 'commaai/openpilot'
-          submodules: true
+          path: upstream_openpilot
+          sparse-checkout: cereal
           ref: "refs/heads/master"
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
-      - name: Download build artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: cereal_validations
-          path: cereal/messaging/tests/cereal_validations
-      - name: 'Run the validation'
+
+      - name: Init upstream opendbc submodule
+        working-directory: upstream_openpilot
+        run: git submodule update --init --depth 1 opendbc_repo
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Generate sunnypilot schema
         run: |
-          chmod +x cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py
-          ${{ env.RUN }} "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f cereal/messaging/tests/cereal_validations/schema_instances.bin"
+          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
+            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
+            -g -f /tmp/sp_schema.json --cereal-dir cereal
+
+      - name: Validate against upstream
+        run: |
+          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
+            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
+            -r -f /tmp/sp_schema.json --cereal-dir upstream_openpilot/cereal

.github/workflows/ci_weekly_report.yaml

@@ -1,101 +0,0 @@
-name: weekly CI test report
-on:
-  schedule:
-    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
-  workflow_dispatch:
-    inputs:
-      ci_runs:
-        description: 'The amount of runs to trigger in CI test report'
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
-
-jobs:
-  setup:
-    if: github.repository == 'sunnypilot/sunnypilot'
-    runs-on: ubuntu-latest
-    outputs:
-      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
-    steps:
-      - id: ci_runs_setup
-        name: CI_RUNS=${{ env.CI_RUNS }}
-        run: |
-          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
-
-  ci_matrix_run:
-    needs: [ setup ]
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
-    uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
-    with:
-      run_number: ${{ matrix.run_number }}
-
-  report:
-    needs: [ci_matrix_run]
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Get job results
-        uses: actions/github-script@v7
-        id: get-job-results
-        with:
-          script: |
-            const jobs = await github
-              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
-                owner: "commaai",
-                repo: "${{ github.event.repository.name }}",
-                run_id: "${{ github.run_id }}",
-                attempt: "${{ github.run_attempt }}",
-              })
-            var report = {}
-            jobs.slice(1, jobs.length-1).forEach(job => {
-              if (job.conclusion === "skipped") return;
-              const jobName = job.name.split(" / ")[2];
-              const runRegex = /\((.*?)\)/;
-              const run = job.name.match(runRegex)[1];
-              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
-              switch (job.conclusion) {
-                case "success":
-                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
-                case "failure":
-                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
-                case "canceled":
-                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
-              }
-            });
-            return JSON.stringify({"jobs": report});
-
-      - name: Add job results to summary
-        env:
-          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
-        run: |
-          cat <<EOF >> template.html
-          <table>
-            <thead>
-              <tr>
-                <th></th>
-                <th>Job</th>
-                <th>✅ Passing</th>
-                <th>❌ Failure Details</th>
-              </tr>
-            </thead>
-            <tbody>
-              {% for key in jobs.keys() %}<tr>
-                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
-                <td>{{ key }}</td>
-                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
-                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
-                </td>
-              </tr>{% endfor %}
-          </table>
-          EOF
-
-          pip install jinja2-cli
-          echo $JOB_RESULTS | jinja2 template.html > report.html
-          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
-          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -1,17 +0,0 @@
-name: weekly CI test run
-on:
-  workflow_call:
-    inputs:
-      run_number:
-        required: true
-        type: string
-
-concurrency:
-  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  selfdrive_tests:
-    uses: sunnypilot/sunnypilot/.github/workflows/selfdrive_tests.yaml@master
-    with:
-      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -1,21 +0,0 @@
-name: 'compile openpilot'
-
-runs:
-  using: "composite"
-  steps:
-    - shell: bash
-      name: Build openpilot with all flags
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - shell: bash
-      name: Cleanup scons cache and rebuild
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-    - name: Save scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/diff_report.yaml

@@ -0,0 +1,45 @@
+name: diff report
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  comment:
+    name: comment
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+    - name: Wait for process replay
+      id: wait
+      continue-on-error: true
+      uses: lewagon/wait-on-check-action@v1.3.4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        check-name: process replay
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        allowed-conclusions: success,failure
+        wait-interval: 20
+    - name: Download diff
+      if: steps.wait.outcome == 'success'
+      uses: dawidd6/action-download-artifact@v6
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        workflow: tests.yaml
+        workflow_conclusion: ''
+        pr: ${{ github.event.number }}
+        name: diff_report_${{ github.event.number }}
+        path: .
+        allow_forks: true
+    - name: Comment on PR
+      if: steps.wait.outcome == 'success'
+      uses: thollander/actions-comment-pull-request@v2
+      with:
+        filePath: diff_report.txt
+        comment_tag: diff_report
+        pr_number: ${{ github.event.number }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -22,19 +22,19 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: true
 
     # Build
     - name: Build docs
       run: |
-        # TODO: can we install just the "docs" dependency group without the normal deps?
-        pip install mkdocs
-        mkdocs build
+        git lfs pull
+        pip install zensical
+        python scripts/docs.py build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,7 +5,44 @@ on:
     types: [created, edited]
 
 jobs:
-  # TODO: gc old branches in a separate job in this workflow
+  cleanup-branches:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Delete stale Jenkins branches
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
+            const prefixes = ['tmp-jenkins', '__jenkins'];
+
+            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              per_page: 100,
+            })) {
+              for (const branch of response.data) {
+                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
+
+                const { data: commit } = await github.rest.repos.getCommit({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  ref: branch.commit.sha,
+                });
+
+                const commitDate = new Date(commit.commit.committer.date).getTime();
+                if (commitDate < cutoff) {
+                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
+                  await github.rest.git.deleteRef({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    ref: `heads/${branch.name}`,
+                  });
+                }
+              }
+            }
+
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
@@ -15,7 +52,7 @@ jobs:
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v7
+      uses: actions/github-script@v8
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -35,7 +72,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -49,7 +86,7 @@ jobs:
 
     - name: Delete trigger comment
       if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v7
+      uses: actions/github-script@v8
       with:
         script: |
           await github.rest.issues.deleteComment({

.github/workflows/model_review.yaml

@@ -16,23 +16,23 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+        with:
+          submodules: true
       - name: Checkout master
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           ref: master
           path: base
       - run: git lfs pull
       - run: cd base && git lfs pull
 
-      - run: pip install onnx
-
       - name: scripts/reporter.py
         id: report
         run: |
           echo "content<<EOF" >> $GITHUB_OUTPUT
           echo "## Model Review" >> $GITHUB_OUTPUT
-          MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Post model report comment

.github/workflows/post-to-discourse/action.yml

@@ -0,0 +1,105 @@
+name: 'Post to Discourse'
+description: 'Posts a message to a Discourse topic (existing or new)'
+
+inputs:
+  discourse-url:
+    description: 'Discourse instance URL (e.g., https://discourse.example.com)'
+    required: true
+  api-key:
+    description: 'Discourse API key'
+    required: true
+  api-username:
+    description: 'Discourse API username'
+    required: true
+  topic-id:
+    description: 'Discourse topic ID to post to (use this OR category-id + title)'
+    required: false
+  category-id:
+    description: 'Category ID for new topic (required if topic-id not provided)'
+    required: false
+  title:
+    description: 'Title for new topic (required if topic-id not provided)'
+    required: false
+  message:
+    description: 'Message content (markdown supported)'
+    required: true
+
+outputs:
+  post-number:
+    description: 'The post number in the topic'
+    value: ${{ steps.post.outputs.post_number }}
+  post-url:
+    description: 'Direct URL to the post'
+    value: ${{ steps.post.outputs.post_url }}
+  topic-id:
+    description: 'The topic ID (useful when creating a new topic)'
+    value: ${{ steps.post.outputs.topic_id }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Post to Discourse
+      id: post
+      shell: bash
+      run: |
+        # Validate inputs
+        if [ -z "${{ inputs.topic-id }}" ] && ([ -z "${{ inputs.category-id }}" ] || [ -z "${{ inputs.title }}" ]); then
+          echo "❌ Error: Must provide either topic-id OR both category-id and title"
+          exit 1
+        fi
+        
+        if [ -n "${{ inputs.topic-id }}" ] && ([ -n "${{ inputs.category-id }}" ] || [ -n "${{ inputs.title }}" ]); then
+          echo "⚠️  Warning: Both topic-id and category-id/title provided. Will post to existing topic."
+        fi
+        
+        # Determine if creating new topic or posting to existing
+        if [ -n "${{ inputs.topic-id }}" ]; then
+          echo "📝 Posting to existing topic ID: ${{ inputs.topic-id }}"
+        
+          # Create JSON payload for posting to existing topic
+          PAYLOAD=$(jq -n \
+            --arg content '${{ inputs.message }}' \
+            --arg topic_id "${{ inputs.topic-id }}" \
+            '{topic_id: $topic_id, raw: $content}')
+        else
+          echo "✨ Creating new topic: ${{ inputs.title }}"
+        
+          # Create JSON payload for new topic
+          PAYLOAD=$(jq -n \
+            --arg content '${{ inputs.message }}' \
+            --arg title "${{ inputs.title }}" \
+            --arg category "${{ inputs.category-id }}" \
+            '{title: $title, category: ($category | tonumber), raw: $content}')
+        fi
+        
+        # Post to Discourse
+        RESPONSE=$(curl -s -w "\n%{http_code}" \
+          -X POST "${{ inputs.discourse-url }}/posts.json" \
+          -H "Content-Type: application/json" \
+          -H "Api-Key: ${{ inputs.api-key }}" \
+          -H "Api-Username: ${{ inputs.api-username }}" \
+          -d "$PAYLOAD")
+        
+        HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
+        BODY=$(echo "$RESPONSE" | sed '$d')
+        
+        if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
+          echo "✅ Successfully posted to Discourse!"
+        
+          POST_NUMBER=$(echo "$BODY" | jq -r '.post_number // "unknown"')
+          TOPIC_ID=$(echo "$BODY" | jq -r '.topic_id // "${{ inputs.topic-id }}"')
+          POST_URL="${{ inputs.discourse-url }}/t/${TOPIC_ID}/${POST_NUMBER}"
+        
+          echo "post_number=${POST_NUMBER}" >> $GITHUB_OUTPUT
+          echo "post_url=${POST_URL}" >> $GITHUB_OUTPUT
+          echo "topic_id=${TOPIC_ID}" >> $GITHUB_OUTPUT
+        
+          echo "Topic ID: ${TOPIC_ID}"
+          echo "Post number: ${POST_NUMBER}"
+          echo "URL: ${POST_URL}"
+        else
+          echo "❌ Failed to post to Discourse"
+          echo "HTTP Code: ${HTTP_CODE}"
+          echo "Response: ${BODY}"
+          exit 1
+        fi

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh prebuilt
+  BUILD: release/ci/docker_build_sp.sh
 
 jobs:
   build_prebuilt:
@@ -28,8 +28,8 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v4
+        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
+    - uses: actions/checkout@v6
       with:
         submodules: true
     - run: git lfs pull

.github/workflows/release.yaml

@@ -7,20 +7,12 @@ on:
 jobs:
   build___nightly:
     name: build __nightly
-    env:
-      ImageOS: ubuntu24
-    container:
-      image: ghcr.io/sunnypilot/sunnypilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     permissions:
       checks: read
       contents: write
     steps:
-    - name: Install wait-on-check-action dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
       uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
@@ -29,14 +21,11 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build __nightly'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt).*).)*$
+        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-    - name: Pull LFS
-      run: |
-        git config --global --add safe.directory '*'
-        git lfs pull
+    - run: ./tools/op.sh setup
     - name: Push __nightly
       run: BRANCH=__nightly release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -6,61 +6,54 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: sunnypilot-base
-  BUILD: release/ci/docker_build_sp.sh base
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  PYTHONPATH: ${{ github.workspace }}
 
 jobs:
-  update_translations:
-    runs-on: ubuntu-latest
-    if: github.repository == 'sunnypilot/sunnypilot'
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Update translations
-        run: |
-          ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish"
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
-        with:
-          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
-          commit-message: "Update translations"
-          title: "[bot] Update translations"
-          body: "Automatic PR from repo-maintenance -> update_translations"
-          branch: "update-translations"
-          base: "master"
-          delete-branch: true
-          labels: bot
-
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/sunnypilot/sunnypilot-base:latest
     if: github.repository == 'sunnypilot/sunnypilot'
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: true
+    - run: ./tools/op.sh setup
     - name: uv lock
+      run: uv lock --upgrade
+    - name: uv pip tree
+      id: pip_tree
       run: |
-        python3 -m ensurepip --upgrade
-        pip3 install uv
-        uv lock --upgrade
+        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
+        uv pip tree >> $GITHUB_OUTPUT
+        echo 'EOF' >> $GITHUB_OUTPUT
+    - name: venv size
+      id: venv_size
+      run: |
+        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
+        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
+        echo "" >> $GITHUB_OUTPUT
+        echo "Top 10 by size:" >> $GITHUB_OUTPUT
+        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
+          | grep -v '\.dist-info' \
+          | grep -v '__pycache__' \
+          | sort -rh \
+          | head -10 \
+          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
+        echo 'EOF' >> $GITHUB_OUTPUT
     - name: bump submodules
       run: |
-        git config --global --add safe.directory '*'
+        git config submodule.msgq.update none
+        git config submodule.rednose_repo.update none
+        git config submodule.teleoprtc_repo.update none
         git config submodule.tinygrad.update none
         git submodule update --remote
         git add .
     - name: update car docs
       run: |
-        export PYTHONPATH="$PWD"
-        scons -j$(nproc) --minimal opendbc_repo
         python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
+      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
       with:
         author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
         token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
@@ -69,5 +62,16 @@ jobs:
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: 'Automatic PR from repo-maintenance -> package_updates'
+        body: |
+          Automatic PR from repo-maintenance -> package_updates
+
+          ```
+          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
+          ${{ steps.venv_size.outputs.VENV_SIZE }}
+          ```
+
+          ```
+          $ uv pip tree
+          ${{ steps.pip_tree.outputs.PIP_TREE }}
+          ```
         labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -1,309 +0,0 @@
-name: selfdrive
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  PYTHONWARNINGS: error
-  BASE_IMAGE: sunnypilot-base
-  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
-
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh base
-
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
-  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
-
-jobs:
-  build_release:
-    name: build release
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot and run checks
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "python3 system/manager/build.py"
-    - name: Run tests
-      timeout-minutes: 1
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - name: Check submodules
-      if: github.repository == 'sunnypilot/sunnypilot'
-      timeout-minutes: 3
-      run: |
-        if [ "${{ github.ref }}" != "refs/heads/master" ]; then
-          git fetch  origin master:refs/remotes/origin/master
-
-          SUBMODULE_PATHS=$(git diff origin/master HEAD --name-only | grep -E '^[^/]+$' | while read path; do
-            if git ls-files --stage "$path" | grep -q "^160000"; then
-              echo "$path"
-            fi
-          done | tr '\n' ' ')
-
-          if [ -n "$SUBMODULE_PATHS" ]; then
-            echo "Changed submodule paths: $SUBMODULE_PATHS"
-            export SUBMODULE_PATHS="$SUBMODULE_PATHS"
-            export CHECK_PR_REFS=true
-          fi
-        fi
-        release/check-submodules.sh
-
-  build:
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Setup docker push
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'sunnypilot/sunnypilot'
-      run: |
-        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
-    - uses: ./.github/workflows/setup-with-retry
-    - uses: ./.github/workflows/compile-openpilot
-      timeout-minutes: 30
-
-  build_mac:
-    name: build macOS
-    if: false  # temp disable since homebrew install is getting stuck
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - name: Homebrew cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: ~/Library/Caches/Homebrew
-        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          brew-macos-${{ env.CACHE_COMMIT_DATE }}
-          brew-macos
-    - name: Install dependencies
-      run: ./tools/mac_setup.sh
-      env:
-        # package install has DeprecationWarnings
-        PYTHONWARNINGS: default
-    - name: Save Homebrew cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: ~/Library/Caches/Homebrew
-        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-    - run: git lfs pull
-    - name: Getting scons cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}-macos
-    - name: Building openpilot
-      run: . .venv/bin/activate && scons -j$(nproc)
-    - name: Save scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-
-  static_analysis:
-    name: static analysis
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      PYTHONWARNINGS: default
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Static analysis
-      timeout-minutes: 1
-      run: ${{ env.RUN }} "scripts/lint/lint.sh"
-
-  unit_tests:
-    name: unit tests
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 999 }}
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        # Pre-compile Python bytecode so each pytest worker doesn't need to
-                        $PYTEST --collect-only -m 'not slow' -qq && \
-                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
-                        ./selfdrive/ui/tests/create_test_translations.sh && \
-                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
-                        chmod -R 777 /tmp/comma_download_cache"
-
-  process_replay:
-    name: process replay
-    if: false  # disable process_replay for forks
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v4
-      with:
-        path: .ci_cache/comma_download_cache
-        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
-      run: |
-        ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
-                        chmod -R 777 /tmp/comma_download_cache"
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v4
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload reference logs
-      if: false # TODO: move this to github instead of azure
-      run: |
-        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      run: |
-        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-                        chmod -R 777 /tmp/comma_download_cache"
-
-  simulator_driving:
-    name: simulator driving
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Driving test
-      timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        source selfdrive/test/setup_vsound.sh && \
-                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
-
-  create_ui_report:
-    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
-    name: Create UI Report
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: false  # FIXME: FrameReader is broken on CI runners
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - uses: ./.github/workflows/setup-with-retry
-      - name: caching frames
-        id: frames-cache
-        uses: actions/cache@v4
-        with:
-          path: .ci_cache/comma_download_cache
-          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc)"
-      - name: Create Test Report
-        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 2 || 4) }}
-        run: >
-            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-                            source selfdrive/test/setup_xvfb.sh &&
-                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
-                            chmod -R 777 /tmp/comma_download_cache"
-      - name: Upload Test Report
-        uses: actions/upload-artifact@v4
-        with:
-          name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-with-retry/action.yaml

@@ -1,52 +0,0 @@
-name: 'openpilot env setup, with retry on failure'
-
-inputs:
-  docker_hub_pat:
-    description: 'Auth token for Docker Hub, required for BuildJet jobs'
-    required: false
-    default: ''
-  sleep_time:
-    description: 'Time to sleep between retries'
-    required: false
-    default: 30
-
-outputs:
-  duration:
-    description: 'Duration of the setup process in seconds'
-    value: ${{ steps.get_duration.outputs.duration }}
-
-runs:
-  using: "composite"
-  steps:
-    - id: start_time
-      shell: bash
-      run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
-    - id: setup1
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup1.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup2
-      if: steps.setup1.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup2.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup3
-      if: steps.setup2.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      with:
-        is_retried: true
-    - id: get_duration
-      shell: bash
-      run: |
-        END_TIME=$(date +%s)
-        DURATION=$((END_TIME - START_TIME))
-        echo "Total duration: $DURATION seconds"
-        echo "duration=$DURATION" >> $GITHUB_OUTPUT

.github/workflows/setup/action.yaml

@@ -1,56 +0,0 @@
-name: 'openpilot env setup'
-
-inputs:
-  is_retried:
-    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
-    required: false
-    default: 'false'
-
-runs:
-  using: "composite"
-  steps:
-    # assert that this action is retried using the setup-with-retry
-    - shell: bash
-      if: ${{ inputs.is_retried == 'false' }}
-      run: |
-        echo "You should not run this action directly. Use setup-with-retry instead"
-        exit 1
-
-    - shell: bash
-      name: No retries!
-      run: |
-        if [ "${{ github.run_attempt }}" -gt ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association == 'NONE' && 2 || 1}} ]; then
-          echo -e "\033[0;31m##################################################"
-          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
-          echo -e "\033[0;31m##################################################\033[0m"
-          exit 1
-        fi
-
-    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
-    - shell: bash
-      run: git lfs pull
-
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
-    - id: scons-cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}
-    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
-    - id: normalize-file-permissions
-      shell: bash
-      name: Normalize file permissions to ensure a consistent docker build cache
-      run: |
-        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
-        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
-    # build our docker image
-    - shell: bash
-      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -5,15 +5,15 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 2
-  DAYS_BEFORE_PR_STALE: 9
+  DAYS_BEFORE_PR_CLOSE: 7
+  DAYS_BEFORE_PR_STALE: 24
   DAYS_BEFORE_PR_STALE_DRAFT: 30
 
 jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@v10
         with:
           exempt-all-milestones: true
 
@@ -34,7 +34,7 @@ jobs:
   stale_drafts:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@v10
         with:
           exempt-all-milestones: true
 

.github/workflows/sunnypilot-build-model.yaml

@@ -156,24 +156,71 @@ jobs:
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
           path: ${{ github.workspace }}/selfdrive/modeld/models
+      - run: |
+          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |
           source /etc/profile
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}"
+          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
 
-          # Loop through all .onnx files
+          COMPILE_MODELD="${{ github.workspace }}/sunnypilot/modeld_v2/compile_modeld.py"
+          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
+          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
+          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+
+          # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
-            base_name=$(basename "$onnx_file" .onnx)
-            output_file="${{ env.MODELS_DIR }}/${base_name}_tinygrad.pkl"
-
-            echo "Compiling: $onnx_file -> $output_file"
-            QCOM=1 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file"
-            QCOM=1 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            echo "Generating metadata: $onnx_file"
+            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
+          # Detect model type and build compile args
+          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
+          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
+          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
+          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
+          SUPERCOMBO_ONNX="${{ env.MODELS_DIR }}/supercombo.onnx"
+
+          MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
+          if [ -f "$VISION_ONNX" ]; then
+            ONNX_ARGS="--vision-onnx $VISION_ONNX"
+            if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
+              MODEL_TYPE=vision_multi_policy
+              ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
+            elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
+              MODEL_TYPE=vision_multi_policy
+              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
+            elif [ -f "$POLICY_ONNX" ]; then
+              MODEL_TYPE=vision_policy
+              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
+            fi
+          elif [ -f "$SUPERCOMBO_ONNX" ]; then
+            MODEL_TYPE=supercombo
+            ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
+          fi
+
+          if [ -n "$MODEL_TYPE" ]; then
+            echo "Detected: $MODEL_TYPE -> driving_tinygrad.pkl"
+            env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
+              --model-type $MODEL_TYPE \
+              --model-size $MODEL_SIZE \
+              --camera-resolutions $CAMERA_RES \
+              $ONNX_ARGS \
+              --output "${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
+          fi
+
+      - name: Validate Model Outputs
+        run: |
+          source /etc/profile
+          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
+          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
+          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
+            --validate-only \
+            --model-dir "${{ env.MODELS_DIR }}"
+
       - name: Prepare Output
         run: |
           sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -182,8 +229,9 @@ jobs:
           # Copy the model files
           rsync -avm \
             --include='*.dlc' \
-            --include='*.thneed' \
             --include='*.pkl' \
+            --include='*.chunk*' \
+            --include='*.chunkmanifest' \
             --include='*.onnx' \
             --exclude='*' \
             --delete-excluded \

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -6,57 +6,121 @@ env:
   CI_DIR: ${{ github.workspace }}/release/ci
   SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
-  
+
   # Branch configurations
-  STAGING_C3_SOURCE_BRANCH: ${{ vars.STAGING_C3_SOURCE_BRANCH || 'master' }} # vars are set on repo settings.
-  DEV_C3_SOURCE_BRANCH: ${{ vars.DEV_C3_SOURCE_BRANCH || 'master-dev-c3-new' }} # vars are set on repo settings.
-  
-  # Target branch configurations
-  STAGING_TARGET_BRANCH: ${{ vars.STAGING_TARGET_BRANCH || 'staging-c3-new' }} # vars are set on repo settings.
-  DEV_TARGET_BRANCH: ${{ vars.DEV_TARGET_BRANCH || 'dev-c3-new' }} # vars are set on repo settings.
-  RELEASE_TARGET_BRANCH: ${{ vars.RELEASE_TARGET_BRANCH || 'release-c3-new' }} # vars are set on repo settings.
-  
+  STAGING_SOURCE_BRANCH: 'master'
+
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
 on:
   push:
-    branches: [ master, master-dev-c3-new ]
-    tags: [ '*' ]
+    branches: [ master, master-dev ]
+    tags: [ 'release/*' ]
   pull_request_target:
     types: [ labeled ]
   workflow_dispatch:
     inputs:
       wait_for_tests:
-        description: 'Wait for selfdrive_tests to finish'
+        description: 'Wait for tests to finish'
         required: false
         type: boolean
         default: false
 
 jobs:
+  prepare_strategy:
+    runs-on: ubuntu-24.04
+    if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+    outputs:
+      environment: ${{ steps.strategy.outputs.environment }}
+      new_branch: ${{ steps.strategy.outputs.new_branch }}
+      extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
+      version: ${{ steps.strategy.outputs.version }}
+      cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
+      publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
+      is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
+      build: ${{ steps.strategy.outputs.build }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Extract deploy strategy
+        id: strategy
+        run: |
+          echo '::group::Strategy Extraction'
+          BRANCH="${{ github.head_ref || github.ref_name }}"
+          echo "Current branch: $BRANCH"
+
+          STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
+          CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
+            .configs[] | select(.branch == $branch)
+          ')
+
+          BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
+          if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
+            echo "No exact strategy match found. Falling back to feature/fork logic."
+            IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
+            FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
+            NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
+
+            echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
+            echo "version=$BUILD" >> $GITHUB_OUTPUT
+            echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
+            echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
+            echo "environment=feature-branch" >> $GITHUB_OUTPUT
+            echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
+          else
+          echo "Matched config: $CONFIG"
+            environment=$(echo "$CONFIG" | jq -r '.environment')
+            echo "environment=$environment" >> $GITHUB_OUTPUT
+            echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
+            cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
+            echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
+            echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
+
+            is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
+            echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
+
+            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
+            echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
+            echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
+          fi
+          echo "build=$BUILD" >> $GITHUB_OUTPUT
+          cat $GITHUB_OUTPUT
+
   validate_tests:
     runs-on: ubuntu-24.04
-    if: ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) || contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+    needs: [ prepare_strategy ]
+    if: ${{
+      ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
+      (github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
+      contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+      }}
     steps:
       - uses: actions/checkout@v4
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
         with:
-          workflow: selfdrive_tests.yaml # The workflow file to monitor
+          workflow: tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
+          should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
 
   build:
-    needs: [ validate_tests ]
+    needs: [ validate_tests, prepare_strategy ]
     concurrency:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
     runs-on: [self-hosted, tici]
     outputs:
-      new_branch: ${{ steps.set-env.outputs.new_branch }}
-      version: ${{ steps.set-env.outputs.version }}
-      extra_version_identifier: ${{ steps.set-env.outputs.extra_version_identifier }}
-      commit_sha: ${{ steps.set-env.outputs.commit_sha }}
-    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
+      version: ${{ needs.prepare_strategy.outputs.version }}
+      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
+      commit_sha: ${{ github.sha }}
+    if: ${{
+      (always() && !cancelled() && !failure()) &&
+      needs.prepare_strategy.result == 'success' &&
+      (needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
+      (!contains(github.event_name, 'pull_request') ||
+      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+      }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -70,70 +134,21 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_C3_SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Set Feature Branch Prebuilt Configuration
-        id: set_feature_configuration
-        if: (
-          !(env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH) &&
-          !(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH) &&
-          !(startsWith(github.ref, 'refs/tags/'))
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.SOURCE_BRANCH }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-prebuilt" >> $GITHUB_ENV
-          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
-
-      - name: Set dev-c3-new prebuilt Configuration
-        id: set_dev_configuration
-        if: (
-          steps.set_feature_configuration.outcome == 'skipped' &&
-          env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV
-          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
-          echo "EXTRA_VERSION_IDENTIFIER=${{ github.run_number }}" >> $GITHUB_ENV
-
-      - name: Set staging-c3-new prebuilt Configuration
-        id: set_staging_configuration
-        if: (
-          steps.set_feature_configuration.outcome == 'skipped' &&
-          !contains(github.event_name, 'pull_request') &&
-          steps.set_dev_configuration.outcome == 'skipped' &&
-          (env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH)
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV
-          echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV
-          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-staging" >> $GITHUB_ENV
-
-      - name: Set release-c3-new prebuilt Configuration
-        id: set_tag_configuration
-        if: (
-          steps.set_feature_configuration.outcome == 'skipped' &&
-          !contains(github.event_name, 'pull_request') &&
-          steps.set_staging_configuration.outcome == 'skipped' &&
-          startsWith(github.ref, 'refs/tags/')
-          )
-        run: |
-          echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV
-          echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV
-          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-release" >> $GITHUB_ENV
-
       - name: Set environment variables
         id: set-env
         run: |
-          # Write to GITHUB_OUTPUT from environment variables
-          echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
-          [[ ! -z "$EXTRA_VERSION_IDENTIFIER" ]] && echo "extra_version_identifier=$EXTRA_VERSION_IDENTIFIER" >> $GITHUB_OUTPUT
-          [[ ! -z "$VERSION" ]] && echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
+          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
+          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
           echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
-          
+
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
@@ -165,6 +180,13 @@ jobs:
           ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
           cd $BUILD_DIR
           sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
+          echo "Building sunnypilot's modeld_v2..."
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
+          echo "Building sunnypilot's locationd..."
+          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
+          echo "Building openpilot's locationd..."
+          scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
+          echo "Building rest of sunnypilot"
           scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
@@ -176,37 +198,27 @@ jobs:
           sudo rm -rf ${OUTPUT_DIR}
           mkdir -p ${OUTPUT_DIR}
           rsync -am${RUNNER_DEBUG:+v} \
-            --include='**/panda/board/' \
-            --include='**/panda/board/obj' \
-            --include='**/panda/board/obj/panda.bin.signed' \
-            --include='**/panda/board/obj/panda_h7.bin.signed' \
-            --include='**/panda/board/obj/bootstub.panda.bin' \
-            --include='**/panda/board/obj/bootstub.panda_h7.bin' \
             --exclude='.sconsign.dblite' \
             --exclude='*.a' \
             --exclude='*.o' \
             --exclude='*.os' \
             --exclude='*.pyc' \
             --exclude='moc_*' \
-            --exclude='*.cc' \
+            --exclude='__pycache__' \
             --exclude='Jenkinsfile' \
-            --exclude='supercombo.onnx' \
-            --exclude='**/panda/board/*' \
-            --exclude='**/panda/board/obj/**' \
-            --exclude='**/panda/certs/' \
-            --exclude='**/panda/crypto/' \
             --exclude='**/release/' \
             --exclude='**/.github/' \
             --exclude='**/selfdrive/ui/replay/' \
             --exclude='**/__pycache__/' \
-            --exclude='**/selfdrive/ui/*.h' \
-            --exclude='**/selfdrive/ui/**/*.h' \
-            --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' \
             --exclude='${{env.SCONS_CACHE_DIR}}' \
             --exclude='**/.git/' \
             --exclude='**/SConstruct' \
             --exclude='**/SConscript' \
             --exclude='**/.venv/' \
+            --exclude='selfdrive/modeld/models/*.onnx*' \
+            --exclude='sunnypilot/modeld*/models/*.onnx*' \
+            --exclude='third_party/*x86*' \
+            --exclude='third_party/*Darwin*' \
             --delete-excluded \
             --chown=comma:comma \
             ${BUILD_DIR}/ ${OUTPUT_DIR}/
@@ -227,14 +239,18 @@ jobs:
         run: |
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
 
+
   publish:
     concurrency:
-      group: publish-${{ github.head_ref || github.ref_name }}
-      cancel-in-progress: true
-    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build ]
+      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
+      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
+      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
+      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
+      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
+    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+    needs: [ build, prepare_strategy ]
     runs-on: ubuntu-24.04
-    environment: ${{ (contains(fromJSON(vars.AUTO_DEPLOY_PREBUILT_BRANCHES), github.head_ref || github.ref_name) || contains(github.event.pull_request.labels.*.name, 'prebuilt')) && 'auto-deploy' || 'feature-branch' }}
+    environment: ${{ needs.prepare_strategy.outputs.environment }}
     steps:
       - uses: actions/checkout@v4
 
@@ -266,7 +282,7 @@ jobs:
             "${{ needs.build.outputs.new_branch }}" \
             "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "-${{ needs.build.outputs.extra_version_identifier }}"
+            "${{ needs.build.outputs.extra_version_identifier }}"
 
           echo ""
           echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
@@ -275,37 +291,59 @@ jobs:
           echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
           echo "3. Update as needed (JSON array with no spaces)"
 
+      - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
+        if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
+        run: |
+            TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
+            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
+            git push -f origin ${TAG}
+
   notify:
-    needs: [ build, publish ]
+    needs:
+    - prepare_strategy
+    - build
+    - publish
     runs-on: ubuntu-24.04
-    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+    if: ${{ (always() && !cancelled() && !failure())
+      && needs.publish.result == 'success'
+      && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+      && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
     steps:
       - uses: actions/checkout@v4
-      - name: Setup Alpine Linux environment
-        uses: jirutka/setup-alpine@v1.2.0
-        with:
-          packages: 'jq gettext curl'
 
-      - name: Send Discord Notification
-        env:
-          DISCORD_WEBHOOK: ${{ contains(fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES), env.SOURCE_BRANCH) && secrets.DISCORD_DEV_FEEDBACK_CHANNEL_WEBHOOK || secrets.DISCORD_DEV_PRIVATE_CHANNEL_WEBHOOK }}
+      - name: Prepare notification message
+        id: message
         run: |
-          TEMPLATE='${{ vars.DISCORD_GENERAL_UPDATE_NOTICE }}'
-          export EXTRA_VERSION_IDENTIFIER="${{ needs.build.outputs.extra_version_identifier }}"
-          export VERSION="${{ needs.build.outputs.version }}"
-          export branch_name=${{ env.SOURCE_BRANCH }}
-          export new_branch=${{ needs.build.outputs.new_branch }}
-          export extra_version_identifier=${{ needs.build.outputs.extra_version_identifier || github.run_number}}
-          echo ${TEMPLATE} | envsubst | jq -c '.' | tee payload.json
-          curl -X POST -H "Content-Type: application/json" -d @payload.json $DISCORD_WEBHOOK
+          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
+          export VERSION="${{ needs.prepare_strategy.outputs.version }}"
+          export branch_name="${{ env.SOURCE_BRANCH }}"
+          export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
+          export commit_sha="${{ github.sha }}"
+          export commit_short_sha="${{ github.sha }}"
+          export commit_short_sha="${commit_short_sha:0:7}"
+          export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
+          export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
 
-          echo ""
-          echo "----   ℹ️ To update the list of branches that notify to dev-feedback  -----"
-          echo ""
-          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/DEV_FEEDBACK_NOTIFICATION_BRANCHES"
-          echo "2. Current value: ${{ vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES }}"
-          echo "3. Update as needed (JSON array with no spaces)"
-        shell: alpine.sh {0}
+          MESSAGE=$(cat << 'EOF' | envsubst
+          ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
+          EOF
+          )
+
+          {
+            echo 'content<<EOFMARKER'
+            echo "$MESSAGE"
+            echo 'EOFMARKER'
+          } >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Post to Discourse
+        uses: ./.github/workflows/post-to-discourse
+        with:
+          discourse-url: ${{ vars.DISCOURSE_URL }}
+          api-key: ${{ secrets.DISCOURSE_API_KEY }}
+          api-username: "system"
+          topic-id: ${{ fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name].topic_id }}
+          message: ${{ steps.message.outputs.content }}
 
   manage-pr-labels:
     name: Remove prebuilt label

.github/workflows/sunnypilot-master-dev-prep.yaml

@@ -1,9 +1,8 @@
-name: Build dev-c3-new
+name: Build dev
 
 env:
   DEFAULT_SOURCE_BRANCH: "master"
-  DEFAULT_TARGET_BRANCH: "master-dev-c3-new"
-  PR_LABEL: "dev-c3"
+  DEFAULT_TARGET_BRANCH: "master-dev"
   LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
   LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
 
@@ -25,7 +24,7 @@ on:
       target_branch:
         description: 'Target branch to reset and squash into'
         required: true
-        default: 'master-dev-c3-new'
+        default: 'master-dev'
         type: string
       cancel_in_progress:
         description: 'Cancel any in-progress runs of this workflow'
@@ -43,22 +42,23 @@ jobs:
     if: (
       (github.event_name == 'workflow_dispatch')
       || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-      || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
+      || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
       )
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Fetch all history for all branches
           token: ${{ secrets.GITHUB_TOKEN }}
+          persist-credentials: false
 
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
         if: (
           (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-          || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
+          || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
           )
         with:
-          workflow: selfdrive_tests.yaml # The workflow file to monitor
+          workflow: tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Configure Git
@@ -118,8 +118,8 @@ jobs:
         run: |
           # Use GitHub API to get PRs with specific label, ordered by creation date
           PR_LIST=$(gh api graphql -f query='
-            query($label:String!) {
-              search(query: $label, type:ISSUE, first:100) {
+            query($search_query:String!) {
+              search(query: $search_query, type:ISSUE, first:40) {
                 nodes {
                   ... on PullRequest {
                     number
@@ -149,7 +149,7 @@ jobs:
                   }
                 }
               }
-            }' -F label="is:pr is:open label:${PR_LABEL} draft:false sort:created-asc")
+            }' -F search_query="repo:${{ github.repository }} is:pr is:open label:${{ vars.PREBUILT_PR_LABEL }},${{ vars.PREBUILT_PR_LABEL }}-c3 draft:false sort:created-asc")
 
           PR_LIST=${PR_LIST//\'/}
           echo "PR_LIST=${PR_LIST}" >> $GITHUB_OUTPUT
@@ -174,10 +174,37 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Push changes if there are diffs
-        id: push-changes  # Add an id so we can reference this step
+      - name: Restore workflows from source
         run: |
           TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+          SOURCE_BRANCH="${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
+
+          # Ensure we are on the target branch
+          git checkout $TARGET_BRANCH
+
+          echo "Restoring .github/workflows from $SOURCE_BRANCH"
+          git checkout origin/$SOURCE_BRANCH -- .github/workflows
+
+          if ! git diff --cached --quiet; then
+            echo "Workflows differ. Committing restoration."
+            git commit -m "chore: restore .github/workflows from $SOURCE_BRANCH"
+          else
+            echo "Workflows match $SOURCE_BRANCH."
+          fi
+
+      - uses: actions/create-github-app-token@v2
+        id: ci-token
+        with:
+          app-id: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_ID }}
+          private-key: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_PRIVATE_KEY }}
+
+      - name: Push changes if there are diffs
+        id: push-changes
+        run: |
+          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+
+          # Use the App Token to set the remote URL with authentication
+          git remote set-url origin "https://x-access-token:${{ steps.ci-token.outputs.token }}@github.com/${{ github.repository }}.git"
 
           # Fetch the latest from remote
           git fetch origin $TARGET_BRANCH
@@ -189,7 +216,7 @@ jobs:
             exit 0
           fi
 
-          # If we get here, there are diffs, so push
+          # Push with the authenticated origin
           if ! git push origin $TARGET_BRANCH --force; then
             echo "Failed to push changes to $TARGET_BRANCH"
             exit 1
@@ -202,22 +229,15 @@ jobs:
         if: steps.push-changes.outputs.has_changes == 'true'
         run: |
           echo "Triggering selfdrive tests..."
-          gh workflow run selfdrive_tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+          gh workflow run tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
 
           echo "Sleeping for 120s to give plenty of time for the action to start and then we wait"
           sleep 120
 
           echo "Getting latest run ID..."
-          RUN_ID=$(gh run list --workflow=selfdrive_tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+          RUN_ID=$(gh run list --workflow=tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
 
           echo "Watching run ID: $RUN_ID"
           gh run watch "$RUN_ID"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Trigger prebuilt workflow
-        if: success() && steps.push-changes.outputs.has_changes == 'true'
-        run: |
-          gh workflow run sunnypilot-build-prebuilt.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-discourse.yaml.yml

@@ -0,0 +1,78 @@
+name: Debug Discourse Posting
+
+on:
+  push:
+
+jobs:
+  test-discourse-post:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Post test message to Discourse
+        uses: ./.github/workflows/post-to-discourse
+        with:
+          discourse-url: ${{ vars.DISCOURSE_URL }}
+          api-key: ${{ secrets.DISCOURSE_API_KEY }}
+          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
+          topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
+          message: |
+            ## 🧪 Test Post from GitHub Actions
+            
+            **This is a test post to verify Discourse integration**
+            
+            - **Workflow**: ${{ github.workflow }}
+            - **Run Number**: #${{ github.run_number }}
+            - **Branch**: `${{ github.ref_name }}`
+            - **Commit**: ${{ github.sha }}
+            - **Actor**: @${{ github.actor }}
+            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
+            
+            ---
+            
+            ### Fake Build Info (for testing)
+            - **Version**: 0.9.8-test
+            - **Build**: #42
+            - **Branch**: release-test
+            
+            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+            
+            *This is an automated test message. Drive safe! 🚗💨*
+      
+      
+      - name: Create topic on Discourse
+        uses: ./.github/workflows/post-to-discourse
+        with:
+          discourse-url: ${{ vars.DISCOURSE_URL }}
+          api-key: ${{ secrets.DISCOURSE_API_KEY }}
+          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
+          #topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
+          category-id: 4
+          title: "This is a test of a new topic instead of a reply"
+          message: |
+            ## 🧪 Test Post from GitHub Actions
+            
+            **This is a test post to verify Discourse integration**
+            
+            - **Workflow**: ${{ github.workflow }}
+            - **Run Number**: #${{ github.run_number }}
+            - **Branch**: `${{ github.ref_name }}`
+            - **Commit**: ${{ github.sha }}
+            - **Actor**: @${{ github.actor }}
+            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
+            
+            ---
+            
+            ### Fake Build Info (for testing)
+            - **Version**: 0.9.8-test
+            - **Build**: #42
+            - **Branch**: release-test
+            
+            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+            
+            *This is an automated test message. Drive safe! 🚗💨*
+      - name: Display results
+        if: always()
+        run: |
+          echo "::notice::Discourse post test completed"
+          echo "Check your Discourse topic to verify the post appeared correctly"

.github/workflows/tests.yaml

@@ -0,0 +1,254 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
+
+concurrency:
+  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  CI: 1
+  PYTHONPATH: ${{ github.workspace }}
+  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
+
+jobs:
+  build_release:
+    name: build release
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    env:
+      STRIPPED_DIR: /tmp/releasepilot
+      PYTHONPATH: /tmp/releasepilot
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - name: Getting LFS files
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: git lfs pull
+    - name: Build devel
+      timeout-minutes: 1
+      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
+    - run: ./tools/op.sh setup
+    - name: Build openpilot and run checks
+      timeout-minutes: 30
+      working-directory: ${{ env.STRIPPED_DIR }}
+      run: python3 system/manager/build.py
+    - name: Run tests
+      timeout-minutes: 1
+      working-directory: ${{ env.STRIPPED_DIR }}
+      run: release/check-dirty.sh
+    - name: Check submodules
+      if: github.repository == 'sunnypilot/sunnypilot'
+      timeout-minutes: 3
+      run: |
+        if [ "${{ github.ref }}" != "refs/heads/master" ]; then
+          git fetch  origin master:refs/remotes/origin/master
+
+          SUBMODULE_PATHS=$(git diff origin/master HEAD --name-only | grep -E '^[^/]+$' | while read path; do
+            if git ls-files --stage "$path" | grep -q "^160000"; then
+              echo "$path"
+            fi
+          done | tr '\n' ' ')
+
+          if [ -n "$SUBMODULE_PATHS" ]; then
+            echo "Changed submodule paths: $SUBMODULE_PATHS"
+            export SUBMODULE_PATHS="$SUBMODULE_PATHS"
+            export CHECK_PR_REFS=true
+          fi
+        fi
+        release/check-submodules.sh
+
+  build_mac:
+    name: build macOS
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - name: Remove Homebrew from environment
+      run: |
+        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
+        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
+    - run: ./tools/op.sh setup
+    - name: Building openpilot
+      run: scons
+
+  static_analysis:
+    name: static analysis
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Static analysis
+      timeout-minutes: 1
+      run: scripts/lint/lint.sh
+
+  unit_tests:
+    name: unit tests
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Build openpilot
+      run: scons
+    - name: Run unit tests
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
+      run: |
+        source selfdrive/test/setup_xvfb.sh
+        # Pre-compile Python bytecode so each pytest worker doesn't need to
+        $PYTEST --collect-only -m 'not slow' -qq
+        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
+
+  process_replay:
+    name: process replay
+    if: false  # disable process_replay for forks
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Build openpilot
+      run: scons
+    - name: Run replay
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
+      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
+      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
+    - name: Print diff
+      id: print-diff
+      if: always()
+      run: cat selfdrive/test/process_replay/diff.txt
+    - name: Print diff report
+      if: always()
+      run: cat selfdrive/test/process_replay/diff_report.txt
+    - uses: actions/upload-artifact@v6
+      if: always()
+      continue-on-error: true
+      with:
+        name: process_replay_diff.txt
+        path: selfdrive/test/process_replay/diff.txt
+    - name: Upload diff report
+      uses: actions/upload-artifact@v6
+      if: always() && github.event_name == 'pull_request'
+      continue-on-error: true
+      with:
+        name: diff_report_${{ github.event.number }}
+        path: selfdrive/test/process_replay/diff_report.txt
+    - name: Checkout ci-artifacts
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
+      uses: actions/checkout@v4
+      with:
+        repository: commaai/ci-artifacts
+        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+        path: ${{ github.workspace }}/ci-artifacts
+    - name: Prepare refs
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
+      working-directory: ${{ github.workspace }}/ci-artifacts
+      run: |
+        git config user.name "GitHub Actions Bot"
+        git config user.email "<>"
+        git fetch origin process-replay || true
+        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
+        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
+        echo "${{ github.sha }}" > ref_commit
+        git add .
+        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
+    - name: Push refs
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force
+    - name: Run regen
+      if: false
+      timeout-minutes: 4
+      env:
+        ONNXCPU: 1
+      run: $PYTEST selfdrive/test/process_replay/test_regen.py
+
+  simulator_driving:
+    name: simulator driving
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    if: false  # FIXME: Started to timeout recently
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Build openpilot
+      run: scons
+    - name: Driving test
+      timeout-minutes: 2
+      run: |
+        source selfdrive/test/setup_xvfb.sh
+        pytest -s tools/sim/tests/test_metadrive_bridge.py
+
+  create_ui_report:
+    name: Create UI Report
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
+      - run: ./tools/op.sh setup
+      - name: Build openpilot
+        run: scons
+      - name: Create UI Report
+        run: |
+          source selfdrive/test/setup_xvfb.sh
+          python3 selfdrive/ui/tests/diff/replay.py
+          python3 selfdrive/ui/tests/diff/replay.py --big
+      - name: Upload UI Report
+        uses: actions/upload-artifact@v6
+        with:
+          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -8,14 +8,20 @@ on:
     branches:
       - 'master'
     paths:
+      - 'selfdrive/assets/**'
       - 'selfdrive/ui/**'
+      - 'system/ui/**'
   workflow_dispatch:
 
 env:
   UI_JOB_NAME: "Create UI Report"
   REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
   SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
+  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
+
+  # variant:video_prefix:master_branch
+  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
 
 jobs:
   preview:
@@ -28,8 +34,9 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - name: Waiting for ui generation to start
-        run: sleep 30
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
 
       - name: Waiting for ui generation to end
         uses: lewagon/wait-on-check-action@v1.3.4
@@ -46,110 +53,93 @@ jobs:
           echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
       - name: Getting proposed ui
-        id: download-artifact
         uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: report-1-${{ env.REPORT_NAME }}
+          name: ui-report-1-${{ env.REPORT_NAME }}
           path: ${{ github.workspace }}/pr_ui
 
-      - name: Getting master ui
-        uses: actions/checkout@v4
+      - name: Getting mici master ui
+        uses: actions/checkout@v6
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_ui
-          ref: openpilot_master_ui
+          path: ${{ github.workspace }}/master_mici
+          ref: openpilot_master_ui_mici_raylib
+
+      - name: Getting big master ui
+        uses: actions/checkout@v6
+        with:
+          repository: sunnypilot/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_big
+          ref: openpilot_master_ui_big_raylib
 
       - name: Saving new master ui
         if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          git checkout --orphan=new_master_ui
-          git rm -rf *
-          git branch -D openpilot_master_ui
-          git branch -m openpilot_master_ui
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          mv ${{ github.workspace }}/pr_ui/*.png .
-          git add .
-          git commit -m "screenshots for commit ${{ env.SHA }}"
-          git push origin openpilot_master_ui --force
+          for variant in $VARIANTS; do
+            IFS=':' read -r name video branch <<< "$variant"
+            master_dir="${{ github.workspace }}/master_${name}"
+            cd "$master_dir"
+            git checkout --orphan=new_branch
+            git rm -rf *
+            git branch -D "$branch"
+            git branch -m "$branch"
+            git config user.name "GitHub Actions Bot"
+            git config user.email "<>"
+            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
+            git add .
+            git commit -m "${name} video for commit ${{ env.SHA }}"
+            git push origin "$branch" --force
+          done
 
-      - name: Finding diff
+      - name: Setup FFmpeg
+        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
+
+      - name: Finding diffs
         if: github.event_name == 'pull_request_target'
         id: find_diff
-        run: >-
-          sudo apt-get update && sudo apt-get install -y imagemagick
+        run: |
+          export PYTHONPATH=${{ github.workspace }}
+          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
 
-          scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device')
-          A=($scenes)
+          COMMENT=""
+          for variant in $VARIANTS; do
+            IFS=':' read -r name video _ <<< "$variant"
+            diff_name="${name}_diff"
 
-          DIFF=""
-          TABLE="<details><summary>All Screenshots</summary>"
-          TABLE="${TABLE}<table>"
+            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
+            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
 
-          for ((i=0; i<${#A[*]}; i=i+1));
-          do
-            # Check if the master file exists
-            if [ ! -f "${{ github.workspace }}/master_ui/${A[$i]}.png" ]; then
-              # This is a new file in PR UI that doesn't exist in master
-              DIFF="${DIFF}<details open>"
-              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
-              DIFF="${DIFF}<table>"
+            diff_exit_code=0
+            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
+              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
+              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
+              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
 
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-              DIFF="${DIFF}</tr>"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
 
-              DIFF="${DIFF}</table>"
-              DIFF="${DIFF}</details>"
-            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
-              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
-              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
-              convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
-
-              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
-
-              DIFF="${DIFF}<details open>"
-              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
-              DIFF="${DIFF}<table>"
-
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
-              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-              DIFF="${DIFF}</tr>"
-
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
-              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
-              DIFF="${DIFF}</tr>"
-
-              DIFF="${DIFF}</table>"
-              DIFF="${DIFF}</details>"
+            REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
+            if [ $diff_exit_code -eq 0 ]; then
+              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
             else
-              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
-            fi
-
-            INDEX=$(($i % 2))
-            if [[ $INDEX -eq 0 ]]; then
-              TABLE="${TABLE}<tr>"
-            fi
-            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
-              TABLE="${TABLE}</tr>"
+              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
             fi
           done
 
-          TABLE="${TABLE}</table></details>"
-
-          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
+          {
+            echo "COMMENT<<EOF"
+            echo "$COMMENT"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
 
       - name: Saving proposed ui
         if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_ui
+        working-directory: ${{ github.workspace }}/master_mici
         run: |
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
@@ -157,17 +147,29 @@ jobs:
           git rm -rf *
           mv ${{ github.workspace }}/pr_ui/* .
           git add .
-          git commit -m "screenshots for PR #${{ github.event.number }}"
+          git commit -m "ui videos for PR #${{ github.event.number }}"
           git push origin ${{ env.BRANCH_NAME }} --force
 
-      - name: Comment Screenshots on PR
+          # Append diff reports to report files branch
+          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
+          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
+          for variant in $VARIANTS; do
+            IFS=':' read -r name _ _ <<< "$variant"
+            diff_name="${name}_diff"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
+            git add "${diff_name}_pr_${{ github.event.number }}.html"
+          done
+          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
+          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
+
+      - name: Comment on PR
         if: github.event_name == 'pull_request_target'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
-            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
+            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
             ## UI Preview
-            ${{ steps.find_diff.outputs.DIFF }}
-          comment_tag: run_id_screenshots
+            ${{ steps.find_diff.outputs.COMMENT }}
+          comment_tag: run_id_ui_preview
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/wait-for-action/action.yaml

@@ -4,7 +4,7 @@ inputs:
   workflow:
     description: 'The workflow file name to monitor'
     required: true
-    default: 'selfdrive_tests.yaml'
+    default: 'tests.yaml'
   branch:
     description: 'The branch to monitor (defaults to current branch)'
     required: false

.gitignore

@@ -10,17 +10,16 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
-model2.png
 a.out
 .hypothesis
 .cache/
-
-/docs_site/
+bin/
 
 *.mp4
 *.dylib
 *.DSYM
 *.d
+*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -37,74 +36,60 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
-*.qm
+*.mo
 *_pyx.cpp
+*.stats
+*.pkl
+*.pkl*
 config.json
-clcache
 compile_commands.json
 compare_runtime*.html
+selfdrive/modeld/models/tg_input_devices.json
 
-persist
+# build artifacts
+docs_site/
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
 cereal/messaging/bridge
-selfdrive/mapd/default_speeds_by_region.json
-system/proclogd/proclogd
 selfdrive/ui/translations/tmp
-selfdrive/test/longitudinal_maneuvers/out
 selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 
-notebooks
-hyperthneed
-provisioning
-
 .coverage*
 coverage.xml
 htmlcov
-pandaextra
-
-.mypy_cache/
-flycheck_*
-
-cppcheck_report.txt
-comma*.sh
-
-selfdrive/modeld/models/*.pkl
-sunnypilot/modeld*/thneed/compile
-sunnypilot/modeld*/models/*.thneed
-sunnypilot/modeld*/models/*.pkl
 
+# openpilot log files
 *.bz2
 *.zst
+*.rlog
 
 build/
 
 !**/.gitkeep
 
-poetry.toml
-Pipfile
 
 ### VisualStudioCode ###
+*.vsix
+.history
+.ionide
 .vscode/*
+.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# Local History for Visual Studio Code
-.history/
-
-# Built Visual Studio Code Extensions
-*.vsix
-
-### VisualStudioCode Patch ###
-# Ignore all local history of files
-.history
-.ionide
+# agents
+.claude/
+.context/
+PLAN.md
+TASK.md
+CLAUDE.md
+SKILL.md
 
 ### JetBrains ###
 !.idea/customTargets.xml

.gitmodules

@@ -4,9 +4,10 @@
 [submodule "opendbc"]
   path = opendbc_repo
   url = https://github.com/sunnypilot/opendbc.git
+  branch = tn
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/sunnypilot/msgq.git
+  url = https://github.com/commaai/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git
@@ -15,7 +16,7 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/tinygrad/tinygrad.git
+  url = https://github.com/sunnypilot/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
   path = sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.python-version

@@ -0,0 +1 @@
+3.12.13

.run/Build_BIG_UI.run.xml

@@ -0,0 +1,26 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Build_BIG_UI" type="PythonConfigurationType" factoryName="Python">
+    <module name="sunnypilot" />
+    <option name="ENV_FILES" value="" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="BIG" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
+    <option name="PARAMETERS" value="" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2">
+      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
+    </method>
+  </configuration>
+</component>

.run/Build_SMALL_UI.run.xml

@@ -0,0 +1,23 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Build_SMALL_UI" type="PythonConfigurationType" factoryName="Python">
+    <module name="sunnypilot" />
+    <option name="ENV_FILES" value="" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
+    <option name="PARAMETERS" value="" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2">
+      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
+    </method>
+  </configuration>
+</component>

.vscode/launch.json

@@ -23,6 +23,11 @@
       "id": "args",
       "description": "Arguments to pass to the process",
       "type": "promptString"
+    },
+    {
+      "id": "replayArg",
+      "type": "promptString",
+      "description": "Enter route or segment to replay."
     }
   ],
   "configurations": [
@@ -40,7 +45,44 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Attach LLDB to Replay drive",
+      "type": "lldb",
+      "request": "attach",
+      "pid": "${command:pickMyProcess}",
+      "sourceMap": {
+        ".": "${workspaceFolder}/opendbc/safety"
+      },
+      "initCommands": [
+        "script import time; time.sleep(3)"
+      ]
+    },
+    {
+      "name": "Replay drive",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
+      "args": [
+        "${input:replayArg}"
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      },
+      "subProcess": true,
+      "stopOnEntry": false
+    }
+  ],
+  "compounds": [
+    {
+      "name": "Replay drive + Safety LLDB",
+      "configurations": [
+        "Replay drive",
+        "Attach LLDB to Replay drive"
+      ]
     }
   ]
 }

.vscode/settings.json

@@ -21,7 +21,6 @@
     "common/**",
     "selfdrive/**",
     "system/**",
-    "third_party/**",
     "tools/**",
   ]
 }

Dockerfile.openpilot

@@ -1,12 +1,38 @@
-FROM ghcr.io/commaai/openpilot-base:latest
+FROM ubuntu:24.04
 
 ENV PYTHONUNBUFFERED=1
 
-ENV OPENPILOT_PATH=/home/batman/openpilot
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales && \
+    rm -rf /var/lib/apt/lists/*
 
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+ENV OPENPILOT_PATH=/home/$USER/openpilot
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY . ${OPENPILOT_PATH}/
+COPY --chown=$USER . ${OPENPILOT_PATH}/
 
-RUN scons --cache-readonly -j$(nproc)
+ENV UV_BIN="/home/$USER/.local/bin/"
+ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
+ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
+RUN tools/setup_dependencies.sh && \
+    sudo rm -rf /var/lib/apt/lists/*
+
+USER root
+RUN git config --global --add safe.directory '*'

Dockerfile.openpilot_base

@@ -1,82 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/* && \
-    cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
-
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX=1
-
-RUN dbus-uuidgen > /etc/machine-id
-RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /home/$USER
-COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /home/$USER && \
-    tools/install_python_dependencies.sh && \
-    rm -rf tools/ pyproject.toml uv.lock .cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot

Dockerfile.sunnypilot

@@ -1,12 +0,0 @@
-FROM ghcr.io/sunnypilot/sunnypilot-base:latest
-
-ENV PYTHONUNBUFFERED=1
-
-ENV OPENPILOT_PATH=/home/batman/openpilot
-
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
-
-COPY . ${OPENPILOT_PATH}/
-
-RUN scons --cache-readonly -j$(nproc)

Dockerfile.sunnypilot_base

@@ -1,83 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/* && \
-    cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
-
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-    rsync \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX=1
-
-RUN dbus-uuidgen > /etc/machine-id
-RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /home/$USER
-COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /home/$USER && \
-    tools/install_python_dependencies.sh && \
-    rm -rf tools/ pyproject.toml uv.lock .cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -22,7 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-export LOGPRINT=debug
+#export LOGPRINT=debug # this has gotten too spammy...
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -166,8 +166,8 @@ node {
   env.GIT_BRANCH = checkout(scm).GIT_BRANCH
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
-  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tici', 'testing-closet*', 'hotfix-*']
+  def excludeBranches = ['__nightly', 'devel', 'devel-staging',
+                         'release-tizi', 'release-tizi-staging', 'release-mici', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
   if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
@@ -178,20 +178,20 @@ node {
 
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release3-staging", "tici-needs-can", [], [
-        step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"),
+      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
     if (env.BRANCH_NAME == '__nightly') {
       parallel (
         'nightly': {
-          deviceStage("build nightly", "tici-needs-can", [], [
+          deviceStage("build nightly", "tizi-needs-can", [], [
             step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
-          deviceStage("build nightly-dev", "tici-needs-can", [], [
+          deviceStage("build nightly-dev", "tizi-needs-can", [], [
             step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
@@ -200,63 +200,43 @@ node {
 
     if (!env.BRANCH_NAME.matches(excludeRegex)) {
     parallel (
-      // tici tests
       'onroad tests': {
-        deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [
+        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
           step("check dirty", "release/check-dirty.sh"),
           step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
-        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
+        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
           step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
-          step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py", [diffPaths: ["system/ubloxd/"]]),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
-      'loopback': {
-        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
-          step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
-        ])
-      },
-      'camerad AR0231': {
-        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
-        ])
-      },
       'camerad OX03C10': {
-        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
+        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'sensord': {
-        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
-        ])
-        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
+        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
-        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
+        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
           step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
         ])
@@ -264,13 +244,9 @@ node {
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
-          // TODO: enable once new AGNOS is available
-          // step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),
-          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
         ])
       },
 

README.md

@@ -3,63 +3,18 @@
 ## 🌞 What is sunnypilot?
 [sunnypilot](https://github.com/sunnyhaibin/sunnypilot) is a fork of comma.ai's openpilot, an open source driver assistance system. sunnypilot offers the user a unique driving experience for over 300+ supported car makes and models with modified behaviors of driving assist engagements. sunnypilot complies with comma.ai's safety rules as accurately as possible.
 
-## 💭 Join our Discord
-Join the official sunnypilot Discord server to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
-* https://discord.gg/sunnypilot
-
-  ![](https://dcbadge.vercel.app/api/server/wRW3meAgtx?style=flat) ![Discord Shield](https://discordapp.com/api/guilds/880416502577266699/widget.png?style=shield)
+## 💭 Join our Community Forum
+Join the official sunnypilot community forum to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
+* https://community.sunnypilot.ai/
 
 ## Documentation
 https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
 
 ## 🚘 Running on a dedicated device in a car
-* A supported device to run this software
-    * a [comma three](https://comma.ai/shop/products/three) or a [C3X](https://comma.ai/shop/comma-3x)
-* This software
-* One of [the 300+ supported cars](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
-* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
-
-Detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
+First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
 
 ## Installation
-Please refer to [Recommended Branches](#-recommended-branches) to find your preferred/supported branch. This guide will assume you want to install the latest `release-c3` branch.
-
-* sunnypilot not installed or you installed a version before 0.8.17?
-  1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
-  2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
-  3. Input the installation URL per [Recommended Branches](#-recommended-branches). Example: ```release-c3.sunnypilot.ai```.
-  4. Complete the rest of the installation following the onscreen instructions.
-
-* sunnypilot already installed and you installed a version after 0.8.17?
-  1. On the comma three, go to `Settings` ▶️ `Software`.
-  2. At the `Download` option, press `CHECK`. This will fetch the list of latest branches from sunnypilot.
-  3. At the `Target Branch` option, press `SELECT` to open the Target Branch selector.
-  4. Scroll to select the desired branch per  Recommended Branches (see below). Example: `release-c3`
-
-|    Branch    |         Installation URL         |
-|:------------:|:--------------------------------:|
-| `release-c3` | https://release-c3.sunnypilot.ai |
-| `staging-c3` | https://staging-c3.sunnypilot.ai |
-|   `dev-c3`   | https://dev-c3.sunnypilot.ai     |
-
-### If you want to use our newest branches (our rewrite)
-> [!TIP]
->You can see the rewrite state on our [rewrite project board](https://github.com/orgs/sunnypilot/projects/2), and to install the new branches, you can use the following links
-
-
-> [!IMPORTANT]
-> It is recommended to [re-flash AGNOS](https://flash.comma.ai/) if you intend to downgrade from the new branches.
-> You can still restore the latest sunnylink backup made on the old branches.
-
-|      Branch      |                 Installation URL              |
-|:----------------:|:---------------------------------------------:|
-| `staging-c3-new` | `https://staging-c3-new.sunnypilot.ai`        |
-|   `dev-c3-new`   | `https://dev-c3-new.sunnypilot.ai`            |
-| `custom-branch`  | `https://install.sunnypilot.ai/{branch_name}` |
-| `release-c3-new` |            **Not yet available**.             |
-
-> [!TIP]
-> Do you require further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
+Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
 
 ## 🎆 Pull Requests
 We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.

RELEASES.md

@@ -1,7 +1,51 @@
+Version 0.11.2 (2026-06-15)
+========================
+* Acceleration Profile: select Eco, Normal, or Sport personality to shape acceleration briskness and gentle (ACC) deceleration
+
+Version 0.11.1 (2026-05-18)
+========================
+* New driver monitoring model
+* Improved image processing pipeline for driver camera
+* Improved thermal policy for comma four
+* Acura MDX 2022-24 support thanks to mvl-boston!
+* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
+
+Version 0.11.0 (2026-03-17)
+========================
+* New driving model #36798
+  * Fully trained using a learned simulator
+  * Improved longitudinal performance in Experimental mode
+* Reduce comma four standby power usage by 77% to 52 mW
+* Kia K7 2017 support thanks to royjr!
+* Lexus LS 2018 support thanks to Hacheoy!
+
+Version 0.10.3 (2025-12-17)
+========================
+* New driving model #36249
+  * New temporal policy architecture
+  * New on-policy training physics noise model
+* New driver monitoring model #36409
+  * Trained on a new dataset, including comma four data
+* Improved inter-process communication memory efficiency
+
+Version 0.10.2 (2025-11-19)
+========================
+* comma four support
+
 Version 0.10.1 (2025-09-08)
 ========================
-* Record driving feedback using LKAS button
-* Honda City 2023 support thanks to drFritz!
+* New driving model #36276
+  * World Model: removed global localization inputs
+  * World Model: 2x the number of parameters
+  * World Model: trained on 4x the number of segments
+  * VAE Compression Model: new architecture and training objective
+  * Driving Vision Model: trained on 4x the number of segments
+* New Driver Monitoring model #36198
+* Acura TLX 2021 support thanks to MVL!
+* Honda City 2023 support thanks to vanillagorillaa and drFritz!
+* Honda N-Box 2018 support thanks to miettal!
+* Honda Odyssey 2021-25 support thanks to csouers and MVL!
+* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
 
 Version 0.10.0 (2025-08-05)
 ========================

SConstruct

@@ -3,343 +3,224 @@ import subprocess
 import sys
 import sysconfig
 import platform
+import shlex
+import importlib
 import numpy as np
 
 import SCons.Errors
+from SCons.Defaults import _stripixes
+
+TICI = os.path.isfile('/TICI')
 
 SCons.Warnings.warningAsException(True)
 
-# pending upstream fix - https://github.com/SCons/scons/issues/4461
-#SetOption('warn', 'all')
-
-TICI = os.path.isfile('/TICI')
-AGNOS = TICI
-
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
-
-AddOption('--kaitai',
-          action='store_true',
-          help='Regenerate kaitai struct parsers')
-
-AddOption('--asan',
-          action='store_true',
-          help='turn on ASAN')
-
-AddOption('--ubsan',
-          action='store_true',
-          help='turn on UBSan')
-
-AddOption('--coverage',
-          action='store_true',
-          help='build with test coverage options')
-
-AddOption('--clazy',
-          action='store_true',
-          help='build with clazy')
-
-AddOption('--ccflags',
-          action='store',
-          type='string',
-          default='',
-          help='pass arbitrary flags over the command line')
-
-AddOption('--external-sconscript',
-          action='store',
-          metavar='FILE',
-          dest='external_sconscript',
-          help='add an external SConscript to the build')
-
-AddOption('--mutation',
-          action='store_true',
-          help='generate mutation-ready code')
+SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2))))
 
+AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
+AddOption('--verbose', action='store_true', default=False, help='show full build commands')
+release = not os.path.exists(File('#.gitattributes').abspath) # file absent on release branch, see release_files.py
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
+          default=(not TICI and not release),
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-AddOption('--stock-ui',
-          action='store_true',
-          dest='stock_ui',
-          default=False,
-          help='Build stock openpilot UI instead of sunnypilot UI')
-
-## Architecture name breakdown (arch)
-## - larch64: linux tici aarch64
-## - aarch64: linux pc aarch64
-## - x86_64:  linux pc x64
-## - Darwin:  mac x64 or arm64
-real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+# Detect platform
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-elif arch == "aarch64" and AGNOS:
+elif arch == "aarch64" and TICI:
   arch = "larch64"
-assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
+assert arch in [
+  "larch64",  # linux tici arm64
+  "aarch64",  # linux pc arm64
+  "x86_64",   # linux pc x64
+  "Darwin",   # macOS arm64 (x86 not supported)
+]
 
-lenv = {
-  "PATH": os.environ['PATH'],
-  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+pkg_names = ['acados', 'bzip2', 'capnproto', 'catch2', 'eigen', 'ffmpeg', 'json11', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
+pkgs = [importlib.import_module(name) for name in pkg_names]
+acados = pkgs[pkg_names.index('acados')]
+acados_include_dirs = [
+  acados.INCLUDE_DIR,
+  os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
+  os.path.join(acados.INCLUDE_DIR, "hpipm", "include"),
+]
 
-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+
+# ***** enforce a whitelist of system libraries *****
+# this prevents silently relying on a 3rd party package,
+# e.g. apt-installed libusb. all libraries should either
+# be distributed with all Linux distros and macOS, or
+# vendored in commaai/dependencies.
+allowed_system_libs = {
+  "EGL", "GLESv2", "GL",
+  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
+  "dl", "drm", "gbm", "m", "pthread",
 }
 
-rpath = []
+def _resolve_lib(env, name):
+  for d in env.Flatten(env.get('LIBPATH', [])):
+    p = Dir(str(d)).abspath
+    for ext in ('.a', '.so', '.dylib'):
+      f = File(os.path.join(p, f'lib{name}{ext}'))
+      if f.exists() or f.has_builder():
+        return name
+  if name in allowed_system_libs:
+    return name
+  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
 
-if arch == "larch64":
-  cpppath = [
-    "#third_party/opencl/include",
-  ]
-
-  libpath = [
-    "/usr/local/lib",
-    "/system/vendor/lib64",
-    f"#third_party/acados/{arch}/lib",
-  ]
-
-  libpath += [
-    "#third_party/snpe/larch64",
-    "#third_party/libyuv/larch64/lib",
-    "/usr/lib/aarch64-linux-gnu"
-  ]
-  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  rpath += ["/usr/local/lib"]
-else:
-  cflags = []
-  cxxflags = []
-  cpppath = []
-  rpath += []
-
-  # MacOS
-  if arch == "Darwin":
-    libpath = [
-      f"#third_party/libyuv/{arch}/lib",
-      f"#third_party/acados/{arch}/lib",
-      f"{brew_prefix}/lib",
-      f"{brew_prefix}/opt/openssl@3.0/lib",
-      "/System/Library/Frameworks/OpenGL.framework/Libraries",
-    ]
-
-    cflags += ["-DGL_SILENCE_DEPRECATION"]
-    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
-    cpppath += [
-      f"{brew_prefix}/include",
-      f"{brew_prefix}/opt/openssl@3.0/include",
-    ]
-  # Linux
-  else:
-    libpath = [
-      f"#third_party/acados/{arch}/lib",
-      f"#third_party/libyuv/{arch}/lib",
-      "/usr/lib",
-      "/usr/local/lib",
-    ]
-
-    if arch == "x86_64":
-      libpath += [
-        f"#third_party/snpe/{arch}"
-      ]
-      rpath += [
-        Dir(f"#third_party/snpe/{arch}").abspath,
-      ]
-
-if GetOption('asan'):
-  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
-  ldflags = ["-fsanitize=address"]
-elif GetOption('ubsan'):
-  ccflags = ["-fsanitize=undefined"]
-  ldflags = ["-fsanitize=undefined"]
-else:
-  ccflags = []
-  ldflags = []
-
-# no --as-needed on mac linker
-if arch != "Darwin":
-  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
-
-if not GetOption('stock_ui'):
-  cflags += ["-DSUNNYPILOT"]
-  cxxflags += ["-DSUNNYPILOT"]
-
-ccflags_option = GetOption('ccflags')
-if ccflags_option:
-  ccflags += ccflags_option.split(' ')
+def _libflags(target, source, env, for_signature):
+  libs = []
+  lp = env.subst('$LIBLITERALPREFIX')
+  for lib in env.Flatten(env.get('LIBS', [])):
+    if isinstance(lib, str):
+      if os.sep in lib or lib.startswith('#'):
+        libs.append(File(lib))
+      elif lib.startswith('-') or (lp and lib.startswith(lp)):
+        libs.append(lib)
+      else:
+        libs.append(_resolve_lib(env, lib))
+    else:
+      libs.append(lib)
+  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
+                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
 
 env = Environment(
-  ENV=lenv,
+  ENV={
+    "PATH": os.environ['PATH'],
+    "PYTHONPATH": Dir("#").abspath,
+    "ACADOS_SOURCE_DIR": acados.DIR,
+    "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
+    "TERA_PATH": acados.TERA_PATH
+  },
   CCFLAGS=[
     "-g",
     "-fPIC",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow",
+    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
     "-Wno-vla-cxx-extension",
-  ] + cflags + ccflags,
-
-  CPPPATH=cpppath + [
+  ],
+  CFLAGS=["-std=gnu11"],
+  CXXFLAGS=["-std=c++1z"],
+  CPPPATH=[
     "#",
-    "#third_party/acados/include",
-    "#third_party/acados/include/blasfeo/include",
-    "#third_party/acados/include/hpipm/include",
-    "#third_party/catch2/include",
-    "#third_party/libyuv/include",
-    "#third_party/json11",
-    "#third_party/linux/include",
-    "#third_party/snpe/include",
-    "#third_party",
     "#msgq",
+    acados_include_dirs,
+    [x.INCLUDE_DIR for x in pkgs],
   ],
-
-  CC='clang',
-  CXX='clang++',
-  LINKFLAGS=ldflags,
-
-  RPATH=rpath,
-
-  CFLAGS=["-std=gnu11"] + cflags,
-  CXXFLAGS=["-std=c++1z"] + cxxflags,
-  LIBPATH=libpath + [
-    "#msgq_repo",
-    "#third_party",
-    "#selfdrive/pandad",
+  LIBPATH=[
     "#common",
+    "#msgq_repo",
+    "#selfdrive/pandad",
     "#rednose/helpers",
+    [x.LIB_DIR for x in pkgs],
   ],
+  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
+if arch != "larch64":
+  env['_LIBFLAGS'] = _libflags
 
-if arch == "Darwin":
-  # RPATH is not supported on macOS, instead use the linker flags
-  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
-  env["LINKFLAGS"] += darwin_rpath_link_flags
+# Arch-specific flags and paths
+if arch == "larch64":
+  env["CC"] = "clang"
+  env["CXX"] = "clang++"
+  env.Append(LIBPATH=[
+    "/usr/lib/aarch64-linux-gnu",
+  ])
+  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
+  env.Append(CCFLAGS=arch_flags)
+  env.Append(CXXFLAGS=arch_flags)
+elif arch == "Darwin":
+  env.Append(LIBPATH=[
+    "/System/Library/Frameworks/OpenGL.framework/Libraries",
+  ])
+  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
 
-env.CompilationDatabase('compile_commands.json')
+_extra_cc = shlex.split(GetOption('ccflags') or '')
+if _extra_cc:
+  env.Append(CCFLAGS=_extra_cc)
 
-# Setup cache dir
-default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
-cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+# no --as-needed on mac linker
+if arch != "Darwin":
+  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
 
-node_interval = 5
-node_count = 0
-def progress_function(node):
-  global node_count
-  node_count += node_interval
-  sys.stderr.write("progress: %d\n" % node_count)
+# Shorter build output: show brief descriptions instead of full commands.
+# Full command lines are still printed on failure by scons.
+if not GetOption('verbose'):
+  for action, short in (
+    ("CC",     "CC"),
+    ("CXX",    "CXX"),
+    ("LINK",   "LINK"),
+    ("SHCC",   "CC"),
+    ("SHCXX",  "CXX"),
+    ("SHLINK", "LINK"),
+    ("AR",     "AR"),
+    ("RANLIB", "RANLIB"),
+    ("AS",     "AS"),
+  ):
+    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
 
-if os.environ.get('SCONS_PROGRESS'):
-  Progress(progress_function, interval=node_interval)
-
-# Cython build environment
-py_include = sysconfig.get_paths()['include']
+# ********** Cython build environment **********
 envCython = env.Clone()
-envCython["CPPPATH"] += [py_include, np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
+  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-# Qt build environment
-qt_env = env.Clone()
-qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
+Export('env', 'arch', 'acados', 'release')
 
-qt_libs = []
-if arch == "Darwin":
-  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
-  qt_dirs = [
-    os.path.join(qt_env['QTDIR'], "include"),
-  ]
-  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
-  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
-  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
-  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
-else:
-  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
-  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
+# Setup cache dir
+default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
+cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
+cache_size_limit = 4e9 if "CI" in os.environ else 2e9
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
 
-  qt_env['QTDIR'] = qt_install_prefix
-  qt_dirs = [
-    f"{qt_install_headers}",
-  ]
+def prune_cache_dir(target=None, source=None, env=None):
+  cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime)
+  cache_size = sum(os.path.getsize(f) for f in cache_files)
+  for f in cache_files:
+    if cache_size < cache_size_limit:
+      break
+    cache_size -= os.path.getsize(f)
+    os.unlink(f)
 
-  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
-  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
-  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
-  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
-
-  qt_libs = [f"Qt5{m}" for m in qt_modules]
-  if arch == "larch64":
-    qt_libs += ["GLESv2", "wayland-client"]
-    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
-  elif arch != "Darwin":
-    qt_libs += ["GL"]
-qt_env['QT3DIR'] = qt_env['QTDIR']
-qt_env.Tool('qt3')
-
-qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
-qt_flags = [
-  "-D_REENTRANT",
-  "-DQT_NO_DEBUG",
-  "-DQT_WIDGETS_LIB",
-  "-DQT_GUI_LIB",
-  "-DQT_CORE_LIB",
-  "-DQT_MESSAGELOGCONTEXT",
-]
-qt_env['CXXFLAGS'] += qt_flags
-qt_env['LIBPATH'] += ['#selfdrive/ui', ]
-qt_env['LIBS'] = qt_libs
-
-if GetOption("clazy"):
-  checks = [
-    "level0",
-    "level1",
-    "no-range-loop",
-    "no-non-pod-global-static",
-  ]
-  qt_env['CXX'] = 'clazy'
-  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
-  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
-
-Export('env', 'qt_env', 'arch', 'real_arch')
+# ********** start building stuff **********
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common', '_gpucommon')
-
+Import('_common')
 common = [_common, 'json11', 'zmq']
-gpucommon = [_gpucommon]
-
-Export('common', 'gpucommon')
+Export('common')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
 # Enable swaglog include in submodules
 env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
-SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
 
 SConscript(['cereal/SConscript'])
 
@@ -356,30 +237,65 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
-  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
-if arch != "Darwin":
-  SConscript([
-    'system/logcatd/SConscript',
-    'system/proclogd/SConscript',
-  ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
 
-# Build openpilot
-SConscript(['third_party/SConscript'])
-
-SConscript(['selfdrive/SConscript'])
+# Build selfdrive
+SConscript([
+  'selfdrive/pandad/SConscript',
+  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
+  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
+  'selfdrive/locationd/SConscript',
+  'selfdrive/modeld/SConscript',
+  'selfdrive/ui/SConscript',
+])
 
 SConscript(['sunnypilot/SConscript'])
 
-if Dir('#tools/cabana/').exists() and GetOption('extras'):
-  SConscript(['tools/replay/SConscript'])
-  if arch != "larch64":
-    SConscript(['tools/cabana/SConscript'])
+# Build desktop-only tools
+if GetOption('extras') and arch != "larch64":
+  SConscript([
+    'tools/replay/SConscript',
+    'tools/cabana/SConscript',
+    'tools/jotpluggler/SConscript',
+  ])
 
-external_sconscript = GetOption('external_sconscript')
-if external_sconscript:
-  SConscript([external_sconscript])
+
+env.CompilationDatabase('compile_commands.json')
+
+# progress output
+def count_scons_nodes(nodes):
+  seen = set()
+  stack = list(nodes)
+
+  while stack:
+    node = stack.pop().disambiguate()
+    if node in seen:
+      continue
+    seen.add(node)
+    executor = node.get_executor()
+    if executor is not None:
+      stack += executor.get_all_prerequisites() + executor.get_all_children()
+
+  return len(seen)
+
+progress_interval = 5
+progress_count = 0
+progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
+
+def progress_function(node):
+  global progress_count
+  if progress_count >= progress_total:
+    return
+  progress_count = min(progress_count + progress_interval, progress_total)
+  progress = round(100. * progress_count / progress_total, 1)
+  sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
+  if progress == 100. and sys.stderr.isatty():
+    sys.stderr.write("\n")
+  sys.stderr.flush()
+
+Progress(progress_function, interval=progress_interval)
+AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)

cereal/SConscript

@@ -4,7 +4,7 @@ cereal_dir = Dir('.')
 gen_dir = Dir('gen')
 
 # Build cereal
-schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
+schema_files = ['log.capnp', 'car.capnp', 'deprecated.capnp', 'custom.capnp']
 env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
             schema_files,
             f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
@@ -13,7 +13,7 @@ cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
 socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 

cereal/custom.capnp

@@ -25,6 +25,26 @@ struct ModularAssistiveDrivingSystem {
   }
 }
 
+struct IntelligentCruiseButtonManagement {
+  state @0 :IntelligentCruiseButtonManagementState;
+  sendButton @1 :SendButtonState;
+  vTarget @2 :Float32;
+
+  enum IntelligentCruiseButtonManagementState {
+    inactive @0;      # No button press or default state
+    preActive @1;     # Pre-active state before transitioning to increasing or decreasing
+    increasing @2;    # Increasing speed
+    decreasing @3;    # Decreasing speed
+    holding @4;       # Holding steady speed
+  }
+
+  enum SendButtonState {
+    none @0;
+    increase @1;
+    decrease @2;
+  }
+}
+
 # Same struct as Log.RadarState.LeadData
 struct LeadData {
   dRel @0 :Float32;
@@ -48,6 +68,49 @@ struct LeadData {
 
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
   mads @0 :ModularAssistiveDrivingSystem;
+  intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
+
+  enum AudibleAlert {
+    none @0;
+
+    engage @1;
+    disengage @2;
+    refuse @3;
+
+    warningSoft @4;
+    warningImmediate @5;
+
+    prompt @6;
+    promptRepeat @7;
+    promptDistracted @8;
+
+    # unused, these are reserved for upstream events so we don't collide
+    reserved9 @9;
+    reserved10 @10;
+    reserved11 @11;
+    reserved12 @12;
+    reserved13 @13;
+    reserved14 @14;
+    reserved15 @15;
+    reserved16 @16;
+    reserved17 @17;
+    reserved18 @18;
+    reserved19 @19;
+    reserved20 @20;
+    reserved21 @21;
+    reserved22 @22;
+    reserved23 @23;
+    reserved24 @24;
+    reserved25 @25;
+    reserved26 @26;
+    reserved27 @27;
+    reserved28 @28;
+    reserved29 @29;
+    reserved30 @30;
+
+    promptSingleLow @31;
+    promptSingleHigh @32;
+  }
 }
 
 struct ModelManagerSP @0xaedffd8f31e7b55d {
@@ -90,6 +153,8 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
       navigation @1;
       vision @2;
       policy @3;
+      offPolicy @4;
+      onPolicy @5;
     }
   }
 
@@ -122,6 +187,14 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
 
 struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
   dec @0 :DynamicExperimentalControl;
+  longitudinalPlanSource @1 :LongitudinalPlanSource;
+  smartCruiseControl @2 :SmartCruiseControl;
+  speedLimit @3 :SpeedLimit;
+  vTarget @4 :Float32;
+  aTarget @5 :Float32;
+  events @6 :List(OnroadEventSP.Event);
+  e2eAlerts @7 :E2eAlerts;
+  acceleration @8 :Acceleration;
 
   struct DynamicExperimentalControl {
     state @0 :DynamicExperimentalControlState;
@@ -133,6 +206,111 @@ struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
       blended @1;
     }
   }
+
+  struct SmartCruiseControl {
+    vision @0 :Vision;
+    map @1 :Map;
+
+    struct Vision {
+      state @0 :VisionState;
+      vTarget @1 :Float32;
+      aTarget @2 :Float32;
+      currentLateralAccel @3 :Float32;
+      maxPredictedLateralAccel @4 :Float32;
+      enabled @5 :Bool;
+      active @6 :Bool;
+    }
+
+    struct Map {
+      state @0 :MapState;
+      vTarget @1 :Float32;
+      aTarget @2 :Float32;
+      enabled @3 :Bool;
+      active @4 :Bool;
+    }
+
+    enum VisionState {
+      disabled @0; # System disabled or inactive.
+      enabled @1; # No predicted substantial turn on vision range.
+      entering @2; # A substantial turn is predicted ahead, adapting speed to turn comfort levels.
+      turning @3; # Actively turning. Managing acceleration to provide a roll on turn feeling.
+      leaving @4; # Road ahead straightens. Start to allow positive acceleration.
+      overriding @5; # System overriding with manual control.
+    }
+
+    enum MapState {
+      disabled @0; # System disabled or inactive.
+      enabled @1; # No predicted substantial turn on map range.
+      turning @2; # Actively turning. Managing acceleration to provide a roll on turn feeling.
+      overriding @3; # System overriding with manual control.
+    }
+  }
+
+  struct SpeedLimit {
+    resolver @0 :Resolver;
+    assist @1 :Assist;
+
+    struct Resolver {
+      speedLimit @0 :Float32;
+      distToSpeedLimit @1 :Float32;
+      source @2 :Source;
+      speedLimitOffset @3 :Float32;
+      speedLimitLast @4 :Float32;
+      speedLimitFinal @5 :Float32;
+      speedLimitFinalLast @6 :Float32;
+      speedLimitValid @7 :Bool;
+      speedLimitLastValid @8 :Bool;
+    }
+
+    struct Assist {
+      state @0 :AssistState;
+      enabled @1 :Bool;
+      active @2 :Bool;
+      vTarget @3 :Float32;
+      aTarget @4 :Float32;
+    }
+
+    enum Source {
+      none @0;
+      car @1;
+      map @2;
+    }
+
+    enum AssistState {
+      disabled @0;
+      inactive @1; # No speed limit set or not enabled by parameter.
+      preActive @2;
+      pending @3; # Awaiting new speed limit.
+      adapting @4; # Reducing speed to match new speed limit.
+      active @5; # Cruising at speed limit.
+    }
+  }
+
+  enum LongitudinalPlanSource {
+    cruise @0;
+    sccVision @1;
+    sccMap @2;
+    speedLimitAssist @3;
+  }
+
+  struct E2eAlerts {
+    greenLightAlert @0 :Bool;
+    leadDepartAlert @1 :Bool;
+  }
+
+  struct Acceleration {
+    personality @0 :AccelerationPersonality;
+    enabled @1 :Bool;
+    maxAccel @2 :Float32;
+    tFollowOffset @3 :Float32;
+    jerkFactorScale @4 :Float32;
+  }
+
+  enum AccelerationPersonality {
+    eco @0;
+    normal @1;
+    sport @2;
+  }
 }
 
 struct OnroadEventSP @0xda96579883444c35 {
@@ -172,12 +350,23 @@ struct OnroadEventSP @0xda96579883444c35 {
     experimentalModeSwitched @14;
     wrongCarModeAlertOnly @15;
     pedalPressedAlertOnly @16;
+    laneTurnLeft @17;
+    laneTurnRight @18;
+    speedLimitPreActive @19;
+    speedLimitActive @20;
+    speedLimitChanged @21;
+    speedLimitPending @22;
+    e2eChime @23;
+    laneChangeRoadEdge @24;
   }
 }
 
 struct CarParamsSP @0x80ae746ee2596b11 {
   flags @0 :UInt32;        # flags for car specific quirks in sunnypilot
   safetyParam @1 : Int16;  # flags for sunnypilot's custom safety flags
+  pcmCruiseSpeed @3 :Bool;
+  intelligentCruiseButtonManagementAvailable @4 :Bool;
+  enableGasInterceptor @5 :Bool;
 
   neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
 
@@ -197,10 +386,24 @@ struct CarControlSP @0xa5cd762cd951a455 {
   params @1 :List(Param);
   leadOne @2 :LeadData;
   leadTwo @3 :LeadData;
+  intelligentCruiseButtonManagement @4 :IntelligentCruiseButtonManagement;
 
   struct Param {
     key @0 :Text;
-    value @1 :Text;
+    type @2 :ParamType;
+    value @3 :Data;
+
+    valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
+  }
+
+  enum ParamType {
+    string @0;
+    bool @1;
+    int @2;
+    float @3;
+    time @4;
+    json @5;
+    bytes @6;
   }
 }
 
@@ -247,6 +450,7 @@ struct BackupManagerSP @0xf98d843bfd7004a3 {
 }
 
 struct CarStateSP @0xb86e6369214c01c8 {
+  speedLimit @0 :Float32;
 }
 
 struct LiveMapDataSP @0xf416ec09499d9d19 {
@@ -258,7 +462,16 @@ struct LiveMapDataSP @0xf416ec09499d9d19 {
   roadName @5 :Text;
 }
 
-struct CustomReserved9 @0xa1680744031fdb2d {
+struct ModelDataV2SP @0xa1680744031fdb2d {
+  laneTurnDirection @0 :TurnDirection;
+  leftLaneChangeEdgeBlock @1 :Bool;
+  rightLaneChangeEdgeBlock @2 :Bool;
+
+  enum TurnDirection {
+    none @0;
+    turnLeft @1;
+    turnRight @2;
+  }
 }
 
 struct CustomReserved10 @0xcb9fd56c7057593a {

cereal/deprecated.capnp

@@ -3,7 +3,7 @@ $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
 
-# legacy.capnp: a home for deprecated structs
+# deprecated.capnp: a home for deprecated structs
 
 struct LogRotate @0x9811e1f38f62f2d1 {
   segmentNum @0 :Int32;
@@ -571,4 +571,219 @@ struct LidarPts @0xe3d6685d4e9d8f7a {
   pkt @4 :Data;
 }
 
+struct LiveTracksDEPRECATED @0xb16f60103159415a {
+  trackId @0 :Int32;
+  dRel @1 :Float32;
+  yRel @2 :Float32;
+  vRel @3 :Float32;
+  aRel @4 :Float32;
+  timeStamp @5 :Float32;
+  status @6 :Float32;
+  currentTime @7 :Float32;
+  stationary @8 :Bool;
+  oncoming @9 :Bool;
+}
 
+struct LiveMpcData @0x92a5e332a85f32a0 {
+  x @0 :List(Float32);
+  y @1 :List(Float32);
+  psi @2 :List(Float32);
+  curvature @3 :List(Float32);
+  qpIterations @4 :UInt32;
+  calculationTime @5 :UInt64;
+  cost @6 :Float64;
+}
+
+struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
+  xEgo @0 :List(Float32);
+  vEgo @1 :List(Float32);
+  aEgo @2 :List(Float32);
+  xLead @3 :List(Float32);
+  vLead @4 :List(Float32);
+  aLead @5 :List(Float32);
+  aLeadTau @6 :Float32;    # lead accel time constant
+  qpIterations @7 :UInt32;
+  mpcId @8 :UInt32;
+  calculationTime @9 :UInt64;
+  cost @10 :Float64;
+}
+
+struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
+  frameId @0 :UInt32;
+  modelExecutionTime @14 :Float32;
+  dspExecutionTime @16 :Float32;
+  rawPredictions @15 :Data;
+
+  faceOrientation @3 :List(Float32);
+  facePosition @4 :List(Float32);
+  faceProb @5 :Float32;
+  leftEyeProb @6 :Float32;
+  rightEyeProb @7 :Float32;
+  leftBlinkProb @8 :Float32;
+  rightBlinkProb @9 :Float32;
+  faceOrientationStd @11 :List(Float32);
+  facePositionStd @12 :List(Float32);
+  sunglassesProb @13 :Float32;
+  poorVision @17 :Float32;
+  partialFace @18 :Float32;
+  distractedPose @19 :Float32;
+  distractedEyes @20 :Float32;
+  eyesOnRoad @21 :Float32;
+  phoneUse @22 :Float32;
+  occludedProb @23 :Float32;
+
+  readyProb @24 :List(Float32);
+  notReadyProb @25 :List(Float32);
+
+  irPwrDEPRECATED @10 :Float32;
+  descriptorDEPRECATED @1 :List(Float32);
+  stdDEPRECATED @2 :Float32;
+}
+
+struct NavModelData @0xac3de5c437be057a {
+  frameId @0 :UInt32;
+  locationMonoTime @6 :UInt64;
+  modelExecutionTime @1 :Float32;
+  dspExecutionTime @2 :Float32;
+  features @3 :List(Float32);
+  # predicted future position
+  position @4 :XYData;
+  desirePrediction @5 :List(Float32);
+
+  # All SI units and in device frame
+  struct XYData @0xbe09e615b2507e26 {
+    x @0 :List(Float32);
+    y @1 :List(Float32);
+    xStd @2 :List(Float32);
+    yStd @3 :List(Float32);
+  }
+}
+
+struct AndroidBuildInfo @0xfe2919d5c21f426c {
+  board @0 :Text;
+  bootloader @1 :Text;
+  brand @2 :Text;
+  device @3 :Text;
+  display @4 :Text;
+  fingerprint @5 :Text;
+  hardware @6 :Text;
+  host @7 :Text;
+  id @8 :Text;
+  manufacturer @9 :Text;
+  model @10 :Text;
+  product @11 :Text;
+  radioVersion @12 :Text;
+  serial @13 :Text;
+  supportedAbis @14 :List(Text);
+  tags @15 :Text;
+  time @16 :Int64;
+  type @17 :Text;
+  user @18 :Text;
+
+  versionCodename @19 :Text;
+  versionRelease @20 :Text;
+  versionSdk @21 :Int32;
+  versionSecurityPatch @22 :Text;
+}
+
+struct AndroidSensor @0x9b513b93a887dbcd {
+  id @0 :Int32;
+  name @1 :Text;
+  vendor @2 :Text;
+  version @3 :Int32;
+  handle @4 :Int32;
+  type @5 :Int32;
+  maxRange @6 :Float32;
+  resolution @7 :Float32;
+  power @8 :Float32;
+  minDelay @9 :Int32;
+  fifoReservedEventCount @10 :UInt32;
+  fifoMaxEventCount @11 :UInt32;
+  stringType @12 :Text;
+  maxDelay @13 :Int32;
+}
+
+struct IosBuildInfo @0xd97e3b28239f5580 {
+  appVersion @0 :Text;
+  appBuild @1 :UInt32;
+  osVersion @2 :Text;
+  deviceModel @3 :Text;
+}
+
+enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
+  unknown @0;
+  neo @1;
+  chffrAndroid @2;
+  front @3;
+}
+
+struct AndroidCaptureResult @0xbcc3efbac41d2048 {
+  sensitivity @0 :Int32;
+  frameDuration @1 :Int64;
+  exposureTime @2 :Int64;
+  rollingShutterSkew @3 :UInt64;
+  colorCorrectionTransform @4 :List(Int32);
+  colorCorrectionGains @5 :List(Float32);
+  displayRotation @6 :Int8;
+}
+
+enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
+  none @0;
+  client @1;
+  cdp @2;
+  dcp @3;
+}
+
+struct LateralINDIState @0x939463348632375e {
+  active @0 :Bool;
+  steeringAngleDeg @1 :Float32;
+  steeringRateDeg @2 :Float32;
+  steeringAccelDeg @3 :Float32;
+  rateSetPoint @4 :Float32;
+  accelSetPoint @5 :Float32;
+  accelError @6 :Float32;
+  delayedOutput @7 :Float32;
+  delta @8 :Float32;
+  output @9 :Float32;
+  saturated @10 :Bool;
+  steeringAngleDesiredDeg @11 :Float32;
+  steeringRateDesiredDeg @12 :Float32;
+}
+
+struct LateralLQRState @0x9024e2d790c82ade {
+  active @0 :Bool;
+  steeringAngleDeg @1 :Float32;
+  i @2 :Float32;
+  output @3 :Float32;
+  lqrOutput @4 :Float32;
+  saturated @5 :Bool;
+  steeringAngleDesiredDeg @6 :Float32;
+}
+
+struct LateralCurvatureState @0xad9d8095c06f7c61 {
+  active @0 :Bool;
+  actualCurvature @1 :Float32;
+  desiredCurvature @2 :Float32;
+  error @3 :Float32;
+  p @4 :Float32;
+  i @5 :Float32;
+  f @6 :Float32;
+  output @7 :Float32;
+  saturated @8 :Bool;
+}
+
+struct LateralPlannerSolution @0x84caeca5a6b4acfe {
+  x @0 :List(Float32);
+  y @1 :List(Float32);
+  yaw @2 :List(Float32);
+  yawRate @3 :List(Float32);
+  xStd @4 :List(Float32);
+  yStd @5 :List(Float32);
+  yawStd @6 :List(Float32);
+  yawRateStd @7 :List(Float32);
+}
+
+struct GpsTrajectory @0x8cfeb072f5301000 {
+  x @0 :List(Float32);
+  y @1 :List(Float32);
+}

cereal/messaging/__init__.py

@@ -1,10 +1,8 @@
 # must be built with scons
-from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-from msgq.ipc_pyx import MultiplePublishersError, IpcError
-from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
+from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
+                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
 import msgq
-
 import os
 import capnp
 import time
@@ -13,11 +11,25 @@ from typing import Optional, List, Union, Dict
 
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.util import MovingAverage
+from openpilot.common.utils import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
+def pub_sock(endpoint: str) -> PubSocket:
+  service = SERVICE_LIST.get(endpoint)
+  segment_size = service.queue_size if service else 0
+  return msgq.pub_sock(endpoint, segment_size)
+
+
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
+  service = SERVICE_LIST.get(endpoint)
+  segment_size = service.queue_size if service else 0
+  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
+                       timeout=timeout, segment_size=segment_size)
+
+
 def reset_context():
   msgq.context = Context()
 
@@ -247,11 +259,11 @@ class PubMaster:
     self.sock[s].send(dat)
 
   def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
-    for _ in range(int(timeout*(1./dt))):
-      if self.sock[s].all_readers_updated():
-        return True
-      time.sleep(dt)
-    return False
+    try:
+      self.sock[s].wait_for_readers(timeout=timeout, interval=dt)
+      return True
+    except TimeoutError:
+      return False
 
   def all_readers_updated(self, s: str) -> bool:
-    return self.sock[s].all_readers_updated()  # type: ignore
+    return self.sock[s].all_readers_updated()

cereal/messaging/bridge.cc

@@ -25,15 +25,16 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<ZMQPoller>();
-  auto pub_context = std::make_unique<MSGQContext>();
-  auto sub_context = std::make_unique<ZMQContext>();
-  std::map<SubSocket *, PubSocket *> sub2pub;
+  auto poller = std::make_unique<BridgeZmqPoller>();
+  auto pub_context = std::make_unique<Context>();
+  auto sub_context = std::make_unique<BridgeZmqContext>();
+  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new MSGQPubSocket();
-    auto sub_sock = new ZMQSubSocket();
-    pub_sock->connect(pub_context.get(), endpoint);
+    auto pub_sock = new PubSocket();
+    auto sub_sock = new BridgeZmqSubSocket();
+    size_t queue_size = services.at(endpoint).queue_size;
+    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/bridge_zmq.cc

@@ -0,0 +1,170 @@
+#include "cereal/messaging/bridge_zmq.h"
+
+#include <cassert>
+#include <cstring>
+#include <unistd.h>
+
+static size_t fnv1a_hash(const std::string &str) {
+  const size_t fnv_prime = 0x100000001b3;
+  size_t hash_value = 0xcbf29ce484222325;
+  for (char c : str) {
+    hash_value ^= (unsigned char)c;
+    hash_value *= fnv_prime;
+  }
+  return hash_value;
+}
+
+// FIXME: This is a hack to get the port number from the socket name, might have collisions.
+static int get_port(std::string endpoint) {
+  size_t hash_value = fnv1a_hash(endpoint);
+  int start_port = 8023;
+  int max_port = 65535;
+  return start_port + (hash_value % (max_port - start_port));
+}
+
+BridgeZmqContext::BridgeZmqContext() {
+  context = zmq_ctx_new();
+}
+
+BridgeZmqContext::~BridgeZmqContext() {
+  if (context != nullptr) {
+    zmq_ctx_term(context);
+  }
+}
+
+void BridgeZmqMessage::init(size_t sz) {
+  size = sz;
+  data = new char[size];
+}
+
+void BridgeZmqMessage::init(char *d, size_t sz) {
+  size = sz;
+  data = new char[size];
+  memcpy(data, d, size);
+}
+
+void BridgeZmqMessage::close() {
+  if (size > 0) {
+    delete[] data;
+  }
+  data = nullptr;
+  size = 0;
+}
+
+BridgeZmqMessage::~BridgeZmqMessage() {
+  close();
+}
+
+int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
+  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
+  if (sock == nullptr) {
+    return -1;
+  }
+
+  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
+
+  if (conflate) {
+    int arg = 1;
+    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
+  }
+
+  int reconnect_ivl = 500;
+  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
+
+  full_endpoint = "tcp://" + address + ":";
+  if (check_endpoint) {
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
+
+  return zmq_connect(sock, full_endpoint.c_str());
+}
+
+void BridgeZmqSubSocket::setTimeout(int timeout) {
+  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
+}
+
+Message *BridgeZmqSubSocket::receive(bool non_blocking) {
+  zmq_msg_t msg;
+  assert(zmq_msg_init(&msg) == 0);
+
+  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
+  int rc = zmq_msg_recv(&msg, sock, flags);
+
+  Message *ret = nullptr;
+  if (rc >= 0) {
+    ret = new BridgeZmqMessage;
+    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
+  }
+
+  zmq_msg_close(&msg);
+  return ret;
+}
+
+BridgeZmqSubSocket::~BridgeZmqSubSocket() {
+  if (sock != nullptr) {
+    zmq_close(sock);
+  }
+}
+
+int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
+  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
+  if (sock == nullptr) {
+    return -1;
+  }
+
+  full_endpoint = "tcp://*:";
+  if (check_endpoint) {
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
+
+  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
+  pid = getpid();
+
+  return zmq_bind(sock, full_endpoint.c_str());
+}
+
+int BridgeZmqPubSocket::sendMessage(Message *message) {
+  assert(pid == getpid());
+  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
+}
+
+int BridgeZmqPubSocket::send(char *data, size_t size) {
+  assert(pid == getpid());
+  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
+}
+
+BridgeZmqPubSocket::~BridgeZmqPubSocket() {
+  if (sock != nullptr) {
+    zmq_close(sock);
+  }
+}
+
+void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
+  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
+  polls[num_polls].socket = socket->getRawSocket();
+  polls[num_polls].events = ZMQ_POLLIN;
+
+  sockets.push_back(socket);
+  num_polls++;
+}
+
+std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
+  std::vector<BridgeZmqSubSocket *> ret;
+
+  int rc = zmq_poll(polls, num_polls, timeout);
+  if (rc < 0) {
+    return ret;
+  }
+
+  for (size_t i = 0; i < num_polls; i++) {
+    if (polls[i].revents) {
+      ret.push_back(sockets[i]);
+    }
+  }
+
+  return ret;
+}

cereal/messaging/bridge_zmq.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include <zmq.h>
+
+#include "msgq/ipc.h"
+
+class BridgeZmqContext {
+public:
+  BridgeZmqContext();
+  void *getRawContext() { return context; }
+  ~BridgeZmqContext();
+
+private:
+  void *context = nullptr;
+};
+
+class BridgeZmqMessage : public Message {
+public:
+  void init(size_t size);
+  void init(char *data, size_t size);
+  void close();
+  size_t getSize() { return size; }
+  char *getData() { return data; }
+  ~BridgeZmqMessage();
+
+private:
+  char *data = nullptr;
+  size_t size = 0;
+};
+
+class BridgeZmqSubSocket {
+public:
+  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
+  void setTimeout(int timeout);
+  Message *receive(bool non_blocking = false);
+  void *getRawSocket() { return sock; }
+  ~BridgeZmqSubSocket();
+
+private:
+  void *sock = nullptr;
+  std::string full_endpoint;
+};
+
+class BridgeZmqPubSocket {
+public:
+  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
+  int sendMessage(Message *message);
+  int send(char *data, size_t size);
+  void *getRawSocket() { return sock; }
+  ~BridgeZmqPubSocket();
+
+private:
+  void *sock = nullptr;
+  std::string full_endpoint;
+  int pid = -1;
+};
+
+class BridgeZmqPoller {
+public:
+  void registerSocket(BridgeZmqSubSocket *socket);
+  std::vector<BridgeZmqSubSocket *> poll(int timeout);
+
+private:
+  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
+  std::vector<BridgeZmqSubSocket *> sockets;
+  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
+  size_t num_polls = 0;
+};

cereal/messaging/msgq_to_zmq.cc

@@ -2,6 +2,7 @@
 
 #include <cassert>
 
+#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -21,14 +22,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<ZMQContext>();
-  msgq_context = std::make_unique<MSGQContext>();
+  zmq_context = std::make_unique<BridgeZmqContext>();
+  msgq_context = std::make_unique<Context>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
+    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -48,7 +49,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
+        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -71,7 +72,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -108,7 +109,8 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
+            size_t queue_size = services.at(pair.endpoint).queue_size;
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -128,7 +130,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,9 +7,8 @@
 #include <string>
 #include <vector>
 
-#define private public
 #include "msgq/impl_msgq.h"
-#include "msgq/impl_zmq.h"
+#include "cereal/messaging/bridge_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -22,16 +21,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<ZMQPubSocket> pub_sock;
+    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<MSGQContext> msgq_context;
-  std::unique_ptr<ZMQContext> zmq_context;
+  std::unique_ptr<Context> msgq_context;
+  std::unique_ptr<BridgeZmqContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
+  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -33,7 +33,7 @@ MessageContext message_context;
 struct SubMaster::SubMessage {
   std::string name;
   SubSocket *socket = nullptr;
-  int freq = 0;
+  float freq = 0.0f;
   bool updated = false, alive = false, valid = false, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,7 +187,8 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    PubSocket *socket = PubSocket::create(message_context.context(), name);
+    service serv = services.at(std::string(name));
+    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/messaging/tests/test_messaging.py

@@ -5,7 +5,7 @@ import numbers
 import random
 import threading
 import time
-from parameterized import parameterized
+from openpilot.common.parameterized import parameterized
 import pytest
 
 from cereal import log, car
@@ -30,7 +30,7 @@ def zmq_sleep(t=1):
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "steeringTorque", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:

cereal/messaging/tests/test_services.py

@@ -1,7 +1,7 @@
 import os
 import tempfile
 from typing import Dict
-from parameterized import parameterized
+from openpilot.common.parameterized import parameterized
 
 import cereal.services as services
 from cereal.services import SERVICE_LIST

cereal/messaging/tests/validate_sp_cereal_upstream.py

@@ -1,222 +1,256 @@
 #!/usr/bin/env python3
+"""Schema-level cereal compat check between sunnypilot and upstream openpilot.
+
+Rules (per struct matched across sides by typeId):
+  R1  shared ordinal must reference the same type.
+  R2  sunnypilot-only ordinal in a union -> FAIL (unknown discriminant upstream).
+  R3  sunnypilot-only ordinal on a regular field -> OK (additive struct evolution).
+  R4  upstream-only ordinal -> OK.
+  R5  sunnypilot-only struct referenced via an upstream-shared field -> FAIL.
+"""
+
+from __future__ import annotations
+
 import argparse
+import json
+import os
 import sys
-from typing import Any, List, Tuple
+from typing import Any
 
-DEBUG = False
+NO_DISCRIMINANT = 0xFFFF
 
 
-def print_debug(string: str) -> None:
-  if DEBUG:
-    print(string)
+def hex_id(value: int) -> str:
+  return f"0x{value:016x}"
 
 
-def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
-  """
-  Create a new instance of a schema type, handling different field types.
-
-  Args:
-      struct: The Cap'n Proto schema structure
-      prop: A tuple containing the field name and field metadata
-
-  Returns:
-      A new initialized schema instance
-  """
-  struct_instance = struct.new_message()
-  field_name, field_metadata = prop
-
-  try:
-    field_type = field_metadata.proto.slot.type.which()
-
-    # Initialize different types of fields
-    if field_type in ('list', 'text', 'data'):
-      struct_instance.init(field_name, 1)
-      print_debug(f"Initialized list/text/data field: {field_name}")
-    elif field_type in ('struct', 'object'):
-      struct_instance.init(field_name)
-      print_debug(f"Initialized struct/object field: {field_name}")
-
-    return struct_instance
-
-  except Exception as e:
-    print(f"Error creating instance for {field_name}: {e}")
-    return None
+def encode_type(type_node: Any) -> dict:
+  which = type_node.which()
+  if which == "struct":
+    return {"kind": "struct", "typeId": hex_id(type_node.struct.typeId)}
+  if which == "enum":
+    return {"kind": "enum", "typeId": hex_id(type_node.enum.typeId)}
+  if which == "interface":
+    return {"kind": "interface", "typeId": hex_id(type_node.interface.typeId)}
+  if which == "list":
+    return {"kind": "list", "element": encode_type(type_node.list.elementType)}
+  if which == "anyPointer":
+    return {"kind": "anyPointer"}
+  return {"kind": which}
 
 
-def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
-  """
-  Retrieve all fields from a given schema structure.
+def encode_field(name: str, field: Any) -> dict:
+  proto = field.proto
+  ordinal = proto.ordinal.explicit if proto.ordinal.which() == "explicit" else None
+  discriminant = proto.discriminantValue if proto.discriminantValue != NO_DISCRIMINANT else None
 
-  Args:
-      schema_struct: The Cap'n Proto schema structure
+  if proto.which() == "group":
+    type_desc = {"kind": "group", "typeId": hex_id(proto.group.typeId)}
+  else:
+    type_desc = encode_type(proto.slot.type)
 
-  Returns:
-      A list of field names and their metadata
-  """
-  try:
-    # Get all fields from the schema
-    schema_fields = list(schema_struct.schema.fields.items())
-
-    print_debug("Discovered schema fields:")
-    for field_name, field_metadata in schema_fields:
-      print_debug(f"- {field_name}")
-
-    return schema_fields
-
-  except Exception as e:
-    print(f"Error retrieving schema fields: {e}")
-    return []
+  return {
+    "name": name,
+    "ordinal": ordinal,
+    "discriminant": discriminant,
+    "type": type_desc,
+  }
 
 
-def generate_schema_instances(schema_struct: Any) -> List[Any]:
-  """
-  Generate instances for all fields in a given schema.
-
-  Args:
-      schema_struct: The Cap'n Proto schema structure
-
-  Returns:
-      A list of schema instances
-  """
-  schema_fields = get_schema_fields(schema_struct)
-  instances = []
-
-  for field_prop in schema_fields:
-    try:
-      instance = create_schema_instance(schema_struct, field_prop)
-      if instance is not None:
-        instances.append(instance)
-    except Exception as e:
-      print(f"Skipping field due to error: {e}")
-
-  print(f"Generated {len(instances)} schema instances")
-  return instances
+def encode_struct(schema: Any) -> dict:
+  node = schema.node
+  return {
+    "typeId": hex_id(node.id),
+    "displayName": node.displayName,
+    "hasUnion": node.struct.discriminantCount > 0,
+    "fields": [encode_field(name, field) for name, field in schema.fields.items()],
+  }
 
 
-def persist_instances(instances: List[Any], filename: str) -> None:
-  """
-  Write schema instances to a binary file.
-
-  Args:
-      instances: List of schema instances
-      filename: Output file path
-  """
-  try:
-    with open(filename, 'wb') as f:
-      for instance in instances:
-        f.write(instance.to_bytes())
-
-    print(f"Successfully wrote {len(instances)} instances to {filename}")
-
-  except Exception as e:
-    print(f"Error persisting instances: {e}")
-    sys.exit(1)
+def _child_struct_schema(field: Any) -> Any:
+  proto = field.proto
+  if proto.which() == "group":
+    return field.schema
+  type_node = proto.slot.type
+  which = type_node.which()
+  if which == "struct":
+    return field.schema
+  if which == "list":
+    container = field.schema
+    element_type = type_node.list.elementType
+    while element_type.which() == "list":
+      container = container.elementType
+      element_type = element_type.list.elementType
+    if element_type.which() == "struct":
+      return container.elementType
+  return None
 
 
-def read_instances(filename: str, schema_type: Any) -> List[Any]:
-  """
-  Read schema instances from a binary file.
-
-  Args:
-      filename: Input file path
-      schema_type: The schema type to use for reading
-
-  Returns:
-      A list of read schema instances
-  """
-  try:
-    with open(filename, 'rb') as f:
-      data = f.read()
-
-    instances = list(schema_type.read_multiple_bytes(data))
-
-    print(f"Read {len(instances)} instances from {filename}")
-    return instances
-
-  except Exception as e:
-    print(f"Error reading instances: {e}")
-    sys.exit(1)
+def collect_schema(root: Any) -> dict[str, dict]:
+  structs: dict[str, dict] = {}
+  stack = [root]
+  while stack:
+    schema = stack.pop()
+    type_id = hex_id(schema.node.id)
+    if type_id in structs:
+      continue
+    structs[type_id] = encode_struct(schema)
+    for _name, field in schema.fields.items():
+      try:
+        child = _child_struct_schema(field)
+      except Exception:
+        child = None
+      if child is not None:
+        stack.append(child)
+  return structs
 
 
-def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
-  """
-  Compare original and read-back instances to detect potential breaking changes.
+def load_log(cereal_dir: str) -> Any:
+  import capnp
+  cereal_dir = os.path.abspath(cereal_dir)
+  capnp.remove_import_hook()
+  return capnp.load(os.path.join(cereal_dir, "log.capnp"), imports=[cereal_dir])
 
-  Args:
-      original_instances: List of originally generated instances
-      read_instances: List of instances read back from file
 
-  Returns:
-      Boolean indicating whether schemas appear compatible
-  """
-  if len(original_instances) != len(read_instances):
-    print("❌ Schema Compatibility Warning: Instance count mismatch")
+def dump_schema(cereal_dir: str, path: str) -> None:
+  log = load_log(cereal_dir)
+  payload = {
+    "root": hex_id(log.Event.schema.node.id),
+    "structs": collect_schema(log.Event.schema),
+  }
+  with open(path, "w", encoding="utf-8") as handle:
+    json.dump(payload, handle, indent=2, sort_keys=True)
+  print(f"wrote schema dump with {len(payload['structs'])} structs to {path}")
+
+
+def types_equal(a: dict, b: dict) -> bool:
+  if a.get("kind") != b.get("kind"):
     return False
-
-  compatible = True
-  for struct in read_instances:
-    try:
-      getattr(struct, struct.which())  # Attempting to access the field to validate readability
-    except Exception as e:
-      print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
-      compatible = False
-
-  return compatible
+  kind = a["kind"]
+  if kind in ("struct", "enum", "interface", "group"):
+    return a.get("typeId") == b.get("typeId")
+  if kind == "list":
+    return types_equal(a["element"], b["element"])
+  return True
 
 
-def main():
-  """
-  CLI entry point for schema compatibility testing.
-  """
-  # Setup argument parser
+def type_repr(t: dict) -> str:
+  kind = t.get("kind", "?")
+  if kind in ("struct", "enum", "interface", "group"):
+    return f"{kind}({t.get('typeId')})"
+  if kind == "list":
+    return f"list<{type_repr(t['element'])}>"
+  return kind
+
+
+def field_is_union_variant(field: dict) -> bool:
+  return field.get("discriminant") is not None
+
+
+def index_fields_by_ordinal(struct: dict) -> dict[int, dict]:
+  indexed: dict[int, dict] = {}
+  for field in struct["fields"]:
+    ordinal = field.get("ordinal")
+    if ordinal is None:
+      continue
+    indexed[ordinal] = field
+  return indexed
+
+
+def compare(sunnypilot_dump: dict, upstream_dump: dict) -> list[str]:
+  violations: list[str] = []
+  sunnypilot_structs: dict[str, dict] = sunnypilot_dump["structs"]
+  upstream_structs: dict[str, dict] = upstream_dump["structs"]
+
+  sunnypilot_struct_referenced_from_shared: set[str] = set()
+
+  for type_id, sunnypilot_struct in sunnypilot_structs.items():
+    upstream_struct = upstream_structs.get(type_id)
+    if upstream_struct is None:
+      continue
+
+    sunnypilot_fields = index_fields_by_ordinal(sunnypilot_struct)
+    upstream_fields = index_fields_by_ordinal(upstream_struct)
+    display = sunnypilot_struct["displayName"]
+
+    for ordinal, sunnypilot_field in sunnypilot_fields.items():
+      upstream_field = upstream_fields.get(ordinal)
+      if upstream_field is None:
+        if field_is_union_variant(sunnypilot_field):
+          violations.append(
+            f"[R2] {display} @{ordinal} ('{sunnypilot_field['name']}', {type_repr(sunnypilot_field['type'])}): "
+            f"union variant not present upstream. upstream cannot parse this discriminant."
+          )
+        continue
+
+      if not types_equal(sunnypilot_field["type"], upstream_field["type"]):
+        violations.append(
+          f"[R1] {display} @{ordinal}: type mismatch. "
+          f"sunnypilot='{sunnypilot_field['name']}' {type_repr(sunnypilot_field['type'])} vs "
+          f"upstream='{upstream_field['name']}' {type_repr(upstream_field['type'])}."
+        )
+        continue
+
+      cursor = sunnypilot_field["type"]
+      while cursor.get("kind") == "list":
+        cursor = cursor["element"]
+      if cursor.get("kind") in ("struct", "group", "interface") and cursor.get("typeId"):
+        sunnypilot_struct_referenced_from_shared.add(cursor["typeId"])
+
+  for type_id, sunnypilot_struct in sunnypilot_structs.items():
+    if type_id in upstream_structs:
+      continue
+    if type_id in sunnypilot_struct_referenced_from_shared:
+      violations.append(
+        f"[R5] struct {sunnypilot_struct['displayName']} ({type_id}) exists only on sunnypilot "
+        f"but is referenced from an upstream-shared field. upstream cannot resolve this type."
+      )
+
+  return violations
+
+
+def load_peer(path: str) -> dict:
+  with open(path, "r", encoding="utf-8") as handle:
+    return json.load(handle)
+
+
+def run_read(cereal_dir: str, peer_path: str) -> int:
+  log = load_log(cereal_dir)
+  peer_dump = load_peer(peer_path)
+  local_dump = {
+    "root": hex_id(log.Event.schema.node.id),
+    "structs": collect_schema(log.Event.schema),
+  }
+  violations = compare(sunnypilot_dump=peer_dump, upstream_dump=local_dump)
+
+  if not violations:
+    print("cereal compat OK: upstream openpilot can parse sunnypilot routes "
+          "(no leaked structs, no ordinal collisions).")
+    return 0
+
+  print(f"cereal compat FAIL: upstream openpilot would misparse sunnypilot routes "
+        f"({len(violations)} violation(s)):")
+  for v in violations:
+    print(f"  {v}")
+  return 1
+
+
+def main() -> int:
   parser = argparse.ArgumentParser(
-    description='Cap\'n Proto Schema Compatibility Testing Tool',
-    epilog='Test schema compatibility by generating and reading back instances.'
+    description="sunnypilot <-> upstream cereal compatibility validator (schema-level)."
   )
-
-  # Add mutually exclusive group for generation or reading mode
-  mode_group = parser.add_mutually_exclusive_group(required=True)
-  mode_group.add_argument('-g', '--generate', action='store_true',
-                          help='Generate schema instances')
-  mode_group.add_argument('-r', '--read', action='store_true',
-                          help='Read and validate schema instances')
-
-  # Common arguments
-  parser.add_argument('-f', '--file',
-                      default='schema_instances.bin',
-                      help='Output/input binary file (default: schema_instances.bin)')
-
-  # Parse arguments
+  mode = parser.add_mutually_exclusive_group(required=True)
+  mode.add_argument("-g", "--generate", action="store_true", help="dump local schema to JSON")
+  mode.add_argument("-r", "--read", action="store_true", help="load peer JSON and diff against local")
+  parser.add_argument("-f", "--file", default="schema.json", help="JSON file path (default: schema.json)")
+  parser.add_argument("--cereal-dir", required=True, help="path to cereal directory containing log.capnp")
   args = parser.parse_args()
 
-  # Import the schema dynamically 
-  try:
-    from cereal import log
-    schema_type = log.Event
-  except ImportError:
-    print("Error: Unable to import schema. Ensure 'cereal' is installed.")
-    sys.exit(1)
-
-  # Execute based on mode
   if args.generate:
-    print("🔧 Generating Schema Instances")
-    instances = generate_schema_instances(schema_type)
-    persist_instances(instances, args.file)
-    print("✅ Instance generation complete")
-
-  elif args.read:
-    print("🔍 Reading and Validating Schema Instances")
-    generated_instances = generate_schema_instances(schema_type)
-    read_back_instances = read_instances(args.file, schema_type)
-
-    # Compare schemas
-    if compare_schemas(generated_instances, read_back_instances):
-      print("✅ Schema Compatibility: No breaking changes detected")
-      sys.exit(0)
-    else:
-      print("❌ Potential Schema Breaking Changes Detected")
-      sys.exit(1)
+    dump_schema(args.cereal_dir, args.file)
+    return 0
+  return run_read(args.cereal_dir, args.file)
 
 
 if __name__ == "__main__":
-  main()
+  sys.exit(main())

cereal/services.py

@@ -1,39 +1,43 @@
 #!/usr/bin/env python3
+from enum import IntEnum
 from typing import Optional
 
 
+# TODO: this should be automatically determined using the capnp schema
+class QueueSize(IntEnum):
+  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
+  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
+  SMALL = 250 * 1024          # 250KB - most services
+
+
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
+               queue_size: QueueSize = QueueSize.SMALL):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
+    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
-  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
-  "accelerometer2": (True, 100., 100),
-  "magnetometer": (True, 25.),
-  "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
-  "temperatureSensor2": (True, 2., 200),
-  "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
   "touch": (True, 20., 1),
-  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10),
+  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
   "selfdriveState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139),
-  "logMessage": (True, 0.),
-  "errorLogMessage": (True, 0., 1),
+  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
+  "logMessage": (True, 0., None, QueueSize.BIG),
+  "errorLogMessage": (True, 0., 1, QueueSize.BIG),
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
   "liveDelay": (True, 4., 1),
@@ -42,13 +46,13 @@ _services: dict[str, tuple] = {
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
+  "lateralManeuverPlan": (True, 20.),
   "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15),
+  "procLog": (True, 0.5, 15, QueueSize.BIG),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
   "qcomGnss": (True, 2.),
-  "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
   "livePose": (True, 20., 4),
@@ -65,22 +69,22 @@ _services: dict[str, tuple] = {
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20.),
+  "modelV2": (True, 20., None, QueueSize.BIG),
   "managerState": (True, 2., 1),
-  "uploaderState": (True, 0., 1),
-  "navInstruction": (True, 1., 10),
-  "navRoute": (True, 0.),
-  "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
   "userBookmark": (True, 0., 1),
   "soundPressure": (True, 10., 10),
   "rawAudioData": (False, 20.),
   "bookmarkButton": (True, 0., 1),
   "audioFeedback": (True, 0., 1),
+  "roadEncodeData": (False, 20., None, QueueSize.BIG),
+  "driverEncodeData": (False, 20., None, QueueSize.BIG),
+  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
 
   # sunnypilot
-  "modelManagerSP": (False, 1., 1),
-  "backupManagerSP": (False, 1., 1),
+  "modelManagerSP": (False, 1., 1, QueueSize.BIG),
+  "backupManagerSP": (False, 1., 1, QueueSize.BIG),
   "selfdriveStateSP": (True, 100., 10),
   "longitudinalPlanSP": (True, 20., 10),
   "onroadEventsSP": (True, 1., 1),
@@ -88,24 +92,20 @@ _services: dict[str, tuple] = {
   "carControlSP": (True, 100., 10),
   "carStateSP": (True, 100., 10),
   "liveMapDataSP": (True, 1., 1),
+  "modelDataV2SP": (True, 20., None, QueueSize.BIG),
+  "liveLocationKalman": (True, 20.),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
-  "roadEncodeData": (False, 20.),
-  "driverEncodeData": (False, 20.),
-  "wideRoadEncodeData": (False, 20.),
-  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20.),
-  "livestreamRoadEncodeData": (False, 20.),
-  "livestreamDriverEncodeData": (False, 20.),
+  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
   "customReservedRawData0": (True, 0.),
-  "customReservedRawData1": (True, 0.),
-  "customReservedRawData2": (True, 0.),
 }
 SERVICE_LIST = {name: Service(*vals) for
                 idx, (name, vals) in enumerate(_services.items())}
@@ -120,13 +120,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
+  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation)
+    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation, v.queue_size)
   h += "};\n"
 
   h += "#endif\n"

common/.gitignore

@@ -1 +0,0 @@
-*.cpp

common/SConscript

@@ -1,21 +1,14 @@
-Import('env', 'envCython', 'arch')
+Import('env', 'envCython')
 
 common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
-  'watchdog.cc',
-  'ratekeeper.cc'
+  'ratekeeper.cc',
 ]
 
 _common = env.Library('common', common_libs, LIBS="json11")
-
-files = [
-  'clutil.cc',
-]
-
-_gpucommon = env.Library('gpucommon', files)
-Export('_common', '_gpucommon')
+Export('_common')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
@@ -25,11 +18,6 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-SConscript([
-  'transformations/SConscript',
-])
-
-Import('transformations_python')
-common_python = [params_python, transformations_python]
+common_python = [params_python]
 
 Export('common_python')

common/api/__init__.py

@@ -14,9 +14,13 @@ class Api:
   def post(self, *args, **kwargs):
     return self.service.post(*args, **kwargs)
 
-  def get_token(self, expiry_hours=1):
-    return self.service.get_token(expiry_hours)
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    return self.service.get_token(payload_extra, expiry_hours)
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
-  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, **params)
+def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
+  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
+
+
+def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+  return CommaConnectApi(None).get_key_pair()

common/api/base.py

@@ -1,18 +1,22 @@
 import jwt
+import os
 import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
 from openpilot.system.hardware.hw import Paths
 from openpilot.system.version import get_version
 
+# name: jwt signature algorithm
+KEYS = {"id_rsa": "RS256",
+        "id_ecdsa": "ES256"}
+
 
 class BaseApi:
   def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
     self.dongle_id = dongle_id
     self.api_host = api_host
     self.user_agent = user_agent
-    with open(f'{Paths.persist_root()}/comma/id_rsa') as f:
-      self.private_key = f.read()
+    self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
 
   def get(self, *args, **kwargs):
     return self.request('GET', *args, **kwargs)
@@ -23,7 +27,7 @@ class BaseApi:
   def request(self, method, endpoint, timeout=None, access_token=None, **params):
     return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
 
-  def _get_token(self, expiry_hours=1, **extra_payload):
+  def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
     now = datetime.now(UTC).replace(tzinfo=None)
     payload = {
       'identity': self.dongle_id,
@@ -32,20 +36,22 @@ class BaseApi:
       'exp': now + timedelta(hours=expiry_hours),
       **extra_payload
     }
-    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if payload_extra is not None:
+      payload.update(payload_extra)
+    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
     if isinstance(token, bytes):
       token = token.decode('utf8')
     return token
 
-  def get_token(self, expiry_hours=1):
-    return self._get_token(expiry_hours)
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    return self._get_token(payload_extra, expiry_hours)
 
   def remove_non_ascii_chars(self, text):
     normalized_text = unicodedata.normalize('NFD', text)
     ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
     return ascii_encoded_text.decode()
 
-  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, json=None, **params):
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
     headers = {}
     if access_token is not None:
       headers['Authorization'] = "JWT " + access_token
@@ -53,4 +59,14 @@ class BaseApi:
     version = self.remove_non_ascii_chars(get_version())
     headers['User-Agent'] = self.user_agent + version
 
-    return requests.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
+    # TODO: add session to Api
+    req = requests if session is None else session
+    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
+
+  @staticmethod
+  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+    for key in KEYS:
+      if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
+        with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
+          return KEYS[key], private.read(), public.read()
+    return None, None, None

common/clutil.cc

@@ -1,98 +0,0 @@
-#include "common/clutil.h"
-
-#include <cassert>
-#include <iostream>
-#include <memory>
-
-#include "common/util.h"
-#include "common/swaglog.h"
-
-namespace {  // helper functions
-
-template <typename Func, typename Id, typename Name>
-std::string get_info(Func get_info_func, Id id, Name param_name) {
-  size_t size = 0;
-  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
-  std::string info(size, '\0');
-  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
-  return info;
-}
-inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
-inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
-
-void cl_print_info(cl_platform_id platform, cl_device_id device) {
-  size_t work_group_size = 0;
-  cl_device_type device_type = 0;
-  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
-  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
-  const char *type_str = "Other...";
-  switch (device_type) {
-    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
-    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
-    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
-  }
-
-  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
-  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
-  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
-  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
-  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
-  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
-  LOGD("max work group size: %zu", work_group_size);
-  LOGD("type = %d, %s", (int)device_type, type_str);
-}
-
-void cl_print_build_errors(cl_program program, cl_device_id device) {
-  cl_build_status status;
-  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
-  size_t log_size;
-  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
-  std::string log(log_size, '\0');
-  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
-
-  LOGE("build failed; status=%d, log: %s", status, log.c_str());
-}
-
-}  // namespace
-
-cl_device_id cl_get_device_id(cl_device_type device_type) {
-  cl_uint num_platforms = 0;
-  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
-  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
-  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
-
-  for (size_t i = 0; i < num_platforms; ++i) {
-    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
-
-    // Get first device
-    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
-      cl_print_info(platform_ids[i], device_id);
-      return device_id;
-    }
-  }
-  LOGE("No valid openCL platform found");
-  assert(0);
-  return nullptr;
-}
-
-cl_context cl_create_context(cl_device_id device_id) {
-  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
-}
-
-void cl_release_context(cl_context context) {
-  clReleaseContext(context);
-}
-
-cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
-  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
-}
-
-cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
-  const char *csrc = src.c_str();
-  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
-  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
-    cl_print_build_errors(prg, device_id);
-    assert(0);
-  }
-  return prg;
-}

common/clutil.h

@@ -1,28 +0,0 @@
-#pragma once
-
-#ifdef __APPLE__
-#include <OpenCL/cl.h>
-#else
-#include <CL/cl.h>
-#endif
-
-#include <string>
-
-#define CL_CHECK(_expr)          \
-  do {                           \
-    assert(CL_SUCCESS == (_expr)); \
-  } while (0)
-
-#define CL_CHECK_ERR(_expr)           \
-  ({                                  \
-    cl_int err = CL_INVALID_VALUE;    \
-    __typeof__(_expr) _ret = _expr;   \
-    assert(_ret&& err == CL_SUCCESS); \
-    _ret;                             \
-  })
-
-cl_device_id cl_get_device_id(cl_device_type device_type);
-cl_context cl_create_context(cl_device_id device_id);
-void cl_release_context(cl_context context);
-cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
-cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/dict_helpers.py

@@ -1,9 +0,0 @@
-# remove all keys that end in DEPRECATED
-def strip_deprecated_keys(d):
-  for k in list(d.keys()):
-    if isinstance(k, str):
-      if k.endswith('DEPRECATED'):
-        d.pop(k)
-      elif isinstance(d[k], dict):
-        strip_deprecated_keys(d[k])
-  return d

common/file_chunker.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import sys
+import math
+import os
+from pathlib import Path
+
+CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
+
+def get_chunk_name(name, idx, num_chunks):
+  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
+
+def get_manifest_path(name):
+  return f"{name}.chunkmanifest"
+
+def _chunk_paths(path, num_chunks):
+  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
+
+def get_chunk_targets(path, file_size):
+  num_chunks = math.ceil(file_size / CHUNK_SIZE)
+  return _chunk_paths(path, num_chunks)
+
+def chunk_file(path, targets):
+  manifest_path, *chunk_paths = targets
+  with open(path, 'rb') as f:
+    data = f.read()
+  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
+  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
+  for i, chunk_path in enumerate(chunk_paths):
+    with open(chunk_path, 'wb') as f:
+      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
+  Path(manifest_path).write_text(str(len(chunk_paths)))
+  os.remove(path)
+
+def get_existing_chunks(path):
+  if os.path.isfile(path):
+    return [path]
+  if os.path.isfile(manifest := get_manifest_path(path)):
+    num_chunks = int(Path(manifest).read_text().strip())
+    return _chunk_paths(path, num_chunks)
+  raise FileNotFoundError(path)
+
+def read_file_chunked(path):
+  manifest_path = get_manifest_path(path)
+  if os.path.isfile(manifest_path):
+    num_chunks = int(Path(manifest_path).read_text().strip())
+    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
+  if os.path.isfile(path):
+    return Path(path).read_bytes()
+  raise FileNotFoundError(path)
+
+
+if __name__ == "__main__":
+  path = sys.argv[1]
+  chunk_paths = get_chunk_targets(path, os.path.getsize(path))
+  chunk_file(path, chunk_paths)

common/file_helpers.py

@@ -1,58 +0,0 @@
-import io
-import os
-import tempfile
-import contextlib
-import zstandard as zstd
-
-LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change
-
-
-class CallbackReader:
-  """Wraps a file, but overrides the read method to also
-  call a callback function with the number of bytes read so far."""
-  def __init__(self, f, callback, *args):
-    self.f = f
-    self.callback = callback
-    self.cb_args = args
-    self.total_read = 0
-
-  def __getattr__(self, attr):
-    return getattr(self.f, attr)
-
-  def read(self, *args, **kwargs):
-    chunk = self.f.read(*args, **kwargs)
-    self.total_read += len(chunk)
-    self.callback(*self.cb_args, self.total_read)
-    return chunk
-
-
-@contextlib.contextmanager
-def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
-                        overwrite: bool = False):
-  """Write to a file atomically using a temporary file in the same directory as the destination file."""
-  dir_name = os.path.dirname(path)
-
-  if not overwrite and os.path.exists(path):
-    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
-
-  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
-    yield tmp_file
-    tmp_file_name = tmp_file.name
-  os.replace(tmp_file_name, path)
-
-
-def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
-  if not should_compress:
-    file_size = os.path.getsize(filepath)
-    file_stream = open(filepath, "rb")
-    return file_stream, file_size
-
-  # Compress the file on the fly
-  compressed_stream = io.BytesIO()
-  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
-
-  with open(filepath, "rb") as f:
-    compressor.copy_stream(f, compressed_stream)
-    compressed_size = compressed_stream.tell()
-    compressed_stream.seek(0)
-    return compressed_stream, compressed_size

common/filter_simple.py

@@ -15,3 +15,20 @@ class FirstOrderFilter:
       self.initialized = True
       self.x = x
     return self.x
+
+
+class BounceFilter(FirstOrderFilter):
+  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
+    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
+    self.bounce = bounce
+    super().__init__(x0, rc, dt, initialized)
+
+  def update(self, x):
+    super().update(x)
+    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
+    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
+    self.velocity.update(0.0)
+    if abs(self.velocity.x) < 1e-3:
+      self.velocity.x = 0.0
+    self.x += self.velocity.x
+    return self.x

common/git.py

@@ -1,30 +1,30 @@
 from functools import cache
 import subprocess
-from openpilot.common.run import run_cmd, run_cmd_default
+from openpilot.common.utils import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str = None) -> str:
+def get_short_branch(cwd: str | None = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str = None) -> str:
+def get_branch(cwd: str | None = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str = None) -> str:
+def get_origin(cwd: str | None = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
@@ -34,7 +34,7 @@ def get_origin(cwd: str = None) -> str:
 
 
 @cache
-def get_normalized_origin(cwd: str = None) -> str:
+def get_normalized_origin(cwd: str | None = None) -> str:
   return get_origin(cwd) \
     .replace("git@", "", 1) \
     .replace(".git", "", 1) \

common/i2c.py

@@ -0,0 +1,81 @@
+import os
+import fcntl
+import ctypes
+
+# I2C constants from /usr/include/linux/i2c-dev.h
+I2C_SLAVE = 0x0703
+I2C_SLAVE_FORCE = 0x0706
+I2C_SMBUS = 0x0720
+
+# SMBus transfer types
+I2C_SMBUS_READ = 1
+I2C_SMBUS_WRITE = 0
+I2C_SMBUS_BYTE_DATA = 2
+I2C_SMBUS_I2C_BLOCK_DATA = 8
+
+I2C_SMBUS_BLOCK_MAX = 32
+
+
+class _I2cSmbusData(ctypes.Union):
+  _fields_ = [
+    ("byte", ctypes.c_uint8),
+    ("word", ctypes.c_uint16),
+    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
+  ]
+
+
+class _I2cSmbusIoctlData(ctypes.Structure):
+  _fields_ = [
+    ("read_write", ctypes.c_uint8),
+    ("command", ctypes.c_uint8),
+    ("size", ctypes.c_uint32),
+    ("data", ctypes.POINTER(_I2cSmbusData)),
+  ]
+
+
+class SMBus:
+  def __init__(self, bus: int):
+    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
+
+  def __enter__(self) -> 'SMBus':
+    return self
+
+  def __exit__(self, *args) -> None:
+    self.close()
+
+  def close(self) -> None:
+    if hasattr(self, '_fd') and self._fd >= 0:
+      os.close(self._fd)
+      self._fd = -1
+
+  def _set_address(self, addr: int, force: bool = False) -> None:
+    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
+    fcntl.ioctl(self._fd, ioctl_arg, addr)
+
+  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
+    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
+    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
+
+  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
+    return int(data.byte)
+
+  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    data.byte = value & 0xFF
+    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
+
+  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
+    self._set_address(addr, force)
+    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
+      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
+
+    data = _I2cSmbusData()
+    data.block[0] = length
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
+    read_len = int(data.block[0]) or length
+    read_len = min(read_len, length)
+    return [int(b) for b in data.block[1 : read_len + 1]]

common/mat.h

@@ -1,85 +0,0 @@
-#pragma once
-
-typedef struct vec3 {
-  float v[3];
-} vec3;
-
-typedef struct vec4 {
-  float v[4];
-} vec4;
-
-typedef struct mat3 {
-  float v[3*3];
-} mat3;
-
-typedef struct mat4 {
-  float v[4*4];
-} mat4;
-
-static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
-  mat3 ret = {{0.0}};
-  for (int r=0; r<3; r++) {
-    for (int c=0; c<3; c++) {
-      float v = 0.0;
-      for (int k=0; k<3; k++) {
-        v += a.v[r*3+k] * b.v[k*3+c];
-      }
-      ret.v[r*3+c] = v;
-    }
-  }
-  return ret;
-}
-
-static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
-  vec3 ret = {{0.0}};
-  for (int r=0; r<3; r++) {
-    for (int c=0; c<3; c++) {
-      ret.v[r] += a.v[r*3+c] * b.v[c];
-    }
-  }
-  return ret;
-}
-
-static inline mat4 matmul(const mat4 &a, const mat4 &b) {
-  mat4 ret = {{0.0}};
-  for (int r=0; r<4; r++) {
-    for (int c=0; c<4; c++) {
-      float v = 0.0;
-      for (int k=0; k<4; k++) {
-        v += a.v[r*4+k] * b.v[k*4+c];
-      }
-      ret.v[r*4+c] = v;
-    }
-  }
-  return ret;
-}
-
-static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
-  vec4 ret = {{0.0}};
-  for (int r=0; r<4; r++) {
-    for (int c=0; c<4; c++) {
-      ret.v[r] += a.v[r*4+c] * b.v[c];
-    }
-  }
-  return ret;
-}
-
-// scales the input and output space of a transformation matrix
-// that assumes pixel-center origin.
-static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
-  // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
-
-  mat3 transform_out = {{
-    1.0f/s, 0.0f, 0.5f,
-    0.0f, 1.0f/s, 0.5f,
-    0.0f, 0.0f, 1.0f,
-  }};
-
-  mat3 transform_in = {{
-    s,  0.0f, -0.5f*s,
-    0.0f, s, -0.5f*s,
-    0.0f, 0.0f, 1.0f,
-  }};
-
-  return matmul3(transform_in, matmul3(in, transform_out));
-}

common/model.h

@@ -1 +0,0 @@
-#define DEFAULT_MODEL "Steam Powered (Default)"

common/parameterized.py

@@ -0,0 +1,54 @@
+import re
+import sys
+import pytest
+import inspect
+
+
+def _to_safe_name(s):
+  return re.sub(r"[^a-zA-Z0-9_]+", "_", str(s)).strip("_")
+
+
+class parameterized:
+  @staticmethod
+  def expand(cases):
+    cases = list(cases)
+
+    if not cases:
+      return lambda func: pytest.mark.skip("no parameterized cases")(func)
+
+    def decorator(func):
+      params = [p for p in inspect.signature(func).parameters if p != 'self']
+      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
+      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
+      expand_params = params[: len(normalized[0])]
+      if len(expand_params) == 1:
+        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
+      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
+
+    return decorator
+
+
+def parameterized_class(attrs, input_list=None):
+  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
+    params_list = attrs
+  else:
+    assert input_list is not None
+    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
+    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
+
+  def decorator(cls):
+    globs = sys._getframe(1).f_globals
+    for i, params in enumerate(params_list):
+      # append sanitized string param values so pytest -k can filter by them
+      suffix = "_".join(filter(None, (_to_safe_name(v) for v in params.values() if isinstance(v, str))))
+      name = f"{cls.__name__}_{i}" + (f"_{suffix}" if suffix else "")
+      new_cls = type(name, (cls,), dict(params))
+      new_cls.__module__ = cls.__module__
+      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
+      globs[name] = new_cls
+    # Don't collect the un-parametrised base, but return it so outer decorators
+    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
+    cls.__test__ = False
+    return cls
+
+  return decorator

common/params.py

@@ -14,6 +14,6 @@ if __name__ == "__main__":
   if len(sys.argv) == 3:
     val = sys.argv[2]
     print(f"SET: {key} = {val}")
-    params.put(key, val)
+    params.put(key, val, block=True)
   elif len(sys.argv) == 2:
     print(f"GET: {key} = {params.get(key)}")

common/params_keys.h

@@ -66,11 +66,12 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
     {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "main_en"}},
+    {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
     {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
     {"LastGPSPosition", {PERSISTENT, STRING}},
     {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
+    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
     {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
@@ -79,8 +80,10 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
     {"LiveParameters", {PERSISTENT, JSON}},
     {"LiveParametersV2", {PERSISTENT, BYTES}},
+    {"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
     {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
     {"LocationFilterInitialState", {PERSISTENT, BYTES}},
+    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
     {"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
@@ -94,25 +97,25 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
     {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_StorageMissing", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
     {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
     {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
     {"PrimeType", {PERSISTENT, INT}},
     {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
     {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RecordFront", {PERSISTENT | BACKUP, BOOL}},
     {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
     {"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
+    {"ShowDebugInfo", {PERSISTENT, BOOL}},
     {"RouteCount", {PERSISTENT, INT, "0"}},
     {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
     {"TermsVersion", {PERSISTENT, STRING}},
+    {"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TrainingVersion", {PERSISTENT, STRING}},
     {"UbloxAvailable", {PERSISTENT, BOOL}},
     {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
@@ -128,34 +131,66 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
     {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //
     {"ApiCache_DriveStats", {PERSISTENT, JSON}},
     {"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
+    {"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}},  // seconds
     {"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}},  // MPH or km/h
     {"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
     {"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
+    {"CarList", {PERSISTENT, JSON}},
     {"CarParamsSP", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
     {"CarParamsSPCache", {CLEAR_ON_MANAGER_START, BYTES}},
     {"CarParamsSPPersistent", {PERSISTENT, BYTES}},
     {"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
     {"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
+    {"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
     {"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
     {"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
     {"DeviceBootMode", {PERSISTENT | BACKUP, INT, "0"}},
+    {"DevUIInfo", {PERSISTENT | BACKUP, INT, "0"}},
+    {"EnableCopyparty", {PERSISTENT | BACKUP, BOOL}},
     {"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
+    {"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
+    {"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
+    {"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
     {"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
     {"IsDevelopmentBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsReleaseSpBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"LastGPSPositionLLK", {PERSISTENT, STRING}},
+    {"LeadDepartAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}},
     {"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}},
     {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
+    {"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
+    {"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
+    {"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
+    {"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
+    {"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RoadEdgeLaneChangeEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // toyota specific params
+    {"ToyotaAutoHold", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaEnhancedBsm", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaTSS2Long", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaDriveMode", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     // MADS params
     {"Mads", {PERSISTENT | BACKUP, BOOL, "1"}},
@@ -166,7 +201,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     // Model Manager params
     {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
     {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT, "0"}},
+    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
+    {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
     {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
     {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
 
@@ -176,11 +212,13 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     // sunnylink params
     {"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
     {"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
+    {"ParamsVersion", {PERSISTENT, INT}},
     {"SunnylinkCache_Roles", {PERSISTENT, STRING}},
     {"SunnylinkCache_Users", {PERSISTENT, STRING}},
     {"SunnylinkDongleId", {PERSISTENT, STRING}},
     {"SunnylinkdPid", {PERSISTENT, INT}},
-    {"SunnylinkEnabled", {PERSISTENT, BOOL}},
+    {"SunnylinkEnabled", {PERSISTENT, BOOL, "1"}},
+    {"SunnylinkTempFault", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL, "0"}},
 
     // Backup Manager params
     {"BackupManager_CreateBackup", {PERSISTENT, BOOL}},
@@ -188,14 +226,27 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
 
     // sunnypilot car specific params
     {"HyundaiLongitudinalTuning", {PERSISTENT | BACKUP, INT, "0"}},
+    {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     {"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"AccelPersonalityEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"AccelPersonality", {PERSISTENT | BACKUP, INT, "1"}},
+    {"RadarDistance", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"SmoothBraking", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
 
-    // model panel params
+    // sunnypilot model params
+    {"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
     {"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
     {"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
+    {"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
+    {"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
 
     // mapd
     {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
@@ -216,4 +267,26 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OsmStateTitle", {PERSISTENT, STRING}},
     {"OsmWayTest", {PERSISTENT, STRING}},
     {"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // Speed Limit
+    {"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
+    {"SpeedLimitOffsetType", {PERSISTENT | BACKUP, INT, "0"}},
+    {"SpeedLimitPolicy", {PERSISTENT | BACKUP, INT, "3"}},
+    {"SpeedLimitValueOffset", {PERSISTENT | BACKUP, INT, "0"}},
+
+    // Smart Cruise Control
+    {"MapTargetVelocities", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"SmartCruiseControlMap", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"SmartCruiseControlVision", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // Torque lateral control custom params
+    {"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
+    {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
+    {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
+    {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
+    {"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
+    {"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
+    {"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
 };

common/params_pyx.pyx

@@ -142,33 +142,28 @@ cdef class Params:
     cdef ParamKeyType t = self.p.getKeyType(k)
     return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
 
-  def put(self, key, dat):
+  def put(self, key, dat, bool block = False):
     """
-    Warning: This function blocks until the param is written to disk!
+    Warning: block=True blocks until the param is written to disk!
     In very rare cases this can take over a second, and your code will hang.
-    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
-    in general try to avoid writing params as much as possible.
+    Use block=False in time sensitive code, but in general try to avoid
+    writing params as much as possible.
     """
     cdef string k = self.check_key(key)
     cdef string dat_bytes = self._put_cast(key, dat)
     with nogil:
-      self.p.put(k, dat_bytes)
+      if block:
+        self.p.put(k, dat_bytes)
+      else:
+        self.p.putNonBlocking(k, dat_bytes)
 
-  def put_bool(self, key, bool val):
+  def put_bool(self, key, bool val, bool block = False):
     cdef string k = self.check_key(key)
     with nogil:
-      self.p.putBool(k, val)
-
-  def put_nonblocking(self, key, dat):
-    cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
-    with nogil:
-      self.p.putNonBlocking(k, dat_bytes)
-
-  def put_bool_nonblocking(self, key, bool val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putBoolNonBlocking(k, val)
+      if block:
+        self.p.putBool(k, val)
+      else:
+        self.p.putBoolNonBlocking(k, val)
 
   def remove(self, key):
     cdef string k = self.check_key(key)

common/pid.py

@@ -2,21 +2,14 @@ import numpy as np
 from numbers import Number
 
 class PIDController:
-  def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
-    self._k_p = k_p
-    self._k_i = k_i
-    self._k_d = k_d
-    self.k_f = k_f   # feedforward gain
-    if isinstance(self._k_p, Number):
-      self._k_p = [[0], [self._k_p]]
-    if isinstance(self._k_i, Number):
-      self._k_i = [[0], [self._k_i]]
-    if isinstance(self._k_d, Number):
-      self._k_d = [[0], [self._k_d]]
+  def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
+    self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
+    self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
 
     self.set_limits(pos_limit, neg_limit)
 
-    self.i_rate = 1.0 / rate
+    self.i_dt = 1.0 / rate
     self.speed = 0.0
 
     self.reset()
@@ -46,12 +39,12 @@ class PIDController:
 
   def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
     self.speed = speed
-    self.p = float(error) * self.k_p
-    self.f = feedforward * self.k_f
-    self.d = error_rate * self.k_d
+    self.p = self.k_p * float(error)
+    self.d = self.k_d * error_rate
+    self.f = feedforward
 
     if not freeze_integrator:
-      i = self.i + error * self.k_i * self.i_rate
+      i = self.i + self.k_i * self.i_dt * error
 
       # Don't allow windup if already clipping
       test_control = self.p + i + self.d + self.f

common/prefix.h

@@ -13,7 +13,11 @@ public:
     if (prefix.empty()) {
       prefix = util::random_string(15);
     }
-    msgq_path = Path::shm_path() + "/" + prefix;
+#ifdef __APPLE__
+    msgq_path = "/tmp/msgq_" + prefix;
+#else
+    msgq_path = "/dev/shm/msgq_" + prefix;
+#endif
     bool ret = util::create_directories(msgq_path, 0777);
     assert(ret);
     setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
@@ -23,14 +27,14 @@ public:
     auto param_path = Params().getParamPath();
     if (util::file_exists(param_path)) {
       std::string real_path = util::readlink(param_path);
-      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
+      util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
       unlink(param_path.c_str());
     }
     if (getenv("COMMA_CACHE") == nullptr) {
-      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
+      util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
     }
-    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
-    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
+    util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
+    util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
     unsetenv("OPENPILOT_PREFIX");
   }
 

common/prefix.py

@@ -1,4 +1,5 @@
 import os
+import platform
 import shutil
 import uuid
 
@@ -9,9 +10,10 @@ from openpilot.system.hardware.hw import Paths
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
-  def __init__(self, prefix: str = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
     self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
+    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
+    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
     self.create_dirs_on_enter = create_dirs_on_enter
     self.clean_dirs_on_exit = clean_dirs_on_exit
     self.shared_download_cache = shared_download_cache

common/ratekeeper.cc

@@ -6,9 +6,9 @@
 #include "common/timing.h"
 #include "common/util.h"
 
-RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
-    : name(name),
-      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
+RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
+    : name(name_),
+      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
   interval = 1 / rate;
   last_monitor_time = seconds_since_boot();
   next_frame_time = last_monitor_time + interval;

common/realtime.py

@@ -6,7 +6,7 @@ import time
 
 from setproctitle import getproctitle
 
-from openpilot.common.util import MovingAverage
+from openpilot.common.utils import MovingAverage
 from openpilot.system.hardware import PC
 
 
@@ -28,6 +28,11 @@ class Priority:
   CTRL_HIGH = 53
 
 
+def drop_realtime() -> None:
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
+
+
 def set_core_affinity(cores: list[int]) -> None:
   if sys.platform == 'linux' and not PC:
     os.sched_setaffinity(0, cores)

common/retry.py

@@ -1,30 +0,0 @@
-import time
-import functools
-
-from openpilot.common.swaglog import cloudlog
-
-
-def retry(attempts=3, delay=1.0, ignore_failure=False):
-  def decorator(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-      for _ in range(attempts):
-        try:
-          return func(*args, **kwargs)
-        except Exception:
-          cloudlog.exception(f"{func.__name__} failed, trying again")
-          time.sleep(delay)
-
-      if ignore_failure:
-        cloudlog.error(f"{func.__name__} failed after retry")
-      else:
-        raise Exception(f"{func.__name__} failed after retry")
-    return wrapper
-  return decorator
-
-
-if __name__ == "__main__":
-  @retry(attempts=10)
-  def abc():
-    raise ValueError("abc failed :(")
-  abc()

common/run.py

@@ -1,28 +0,0 @@
-import subprocess
-from contextlib import contextmanager
-from subprocess import Popen, PIPE, TimeoutExpired
-
-
-def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
-  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
-
-
-def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
-  try:
-    return run_cmd(cmd, cwd=cwd, env=env)
-  except subprocess.CalledProcessError:
-    return default
-
-
-@contextmanager
-def managed_proc(cmd: list[str], env: dict[str, str]):
-  proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE)
-  try:
-    yield proc
-  finally:
-    if proc.poll() is None:
-      proc.terminate()
-    try:
-      proc.wait(timeout=5)
-    except TimeoutExpired:
-      proc.kill()

common/swaglog.cc

@@ -11,10 +11,12 @@
 
 #include <zmq.h>
 #include <stdarg.h>
-#include "third_party/json11/json11.hpp"
+#include "json11/json11.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"
 
+#include "sunnypilot/common/version.h"
+
 class SwaglogState {
 public:
   SwaglogState() {
@@ -56,7 +58,7 @@ public:
     if (char* daemon_name = getenv("MANAGER_DAEMON")) {
       ctx_j["daemon"] = daemon_name;
     }
-    ctx_j["version"] = COMMA_VERSION;
+    ctx_j["version"] = SUNNYPILOT_VERSION;
     ctx_j["dirty"] = !getenv("CLEAN");
     ctx_j["device"] = Hardware::get_name();
   }

common/tests/test_file_helpers.py

@@ -1,7 +1,7 @@
 import os
 from uuid import uuid4
 
-from openpilot.common.file_helpers import atomic_write_in_dir
+from openpilot.common.utils import atomic_write
 
 
 class TestFileHelpers:
@@ -15,5 +15,5 @@ class TestFileHelpers:
       assert f.read() == "test"
     os.remove(path)
 
-  def test_atomic_write_in_dir(self):
-    self.run_atomic_write_func(atomic_write_in_dir)
+  def test_atomic_write(self):
+    self.run_atomic_write_func(atomic_write)

common/tests/test_markdown.py

@@ -6,7 +6,7 @@ from openpilot.common.markdown import parse_markdown
 
 class TestMarkdown:
   def test_all_release_notes(self):
-    with open(os.path.join(BASEDIR, "RELEASES.md")) as f:
+    with open(os.path.join(BASEDIR, "CHANGELOG.md")) as f:
       release_notes = f.read().split("\n\n")
       assert len(release_notes) > 10
 

common/tests/test_params.py

@@ -12,17 +12,17 @@ class TestParams:
     self.params = Params()
 
   def test_params_put_and_get(self):
-    self.params.put("DongleId", "cb38263377b873ee")
+    self.params.put("DongleId", "cb38263377b873ee", block=True)
     assert self.params.get("DongleId") == "cb38263377b873ee"
 
   def test_params_non_ascii(self):
     st = b"\xe1\x90\xff"
-    self.params.put("CarParams", st)
+    self.params.put("CarParams", st, block=True)
     assert self.params.get("CarParams") == st
 
   def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", b"test")
-    self.params.put("DongleId", "cb38263377b873ee")
+    self.params.put("CarParams", b"test", block=True)
+    self.params.put("DongleId", "cb38263377b873ee", block=True)
     assert self.params.get("CarParams") == b"test"
 
     undefined_param = self.params.get_param_path(uuid.uuid4().hex)
@@ -36,15 +36,15 @@ class TestParams:
     assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
-    self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", 123)
+    self.params.put("DongleId", "bob", block=True)
+    self.params.put("AthenadPid", 123, block=True)
     assert self.params.get("DongleId") == "bob"
     assert self.params.get("AthenadPid") == 123
 
   def test_params_get_block(self):
     def _delayed_writer():
       time.sleep(0.1)
-      self.params.put("CarParams", b"test")
+      self.params.put("CarParams", b"test", block=True)
     threading.Thread(target=_delayed_writer).start()
     assert self.params.get("CarParams") is None
     assert self.params.get("CarParams", block=True) == b"test"
@@ -57,10 +57,10 @@ class TestParams:
       self.params.get_bool("swag")
 
     with pytest.raises(UnknownKeyName):
-      self.params.put("swag", "abc")
+      self.params.put("swag", "abc", block=True)
 
     with pytest.raises(UnknownKeyName):
-      self.params.put_bool("swag", True)
+      self.params.put_bool("swag", True, block=True)
 
   def test_remove_not_there(self):
     assert self.params.get("CarParams") is None
@@ -71,23 +71,23 @@ class TestParams:
     self.params.remove("IsMetric")
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put_bool("IsMetric", True)
+    self.params.put_bool("IsMetric", True, block=True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put_bool("IsMetric", False)
+    self.params.put_bool("IsMetric", False, block=True)
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", True)
+    self.params.put("IsMetric", True, block=True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", False)
+    self.params.put("IsMetric", False, block=True)
     assert not self.params.get_bool("IsMetric")
 
   def test_put_non_blocking_with_get_block(self):
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_nonblocking("CarParams", b"test")
+      Params().put("CarParams", b"test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
@@ -96,7 +96,7 @@ class TestParams:
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_bool_nonblocking("CarParams", True)
+      Params().put_bool("CarParams", True)
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"1"
@@ -123,19 +123,19 @@ class TestParams:
 
   def test_params_get_type(self):
     # json
-    self.params.put("ApiCache_FirehoseStats", {"a": 0})
+    self.params.put("ApiCache_FirehoseStats", {"a": 0}, block=True)
     assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
 
     # int
-    self.params.put("BootCount", 1441)
+    self.params.put("BootCount", 1441, block=True)
     assert self.params.get("BootCount") == 1441
 
     # bool
-    self.params.put("AdbEnabled", True)
+    self.params.put("AdbEnabled", True, block=True)
     assert self.params.get("AdbEnabled")
     assert isinstance(self.params.get("AdbEnabled"), bool)
 
     # time
     now = datetime.datetime.now(datetime.UTC)
-    self.params.put("InstallDate", now)
+    self.params.put("InstallDate", now, block=True)
     assert self.params.get("InstallDate") == now

common/tests/test_swaglog.cc

@@ -7,7 +7,9 @@
 #include "common/util.h"
 #include "common/version.h"
 #include "system/hardware/hw.h"
-#include "third_party/json11/json11.hpp"
+#include "json11/json11.hpp"
+
+#include "sunnypilot/common/version.h"
 
 std::string daemon_name = "testy";
 std::string dongle_id = "test_dongle_id";
@@ -53,7 +55,7 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {
     REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
     REQUIRE(ctx["dirty"].bool_value() == true);
 
-    REQUIRE(ctx["version"].string_value() == COMMA_VERSION);
+    REQUIRE(ctx["version"].string_value() == SUNNYPILOT_VERSION);
 
     std::string device = Hardware::get_name();
     REQUIRE(ctx["device"].string_value() == device);

common/tests/test_util.cc

@@ -36,7 +36,7 @@ TEST_CASE("util::read_file") {
     REQUIRE(util::read_file(filename).empty());
 
     std::string content = random_bytes(64 * 1024);
-    write(fd, content.c_str(), content.size());
+    REQUIRE(write(fd, content.c_str(), content.size()) == (ssize_t)content.size());
     std::string ret = util::read_file(filename);
     bool equal = (ret == content);
     REQUIRE(equal);
@@ -114,12 +114,12 @@ TEST_CASE("util::safe_fwrite") {
 }
 
 TEST_CASE("util::create_directories") {
-  system("rm /tmp/test_create_directories -rf");
+  REQUIRE(system("rm /tmp/test_create_directories -rf") == 0);
   std::string dir = "/tmp/test_create_directories/a/b/c/d/e/f";
 
-  auto check_dir_permissions = [](const std::string &dir, mode_t mode) -> bool {
+  auto check_dir_permissions = [](const std::string &path, mode_t mode) -> bool {
     struct stat st = {};
-    return stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
+    return stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
   };
 
   SECTION("create_directories") {
@@ -132,7 +132,7 @@ TEST_CASE("util::create_directories") {
   }
   SECTION("a file exists with the same name") {
     REQUIRE(util::create_directories(dir, 0755));
-    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT);
+    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT, 0644);
     REQUIRE(f != -1);
     close(f);
     REQUIRE(util::create_directories(dir + "/file", 0755) == false);

common/time_helpers.py

@@ -2,6 +2,7 @@ import datetime
 from pathlib import Path
 
 MIN_DATE = datetime.datetime(year=2025, month=2, day=21)
+MAX_DATE = datetime.datetime(year=2035, month=1, day=1)
 
 def min_date():
   # on systemd systems, the default time is the systemd build time
@@ -12,4 +13,4 @@ def min_date():
   return MIN_DATE
 
 def system_time_valid():
-  return datetime.datetime.now() > min_date()
+  return min_date() < datetime.datetime.now() < MAX_DATE

common/transformations/.gitignore

@@ -1,2 +0,0 @@
-transformations
-transformations.cpp

common/transformations/SConscript

@@ -1,5 +0,0 @@
-Import('env', 'envCython')
-
-transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc'])
-transformations_python = envCython.Program('transformations.so', 'transformations.pyx')
-Export('transformations', 'transformations_python')

common/transformations/tests/test_coordinates.py

@@ -102,3 +102,36 @@ class TestNED:
     np.testing.assert_allclose(converter.ned2ecef(ned_offsets_batch),
                                                            ecef_positions_offset_batch,
                                                            rtol=1e-9, atol=1e-7)
+
+  def test_errors(self):
+    # Test wrong shape/type for geodetic2ecef
+    # numpy_wrap raises IndexError for scalar input
+    with np.testing.assert_raises(IndexError):
+      coord.geodetic2ecef(1.0)
+
+    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
+      coord.geodetic2ecef([0, 0])
+
+    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
+      coord.geodetic2ecef([0, 0, 0, 0])
+
+    with np.testing.assert_raises(TypeError):
+      coord.geodetic2ecef(['a', 'b', 'c'])
+
+    # Test LocalCoord constructor errors
+    with np.testing.assert_raises(ValueError):
+      coord.LocalCoord.from_geodetic([0, 0])
+
+    with np.testing.assert_raises(ValueError):
+      coord.LocalCoord.from_geodetic(1)
+
+    with np.testing.assert_raises(TypeError):
+      coord.LocalCoord.from_geodetic(['a', 'b', 'c'])
+
+    # Test wrong shape/type for ecef2geodetic
+    with np.testing.assert_raises(ValueError):
+      coord.ecef2geodetic([1, 2])
+    with np.testing.assert_raises(ValueError):
+      coord.ecef2geodetic([1, 2, 3, 4])
+    with np.testing.assert_raises(IndexError):
+      coord.ecef2geodetic(1.0)