Merge branch 'master-new' into dynamic-outputs-toggle

.clang-tidy

@@ -0,0 +1,19 @@
+---
+Checks: '
+bugprone-*,
+-bugprone-integer-division,
+-bugprone-narrowing-conversions,
+performance-*,
+clang-analyzer-*,
+misc-*,
+-misc-unused-parameters,
+modernize-*,
+-modernize-avoid-c-arrays,
+-modernize-deprecated-headers,
+-modernize-use-auto,
+-modernize-use-using,
+-modernize-use-nullptr,
+-modernize-use-trailing-return-type,
+'
+CheckOptions:
+...

.codespellignore

@@ -3,4 +3,3 @@ REGIST
 PullRequest
 cancelled
 FOF
-NoO

.dockerignore

@@ -13,6 +13,27 @@
 *.o-*
 *.os
 *.os-*
+*.so
+*.a
 
 venv/
 .venv/
+
+notebooks
+phone
+massivemap
+neos
+installer
+chffr/app2
+chffr/backend/env
+selfdrive/nav
+selfdrive/baseui
+selfdrive/test/simulator2
+**/cache_data
+xx/plus
+xx/community
+xx/projects
+!xx/projects/eon_testing_master
+!xx/projects/map3d
+xx/ops
+xx/junk

.gitattributes

@@ -7,12 +7,10 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
-*.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text

.github/labeler.yaml

@@ -1,6 +1,6 @@
 ci:
   - changed-files:
-    - any-glob-to-all-files: "{.github/**,**/test_*,**/test/**,Jenkinsfile}"
+    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
 
 chore:
   - changed-files:
@@ -16,7 +16,7 @@ simulation:
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
+    - any-glob-to-all-files: '{selfdrive/ui/**,system/ui/**}'
 
 tools:
   - changed-files:

.github/workflows/auto-cache/action.yaml

@@ -0,0 +1,58 @@
+name: 'automatically cache based on current runner'
+
+inputs:
+  path:
+    description: 'path to cache'
+    required: true
+  key:
+    description: 'key'
+    required: true
+  restore-keys:
+    description: 'restore-keys'
+    required: true
+  save:
+    description: 'whether to save the cache'
+    default: 'true'
+    required: false
+outputs:
+  cache-hit:
+    description: 'cache hit occurred'
+    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
+               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
+               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: setup namespace cache
+      id: ns-cache
+      if: ${{ contains(runner.name, 'nsc') }}
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        path: ${{ inputs.path }}
+
+    - name: setup github cache
+      id: gha-cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
+      uses: 'actions/cache@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: setup github cache
+      id: gha-cache-ro
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
+      uses: 'actions/cache/restore@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.path }}
+        sudo chmod -R 777 ${{ inputs.path }}
+        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -1,7 +1,7 @@
 name: "PR review"
 on:
   pull_request_target:
-    types: [ opened, reopened, synchronize, edited ]
+    types: [opened, reopened, synchronize, edited]
 
 jobs:
   labeler:
@@ -12,12 +12,12 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v6
+    - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -29,21 +29,21 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
-        target: /^(?!master$).*/
+        target: /^(?!master-new$).*/
         exclude: /sunnypilot:.*/
         change-to: ${{ github.base_ref }}
         already-exists-action: close_this
-        already-exists-comment: "Your PR should be made against the `master` branch"
+        already-exists-comment: "Your PR should be made against the `master-new` branch"
 
   update-pr-labels:
     name: Update fork's PR Labels
     runs-on: ubuntu-latest
     if: (github.event.pull_request.head.repo.fork && (contains(github.event_name, 'pull_request') && github.event.action == 'synchronize'))
-    env:
-      PR_LABEL: 'dev'
+    env: 
+      PR_LABEL: 'dev-c3'
       TRUST_FORK_PR_LABEL: 'trust-fork-pr'
     steps:
-      - name: Check if PR has dev label
+      - name: Check if PR has dev-c3 label
         id: check-labels
         uses: actions/github-script@v7
         with:
@@ -55,33 +55,33 @@ jobs:
               repo: context.repo.repo,
               issue_number: prNumber
             });
-
+            
             const hasDevC3Label = labels.some(label => label.name === process.env.PR_LABEL);
             const hasTrustLabel = labels.some(label => label.name === process.env.TRUST_FORK_PR_LABEL);
-
+            
             console.log(`PR #${prNumber} has ${process.env.PR_LABEL} label: ${hasDevC3Label}`);
             console.log(`PR #${prNumber} has ${process.env.TRUST_FORK_PR_LABEL} label: ${hasTrustLabel}`);
-
-            core.setOutput('has-dev', hasDevC3Label ? 'true' : 'false');
+            
+            core.setOutput('has-dev-c3', hasDevC3Label ? 'true' : 'false');
             core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
-
+      
       - name: Remove trust-fork-pr label if present
-        if: steps.check-labels.outputs.has-dev == 'true' && steps.check-labels.outputs.has-trust == 'true'
+        if: steps.check-labels.outputs.has-dev-c3 == 'true' && steps.check-labels.outputs.has-trust == 'true'
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const prNumber = context.payload.pull_request.number;
-
+            
             await github.rest.issues.removeLabel({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: prNumber,
               name: process.env.TRUST_FORK_PR_LABEL
             });
-
+            
             console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
-
+            
             // Add a comment to the PR
             await github.rest.issues.createComment({
               owner: context.repo.owner,

.github/workflows/badges.yaml

@@ -5,7 +5,9 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: openpilot-base
+  DOCKER_REGISTRY: ghcr.io/commaai
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
   badges:
@@ -15,13 +17,13 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
-        python3 selfdrive/ui/translations/create_badges.py
+        ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py"
 
         rm .gitattributes
 

.github/workflows/build-all-tinygrad-models.yaml

@@ -1,48 +1,29 @@
-name: Build and push all tinygrad models
+name: Build All Tinygrad Models and Push to GitLab
 
 on:
   workflow_dispatch:
     inputs:
-      set_min_version:
-        description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
-        required: true
+      branch:
+        description: 'Branch to run workflow from'
+        required: false
+        default: 'master-new'
         type: string
 
 jobs:
   setup:
     runs-on: ubuntu-latest
     outputs:
-      json_version: ${{ steps.get-json.outputs.json_version }}
-      recompiled_dir: ${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
       json_file: ${{ steps.get-json.outputs.json_file }}
-      model_matrix: ${{ steps.set-matrix.outputs.model_matrix }}
-      tinygrad_ref: ${{ steps.get-tinygrad-ref.outputs.tinygrad_ref }}
     steps:
-      - name: Checkout sunnypilot repo
+      - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot
-          path: sunnypilot
-          submodules: recursive
-
-      - name: Get tinygrad_repo ref
-        id: get-tinygrad-ref
-        run: |
-          cd sunnypilot
-          export PYTHONPATH=$(pwd)
-          ref=$(python3 sunnypilot/models/tinygrad_ref.py)
-          echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
-          echo "tinygrad_ref is $ref"
-
-      - name: Checkout docs repo (sunnypilot-models, gh-pages)
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
 
-      - name: Get next JSON version to use (from GitHub docs repo)
+      - name: Get next JSON version to use
         id: get-json
         run: |
           cd docs/docs
@@ -50,132 +31,19 @@ jobs:
           next=$((latest+1))
           json_file="driving_models_v${next}.json"
           cp "driving_models_v${latest}.json" "$json_file"
-          echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
-          echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
-          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
+          echo "json_file=$json_file" >> $GITHUB_OUTPUT
 
-      - name: Extract tinygrad models
-        id: set-matrix
-        working-directory: docs/docs
-        run: |
-          jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
-          echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
-
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
+      - name: Upload context for next jobs
+        uses: actions/upload-artifact@v4
         with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-      - run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+          name: context
+          path: docs
 
-      - name: Clone GitLab docs repo and create new recompiled dir
-        id: create-recompiled-dir
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          git checkout main
-          git sparse-checkout set --no-cone models/
-          cd models
-          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          if [[ -z "$latest_dir" ]]; then
-            next_dir=1
-          else
-            next_dir=$((latest_dir+1))
-          fi
-          recompiled_dir="${next_dir}"
-          mkdir -p "recompiled${recompiled_dir}"
-          touch "recompiled${recompiled_dir}/.gitkeep"
-          cd ../..
-          echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
-
-      - name: Push empty recompiled dir to GitLab
-        run: |
-          cd gitlab_docs
-          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
-          git push origin main
-
-      - name: Push new JSON to GitHub docs repo
-        run: |
-          cd docs
-          git pull origin gh-pages
-          git add docs/"$(basename ${{ steps.get-json.outputs.json_file }})"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add new ${{ steps.get-json.outputs.json_file }} for build-all" || echo "No changes to commit"
-          git push origin gh-pages
-
-  get_and_build:
-    needs: [setup]
-    strategy:
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/build-single-tinygrad-model.yaml
-    with:
-      upstream_branch: ${{ matrix.model.ref }}
-      custom_name: ${{ matrix.model.display_name }}
-      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
-      json_version: ${{ needs.setup.outputs.json_version }}
-    secrets: inherit
-
-  retry_failed_models:
-    needs: [setup, get_and_build]
+  build-all:
     runs-on: ubuntu-latest
-    if: ${{ needs.setup.result != 'failure' && !cancelled() }}
-    outputs:
-      retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: model-*
-          path: output
-
-      - id: set-retry-matrix
-        run: |
-          echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
-          built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
-          )
-          jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
-            'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
-          echo "retry_matrix=$(cat retry_matrix.json)" >> $GITHUB_OUTPUT
-
-  retry_get_and_build:
-    needs: [setup, get_and_build, retry_failed_models]
-    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
-    strategy:
-      matrix:
-        model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/build-single-tinygrad-model.yaml
-    with:
-      upstream_branch: ${{ matrix.model.ref }}
-      custom_name: ${{ matrix.model.display_name }}
-      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
-      json_version: ${{ needs.setup.outputs.json_version }}
-      artifact_suffix: -retry
-    secrets: inherit
-
-  publish_models:
-    name: Publish models sequentially
-    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
-    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      max-parallel: 1
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
+    needs: setup
     env:
-      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
-      JSON_FILE: ${{ needs.setup.outputs.json_file }}
-      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
+      JSON_FILE: docs/docs/${{ needs.setup.outputs.json_file }}
     steps:
       - name: Set up SSH
         uses: webfactory/ssh-agent@v0.9.0
@@ -192,75 +60,140 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
           cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
           git checkout main
           cd ..
 
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-models
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Validate recompiled dir and JSON version
+      - name: Set next recompiled dir
+        id: set-recompiled
         run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
-          if [ ! -f "$JSON_FILE" ]; then
-            echo "JSON file $JSON_FILE does not exist!"
-            exit 1
+          cd gitlab_docs/models
+          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
+          if [[ -z "$latest_dir" ]]; then
+            next_dir=1
+          else
+            next_dir=$((latest_dir+1))
           fi
+          recompiled_dir="recompiled${next_dir}"
+          mkdir -p "$recompiled_dir"
+          echo "RECOMPILED_DIR=$recompiled_dir" >> $GITHUB_ENV
 
-      - name: Download artifact name file
+      - name: Download context
         uses: actions/download-artifact@v4
         with:
-          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
-          path: artifact_name
+          name: context
+          path: .
 
-      - name: Read artifact name
-        id: read-artifact-name
+      - name: Install dependencies
         run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+          sudo apt-get update
+          sudo apt-get install -y jq gh
 
-      - name: Download model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
-        run: |
-          find output -type f -name '*.onnx' -delete
-          find output -type f -name 'big_*.pkl' -delete
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-
-      - name: Copy model artifacts to gitlab
+      - name: Build all tinygrad models
+        id: trigger-builds
         env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
+          set -e
+          > triggered_run_ids.txt
+          BRANCH="${{ github.event.inputs.branch }}"
+          jq -c '.bundles[] | select(.runner=="tinygrad")' "$JSON_FILE" | while read -r bundle; do
+            ref=$(echo "$bundle" | jq -r '.ref')
+            display_name=$(echo "$bundle" | jq -r '.display_name' | sed 's/ ([^)]*)//g')
+            is_20hz=$(echo "$bundle" | jq -r '.is_20hz')
+            echo "Triggering build for: $display_name ($ref) [20Hz: $is_20hz]"
+            START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+            gh workflow run sunnypilot-build-model.yaml \
+              --repo sunnypilot/sunnypilot \
+              --ref "$BRANCH" \
+              -f upstream_branch="$ref" \
+              -f custom_name="$display_name" \
+              -f is_20hz="$is_20hz"
+            for i in {1..24}; do
+              RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="$BRANCH" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
+              if [ -n "$RUN_ID" ]; then
+                break
+              fi
+              sleep 5
+            done
+            if [ -z "$RUN_ID" ]; then
+              echo "ould not find the triggered workflow run for $display_name ($ref)"
+              exit 1
             fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
+            echo "$RUN_ID" >> triggered_run_ids.txt
+          done
+
+      - name: Wait for all model builds to finish
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          SUCCESS_RUNS=()
+          FAILED_RUNS=()
+          declare -A RUN_ID_TO_NAME
+
+          while read -r RUN_ID; do
+            ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name' || echo "unknown")
+            RUN_ID_TO_NAME["$RUN_ID"]="$ARTIFACT_NAME"
+          done < triggered_run_ids.txt
+
+          while read -r RUN_ID; do
+            echo "Watching run ID: $RUN_ID"
+            gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
+            CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
+            ARTIFACT_NAME="${RUN_ID_TO_NAME[$RUN_ID]}"
+            echo "Run $RUN_ID ($ARTIFACT_NAME) concluded with: $CONCLUSION"
+            if [[ "$CONCLUSION" == "success" ]]; then
+              SUCCESS_RUNS+=("$RUN_ID")
             else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
+              FAILED_RUNS+=("$RUN_ID")
             fi
+          done < triggered_run_ids.txt
+
+          if [[ ${#SUCCESS_RUNS[@]} -eq 0 ]]; then
+            echo "All model builds failed. Aborting."
+            exit 1
+          fi
+
+          if [[ ${#FAILED_RUNS[@]} -gt 0 ]]; then
+            echo "WARNING: The following model builds failed:"
+            for RUN_ID in "${FAILED_RUNS[@]}"; do
+              echo "- $RUN_ID (${RUN_ID_TO_NAME[$RUN_ID]})"
+            done
+            echo "You may want to rerun these models manually."
+          fi
+
+      - name: Download and extract all model artifacts
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
+          SUCCESS_RUNS=()
+          while read -r RUN_ID; do
+            CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
+            if [[ "$CONCLUSION" == "success" ]]; then
+              SUCCESS_RUNS+=("$RUN_ID")
+            fi
+          done < triggered_run_ids.txt
+
+          for RUN_ID in "${SUCCESS_RUNS[@]}"; do
+            ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
+            echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
+            mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
+            echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
+            gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
+            echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
+            ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
+            if [ -n "$ZIP_PATH" ]; then
+              echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
+              unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
+              rm -f "$ZIP_PATH"
+              echo "Unzipped and removed $ZIP_PATH"
+            else
+              echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME (This is NOT an error)."
+            fi
+            echo "Done processing $ARTIFACT_NAME"
           done
 
       - name: Push recompiled dir to GitLab
@@ -269,36 +202,25 @@ jobs:
         run: |
           cd gitlab_docs
           git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
+          mkdir -p models/"$(basename $RECOMPILED_DIR)"
+          git add models/"$(basename $RECOMPILED_DIR)"
           git config --global user.name "GitHub Action"
           git config --global user.email "action@github.com"
-          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
+          git commit -m "Add $(basename $RECOMPILED_DIR) from build-all-tinygrad-models"
           git push origin main
-      - run: |
-          cd docs
-          git pull origin gh-pages
 
-      - name: update json
+      - name: Run json_parser.py to update JSON
         run: |
-          ARGS=""
-          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
-          ARGS="$ARGS --sort-by-date"
-          ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
-          eval python3 docs/json_parser.py \
+          python3 docs/json_parser.py \
             --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            $ARGS
+            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR"
 
-      - name: Push updated json to GitHub
+      - name: Push updated JSON to GitHub docs repo
         run: |
           cd docs
           git config --global user.name "GitHub Action"
           git config --global user.email "action@github.com"
           git checkout gh-pages
           git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
+          git commit -m "Update $(basename $JSON_FILE) after recompiling models" || echo "No changes to commit"
           git push origin gh-pages

.github/workflows/build-single-tinygrad-model.yaml

@@ -1,36 +1,13 @@
 name: Build Single Tinygrad Model and Push
 
 on:
-  workflow_call:
-    inputs:
-      upstream_branch:
-        description: 'Upstream commit to build from'
-        required: true
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
-        required: true
-        type: string
-      json_version:
-        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
-        required: true
-        type: string
-      artifact_suffix:
-        description: 'Suffix for artifact name'
-        required: false
-        type: string
-        default: ''
-      bypass_push:
-        description: 'Bypass pushing to GitLab for build-all'
-        required: false
-        default: true
-        type: boolean
   workflow_dispatch:
     inputs:
+      build_model_ref:
+        description: 'Branch to use for build-model workflow'
+        required: false
+        default: 'master-new'
+        type: string
       upstream_branch:
         description: 'Upstream commit to build from'
         required: true
@@ -42,19 +19,17 @@ on:
       recompiled_dir:
         description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
         required: true
-        type: string
+        type: number
       json_version:
         description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
         required: true
-        type: string
+        type: number
       model_folder:
         description: 'Model folder'
+        required: true
         type: choice
-        default: 'None'
         options:
-          - None
           - Simple Plan Models
-          - Space Lab Models
           - TR Models
           - DTR Models
           - Custom Merge Models
@@ -67,32 +42,18 @@ on:
       generation:
         description: 'Model generation'
         required: false
-        type: string
+        type: number
       version:
         description: 'Minimum selector version'
         required: false
-        type: string
-env:
-  RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
-  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
+        type: number
 
 jobs:
-  build_model:
-    uses: ./.github/workflows/sunnypilot-build-model.yaml
-    with:
-      upstream_branch: ${{ inputs.upstream_branch }}
-      custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
-      is_20hz: true
-      artifact_suffix: ${{ inputs.artifact_suffix }}
-    secrets: inherit
-
-  publish_model:
-    if: ${{ !inputs.bypass_push && !cancelled() }}
-    concurrency:
-      group: gitlab-push-${{ inputs.recompiled_dir }}
-      cancel-in-progress: false
-    needs: build_model
+  build-single:
     runs-on: ubuntu-latest
+    env:
+      RECOMPILED_DIR: recompiled${{ github.event.inputs.recompiled_dir }}
+      JSON_FILE: docs/docs/driving_models_v${{ github.event.inputs.json_version }}.json
     steps:
       - name: Set up SSH
         uses: webfactory/ssh-agent@v0.9.0
@@ -109,7 +70,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
@@ -119,7 +80,7 @@ jobs:
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -135,49 +96,66 @@ jobs:
             exit 1
           fi
 
-      - name: Download artifact name file
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: read-artifact-name
+      - name: Install dependencies
         run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+          sudo apt-get update
+          sudo apt-get install -y jq gh
+
+      - name: Build model
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          gh workflow run sunnypilot-build-model.yaml \
+            --repo sunnypilot/sunnypilot \
+            --ref "${{ github.event.inputs.build_model_ref }}" \
+            -f upstream_branch="${{ github.event.inputs.upstream_branch }}" \
+            -f custom_name="${{ github.event.inputs.custom_name }}"
+
+          for i in {1..24}; do
+            RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="${{ github.event.inputs.build_model_ref }}" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
+            if [ -n "$RUN_ID" ]; then
+              break
+            fi
+            sleep 5
+          done
+
+          if [ -z "$RUN_ID" ]; then
+            echo "Could not find the triggered workflow run."
+            exit 1
+          fi
+
+          echo "Watching run ID: $RUN_ID"
+          gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
+          CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
+          echo "Run concluded with: $CONCLUSION"
+          if [[ "$CONCLUSION" != "success" ]]; then
+            echo "Workflow run failed with conclusion: $CONCLUSION"
+            exit 1
+          fi
 
       - name: Download and extract model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove unwanted files
-        run: |
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-          find output -type f -name 'dmonitoring_model.onnx' -delete
-
-      - name: Copy model artifact(s) to GitLab recompiled dir
         env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
+          ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
+          RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="${{ github.event.inputs.build_model_ref }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+          ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
+          echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
+          mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
+          echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
+          gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
+          echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
+          ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
+          if [ -n "$ZIP_PATH" ]; then
+            echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
+            unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
+            rm -f "$ZIP_PATH"
+            echo "Unzipped and removed $ZIP_PATH"
+          else
+            echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME"
+          fi
+          echo "Done processing $ARTIFACT_NAME"
 
       - name: Push recompiled dir to GitLab
         env:
@@ -185,36 +163,28 @@ jobs:
         run: |
           cd gitlab_docs
           git checkout main
-          git pull origin main
           for d in models/"$RECOMPILED_DIR"/*/; do
             git sparse-checkout add "$d"
           done
           git add models/"$RECOMPILED_DIR"
           git config --global user.name "GitHub Action"
           git config --global user.email "action@github.com"
-          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
+          git commit -m "Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
           git push origin main
 
-      - run: |
-          cd docs
-          git pull origin gh-pages
-
       - name: Run json_parser.py to update JSON
         run: |
-          FOLDER="${{ inputs.model_folder }}"
+          FOLDER="${{ github.event.inputs.model_folder }}"
           if [ "$FOLDER" = "Other" ]; then
-            FOLDER="${{ inputs.custom_model_folder }}"
+            FOLDER="${{ github.event.inputs.custom_model_folder }}"
           fi
           ARGS=""
-          if [ "$FOLDER" != "None" ] && [ -n "$FOLDER" ]; then
-            ARGS="$ARGS --model-folder \"$FOLDER\""
-          fi
-          [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
-          [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
+          [ -n "$FOLDER" ] && ARGS="$ARGS --model-folder \"$FOLDER\""
+          [ -n "${{ github.event.inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ github.event.inputs.generation }}\""
+          [ -n "${{ github.event.inputs.version }}" ] && ARGS="$ARGS --version \"${{ github.event.inputs.version }}\""
           eval python3 docs/json_parser.py \
             --json-path "$JSON_FILE" \
             --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            --sort-by-date \
             $ARGS
 
       - name: Push updated JSON to GitHub docs repo

.github/workflows/cereal_validation.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - master-new
   pull_request:
     paths:
       - 'cereal/**'
@@ -16,50 +17,61 @@ on:
         type: string
 
 concurrency:
-  group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:
-  CI: 1
+  PYTHONWARNINGS: error
+  BASE_IMAGE: openpilot-base
+  BUILD: selfdrive/test/docker_build.sh base
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
+  generate_cereal_artifact:
+    name: Generate cereal validation artifacts
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
+      - name: Generate the log file
+        run: |
+          ${{ env.RUN }} "cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema_instances.bin" && \
+          ls -la
+          ls -la cereal/messaging/tests
+      - name: 'Prepare artifact'
+        run: |
+          mkdir -p "cereal/messaging/tests/cereal_validations"
+          cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py"
+          cp schema_instances.bin "cereal/messaging/tests/cereal_validations/schema_instances.bin"
+      - name: 'Upload Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: cereal_validations
+          path: cereal/messaging/tests/cereal_validations
+
   validate_cereal_with_upstream:
     name: Validate cereal with Upstream
     runs-on: ubuntu-24.04
+    needs: generate_cereal_artifact
     steps:
-      - name: Checkout sunnypilot cereal
-        uses: actions/checkout@v6
-        with:
-          sparse-checkout: cereal
-
-      - name: Init sunnypilot opendbc submodule
-        run: git submodule update --init --depth 1 opendbc_repo
-
-      - name: Checkout upstream openpilot cereal
-        uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         with:
           repository: 'commaai/openpilot'
-          path: upstream_openpilot
-          sparse-checkout: cereal
+          submodules: true
           ref: "refs/heads/master"
-
-      - name: Init upstream opendbc submodule
-        working-directory: upstream_openpilot
-        run: git submodule update --init --depth 1 opendbc_repo
-
-      - name: Install uv
-        run: pip install uv
-
-      - name: Generate sunnypilot schema
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: cereal_validations
+          path: cereal/messaging/tests/cereal_validations
+      - name: 'Run the validation'
         run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
-          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
-            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
-            -g -f /tmp/sp_schema.json --cereal-dir cereal
-
-      - name: Validate against upstream
-        run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
-          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
-            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
-            -r -f /tmp/sp_schema.json --cereal-dir upstream_openpilot/cereal
+          chmod +x cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py
+          ${{ env.RUN }} "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f cereal/messaging/tests/cereal_validations/schema_instances.bin"

.github/workflows/ci_weekly_report.yaml

@@ -0,0 +1,101 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
+jobs:
+  setup:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
+    steps:
+      - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
+        run: |
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.run_number }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
+              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
+                case "failure":
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
+                case "canceled":
+                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
+              }
+            });
+            return JSON.stringify({"jobs": report});
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
+
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -0,0 +1,17 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  selfdrive_tests:
+    uses: sunnypilot/sunnypilot/.github/workflows/selfdrive_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,21 @@
+name: 'compile openpilot'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/diff_report.yaml

@@ -1,45 +0,0 @@
-name: diff report
-
-on:
-  pull_request_target:
-    types: [opened, synchronize, reopened]
-
-jobs:
-  comment:
-    name: comment
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-    - name: Wait for process replay
-      id: wait
-      continue-on-error: true
-      uses: lewagon/wait-on-check-action@v1.3.4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        check-name: process replay
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        allowed-conclusions: success,failure
-        wait-interval: 20
-    - name: Download diff
-      if: steps.wait.outcome == 'success'
-      uses: dawidd6/action-download-artifact@v6
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: tests.yaml
-        workflow_conclusion: ''
-        pr: ${{ github.event.number }}
-        name: diff_report_${{ github.event.number }}
-        path: .
-        allow_forks: true
-    - name: Comment on PR
-      if: steps.wait.outcome == 'success'
-      uses: thollander/actions-comment-pull-request@v2
-      with:
-        filePath: diff_report.txt
-        comment_tag: diff_report
-        pr_number: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -22,19 +22,19 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
     # Build
     - name: Build docs
       run: |
-        git lfs pull
-        pip install zensical
-        python scripts/docs.py build
+        # TODO: can we install just the "docs" dependency group without the normal deps?
+        pip install mkdocs
+        mkdocs build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,54 +5,14 @@ on:
     types: [created, edited]
 
 jobs:
-  cleanup-branches:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Delete stale Jenkins branches
-        uses: actions/github-script@v8
-        with:
-          script: |
-            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
-            const prefixes = ['tmp-jenkins', '__jenkins'];
-
-            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              per_page: 100,
-            })) {
-              for (const branch of response.data) {
-                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
-
-                const { data: commit } = await github.rest.repos.getCommit({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: branch.commit.sha,
-                });
-
-                const commitDate = new Date(commit.commit.committer.date).getTime();
-                if (commitDate < cutoff) {
-                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
-                  await github.rest.git.deleteRef({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                    ref: `heads/${branch.name}`,
-                  });
-                }
-              }
-            }
-
+  # TODO: gc old branches in a separate job in this workflow
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
-    permissions:
-      contents: write
-      issues: write
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v8
+      uses: actions/github-script@v7
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +32,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -83,14 +43,3 @@ jobs:
         git config --global user.email "github-actions[bot]@users.noreply.github.com"
         git checkout -b tmp-jenkins-${{ github.event.issue.number }}
         GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
-
-    - name: Delete trigger comment
-      if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v8
-      with:
-        script: |
-          await github.rest.issues.deleteComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            comment_id: context.payload.comment.id,
-          });

.github/workflows/lfs-maintenance.yaml

@@ -9,11 +9,11 @@ on:
     - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
   push:
     branches:
-      - 'master'
+      - 'master-new'
   pull_request:
     branches:
-      - 'master'
-  workflow_dispatch: # enables manual triggering
+      - 'master-new'
+  workflow_dispatch:  # enables manual triggering
     inputs:
       upstream_branch:
         default: 'master'
@@ -30,7 +30,7 @@ jobs:
         with:
           repository: 'commaai/openpilot'
           ref: ${{ inputs.upstream_branch }}
-
+      
       - name: LFS Fetch
         run: |
           git lfs fetch
@@ -48,7 +48,7 @@ jobs:
       - name: Add GitLab public keys
         run: |
           ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
+      
       - name: Ensure branch
         run: |
           if git symbolic-ref -q HEAD >/dev/null; then

.github/workflows/model_review.yaml

@@ -16,23 +16,23 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          submodules: true
+        uses: actions/checkout@v4
       - name: Checkout master
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           ref: master
           path: base
       - run: git lfs pull
       - run: cd base && git lfs pull
 
+      - run: pip install onnx
+
       - name: scripts/reporter.py
         id: report
         run: |
           echo "content<<EOF" >> $GITHUB_OUTPUT
           echo "## Model Review" >> $GITHUB_OUTPUT
-          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Post model report comment

.github/workflows/post-to-discourse/action.yml

@@ -1,105 +0,0 @@
-name: 'Post to Discourse'
-description: 'Posts a message to a Discourse topic (existing or new)'
-
-inputs:
-  discourse-url:
-    description: 'Discourse instance URL (e.g., https://discourse.example.com)'
-    required: true
-  api-key:
-    description: 'Discourse API key'
-    required: true
-  api-username:
-    description: 'Discourse API username'
-    required: true
-  topic-id:
-    description: 'Discourse topic ID to post to (use this OR category-id + title)'
-    required: false
-  category-id:
-    description: 'Category ID for new topic (required if topic-id not provided)'
-    required: false
-  title:
-    description: 'Title for new topic (required if topic-id not provided)'
-    required: false
-  message:
-    description: 'Message content (markdown supported)'
-    required: true
-
-outputs:
-  post-number:
-    description: 'The post number in the topic'
-    value: ${{ steps.post.outputs.post_number }}
-  post-url:
-    description: 'Direct URL to the post'
-    value: ${{ steps.post.outputs.post_url }}
-  topic-id:
-    description: 'The topic ID (useful when creating a new topic)'
-    value: ${{ steps.post.outputs.topic_id }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: Post to Discourse
-      id: post
-      shell: bash
-      run: |
-        # Validate inputs
-        if [ -z "${{ inputs.topic-id }}" ] && ([ -z "${{ inputs.category-id }}" ] || [ -z "${{ inputs.title }}" ]); then
-          echo "❌ Error: Must provide either topic-id OR both category-id and title"
-          exit 1
-        fi
-        
-        if [ -n "${{ inputs.topic-id }}" ] && ([ -n "${{ inputs.category-id }}" ] || [ -n "${{ inputs.title }}" ]); then
-          echo "⚠️  Warning: Both topic-id and category-id/title provided. Will post to existing topic."
-        fi
-        
-        # Determine if creating new topic or posting to existing
-        if [ -n "${{ inputs.topic-id }}" ]; then
-          echo "📝 Posting to existing topic ID: ${{ inputs.topic-id }}"
-        
-          # Create JSON payload for posting to existing topic
-          PAYLOAD=$(jq -n \
-            --arg content '${{ inputs.message }}' \
-            --arg topic_id "${{ inputs.topic-id }}" \
-            '{topic_id: $topic_id, raw: $content}')
-        else
-          echo "✨ Creating new topic: ${{ inputs.title }}"
-        
-          # Create JSON payload for new topic
-          PAYLOAD=$(jq -n \
-            --arg content '${{ inputs.message }}' \
-            --arg title "${{ inputs.title }}" \
-            --arg category "${{ inputs.category-id }}" \
-            '{title: $title, category: ($category | tonumber), raw: $content}')
-        fi
-        
-        # Post to Discourse
-        RESPONSE=$(curl -s -w "\n%{http_code}" \
-          -X POST "${{ inputs.discourse-url }}/posts.json" \
-          -H "Content-Type: application/json" \
-          -H "Api-Key: ${{ inputs.api-key }}" \
-          -H "Api-Username: ${{ inputs.api-username }}" \
-          -d "$PAYLOAD")
-        
-        HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
-        BODY=$(echo "$RESPONSE" | sed '$d')
-        
-        if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
-          echo "✅ Successfully posted to Discourse!"
-        
-          POST_NUMBER=$(echo "$BODY" | jq -r '.post_number // "unknown"')
-          TOPIC_ID=$(echo "$BODY" | jq -r '.topic_id // "${{ inputs.topic-id }}"')
-          POST_URL="${{ inputs.discourse-url }}/t/${TOPIC_ID}/${POST_NUMBER}"
-        
-          echo "post_number=${POST_NUMBER}" >> $GITHUB_OUTPUT
-          echo "post_url=${POST_URL}" >> $GITHUB_OUTPUT
-          echo "topic_id=${TOPIC_ID}" >> $GITHUB_OUTPUT
-        
-          echo "Topic ID: ${TOPIC_ID}"
-          echo "Post number: ${POST_NUMBER}"
-          echo "URL: ${POST_URL}"
-        else
-          echo "❌ Failed to post to Discourse"
-          echo "HTTP Code: ${HTTP_CODE}"
-          echo "Response: ${BODY}"
-          exit 1
-        fi

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh
+  BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
   build_prebuilt:
@@ -28,8 +28,8 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
-    - uses: actions/checkout@v6
+        check-regexp: ^((?!.*(build master-ci).*).)*$
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - run: git lfs pull

.github/workflows/release-drafter.yml

@@ -3,6 +3,7 @@ name: Release Drafter
 on:
   push:
     branches:
+      - master-new
       - master
     tags:
       - 'v*'

.github/workflows/release.yaml

@@ -5,27 +5,38 @@ on:
   workflow_dispatch:
 
 jobs:
-  build___nightly:
-    name: build __nightly
+  build_masterci:
+    name: build master-ci
+    env:
+      ImageOS: ubuntu24
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     permissions:
       checks: read
       contents: write
     steps:
+    - name: Install wait-on-check-action dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
       uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
       with:
         ref: master
         wait-interval: 30
-        running-workflow-name: 'build __nightly'
+        running-workflow-name: 'build master-ci'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
+        check-regexp: ^((?!.*(build prebuilt).*).)*$
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-    - run: ./tools/op.sh setup
-    - name: Push __nightly
-      run: BRANCH=__nightly release/build_stripped.sh
+    - name: Pull LFS
+      run: |
+        git config --global --add safe.directory '*'
+        git lfs pull
+    - name: Push master-ci
+      run: BRANCH=__nightly release/build_devel.sh

.github/workflows/repo-maintenance.yaml

@@ -6,72 +6,67 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: openpilot-base
+  BUILD: selfdrive/test/docker_build.sh base
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
+  update_translations:
+    runs-on: ubuntu-latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Update translations
+        run: |
+          ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish"
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
+        with:
+          author: Vehicle Researcher <user@comma.ai>
+          commit-message: "Update translations"
+          title: "[bot] Update translations"
+          body: "Automatic PR from repo-maintenance -> update_translations"
+          branch: "update-translations"
+          base: "master"
+          delete-branch: true
+          labels: bot
+
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
     if: github.repository == 'sunnypilot/sunnypilot'
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
     - name: uv lock
-      run: uv lock --upgrade
-    - name: uv pip tree
-      id: pip_tree
       run: |
-        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
-        uv pip tree >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
-    - name: venv size
-      id: venv_size
-      run: |
-        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
-        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
-        echo "" >> $GITHUB_OUTPUT
-        echo "Top 10 by size:" >> $GITHUB_OUTPUT
-        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
-          | grep -v '\.dist-info' \
-          | grep -v '__pycache__' \
-          | sort -rh \
-          | head -10 \
-          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
+        python3 -m ensurepip --upgrade
+        pip3 install uv
+        uv lock --upgrade
     - name: bump submodules
       run: |
-        git config submodule.msgq.update none
-        git config submodule.rednose_repo.update none
-        git config submodule.teleoprtc_repo.update none
-        git config submodule.tinygrad.update none
+        git config --global --add safe.directory '*'
         git submodule update --remote
         git add .
     - name: update car docs
       run: |
+        export PYTHONPATH="$PWD"
+        scons -j$(nproc) --minimal opendbc_repo
         python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:
-        author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
-        token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
+        author: Vehicle Researcher <user@comma.ai>
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
         commit-message: Update Python packages
         title: '[bot] Update Python packages'
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: |
-          Automatic PR from repo-maintenance -> package_updates
-
-          ```
-          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
-          ${{ steps.venv_size.outputs.VENV_SIZE }}
-          ```
-
-          ```
-          $ uv pip tree
-          ${{ steps.pip_tree.outputs.PIP_TREE }}
-          ```
+        body: 'Automatic PR from repo-maintenance -> package_updates'
         labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -0,0 +1,346 @@
+name: selfdrive
+
+on:
+  push:
+    branches:
+      - master
+      - master-new
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
+
+concurrency:
+  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  REPORT_NAME: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
+  PYTHONWARNINGS: error
+  BASE_IMAGE: openpilot-base
+  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
+
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
+  PYTEST: pytest --continue-on-collection-errors --durations=0 --durations-min=5 -n logical
+
+jobs:
+  build_release:
+    if: github.repository == 'commaai/openpilot' # build_release blocked for the time being to only comma as we may have a different process.
+    name: build release
+    runs-on:
+      - 'ubuntu-24.04'
+    env:
+      STRIPPED_DIR: /tmp/releasepilot
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Getting LFS files
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: git lfs pull
+    - name: Build devel
+      timeout-minutes: 1
+      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot and run checks
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "python3 system/manager/build.py"
+    - name: Run tests
+      timeout-minutes: 1
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - name: Check submodules
+      if: github.repository == 'sunnypilot/sunnypilot'
+      timeout-minutes: 3
+      run: release/check-submodules.sh
+
+  build:
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup docker push
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
+    - uses: ./.github/workflows/setup-with-retry
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: 30
+
+  build_mac:
+    name: build macOS
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - name: Homebrew cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          brew-macos-${{ env.CACHE_COMMIT_DATE }}
+          brew-macos
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+      env:
+        # package install has DeprecationWarnings
+        PYTHONWARNINGS: default
+    - name: Save Homebrew cache
+      uses: actions/cache/save@v4
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+    - run: git lfs pull
+    - name: Getting scons cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}-macos
+    - name: Building openpilot
+      run: . .venv/bin/activate && scons -j$(nproc)
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+
+  static_analysis:
+    name: static analysis
+    runs-on:
+      - 'ubuntu-latest'
+    env:
+      PYTHONWARNINGS: default
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Static analysis
+      timeout-minutes: 1
+      run: ${{ env.RUN }} "scripts/lint/lint.sh"
+
+  unit_tests:
+    name: unit tests
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run unit tests
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \
+                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
+                        ./selfdrive/ui/tests/create_test_translations.sh && \
+                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
+                        chmod -R 777 /tmp/comma_download_cache"
+
+  process_replay:
+    name: process replay
+    if: github.repository == 'commaai/openpilot'  # disable process_replay for forks
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run replay
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache"
+    - name: Print diff
+      id: print-diff
+      if: always()
+      run: cat selfdrive/test/process_replay/diff.txt
+    - uses: actions/upload-artifact@v4
+      if: always()
+      continue-on-error: true
+      with:
+        name: process_replay_diff.txt
+        path: selfdrive/test/process_replay/diff.txt
+    - name: Upload reference logs
+      if: false # TODO: move this to github instead of azure
+      run: |
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+    - name: Run regen
+      if: false
+      timeout-minutes: 4
+      run: |
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
+
+  test_cars:
+    name: cars
+    runs-on:
+      - 'ubuntu-24.04'
+    strategy:
+      fail-fast: false
+      matrix:
+        job: [0, 1, 2, 3]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: routes-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }}
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Test car models
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 6 }}
+      run: |
+        ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
+      env:
+        NUM_JOBS: 4
+        JOB_ID: ${{ matrix.job }}
+
+  car_docs_diff:
+    name: PR comments
+    runs-on: ubuntu-latest
+    #if: github.event_name == 'pull_request'
+    if: false  # TODO: run this in opendbc?
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.base.ref }}
+      - run: git lfs pull
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Get base car info
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
+          sudo chown -R $USER:$USER ${{ github.workspace }}
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          path: current
+      - run: cd current && git lfs pull
+      - name: Save car docs diff
+        id: save_diff
+        run: |
+          cd current
+          ${{ env.RUN }} "scons -j$(nproc)"
+          output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
+          output="${output//$'\n'/'%0A'}"
+          echo "::set-output name=diff::$output"
+      - name: Find comment
+        if: ${{ env.AZURE_TOKEN != '' }}
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: This PR makes changes to
+      - name: Update comment
+        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: "${{ steps.save_diff.outputs.diff }}"
+          edit-mode: replace
+      - name: Delete comment
+        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.deleteComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: ${{ steps.fc.outputs.comment-id }}
+            })
+
+  simulator_driving:
+    name: simulator driving
+    runs-on:
+      - 'ubuntu-24.04'
+    if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Driving test
+      timeout-minutes: 1
+      run: |
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        source selfdrive/test/setup_vsound.sh && \
+                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
+
+  create_ui_report:
+    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
+    name: Create UI Report
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: caching frames
+        id: frames-cache
+        uses: actions/cache@v4
+        with:
+          path: .ci_cache/comma_download_cache
+          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create Test Report
+        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 2 || 4) }}
+        run: >
+            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                            source selfdrive/test/setup_xvfb.sh &&
+                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
+                            chmod -R 777 /tmp/comma_download_cache"
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.REPORT_NAME }}
+          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-with-retry/action.yaml

@@ -0,0 +1,37 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: false
+    default: ''
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
+
+runs:
+  using: "composite"
+  steps:
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        is_retried: true

.github/workflows/setup/action.yaml

@@ -0,0 +1,56 @@
+name: 'openpilot env setup'
+
+inputs:
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    required: false
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
+    - shell: bash
+      name: No retries!
+      run: |
+        if [ "${{ github.run_attempt }}" -gt ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association == 'NONE' && 2 || 1}} ]; then
+          echo -e "\033[0;31m##################################################"
+          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
+          echo -e "\033[0;31m##################################################\033[0m"
+          exit 1
+        fi
+
+    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
+    - shell: bash
+      run: git lfs pull
+
+    # build cache
+    - id: date
+      shell: bash
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - shell: bash
+      run: echo "$CACHE_COMMIT_DATE"
+    - id: scons-cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    # build our docker image
+    - shell: bash
+      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -5,15 +5,15 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 7
-  DAYS_BEFORE_PR_STALE: 24
+  DAYS_BEFORE_PR_CLOSE: 2
+  DAYS_BEFORE_PR_STALE: 9
   DAYS_BEFORE_PR_STALE_DRAFT: 30
 
 jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 
@@ -34,7 +34,7 @@ jobs:
   stale_drafts:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 

.github/workflows/sunnypilot-build-model.yaml

@@ -9,27 +9,6 @@ env:
   MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
 
 on:
-  workflow_call:
-    inputs:
-      upstream_branch:
-        description: 'Upstream branch to build from'
-        required: true
-        default: 'master'
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
-      artifact_suffix:
-        description: 'Suffix for artifact name'
-        required: false
-        type: string
-        default: ''
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -53,53 +32,34 @@ run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from
 jobs:
   get_model:
     runs-on: ubuntu-latest
-    env:
-      REF: ${{ inputs.upstream_branch }}
     outputs:
       model_date: ${{ steps.commit-date.outputs.model_date }}
     steps:
-      # Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
-      # and fallback to sunnypilot if the ref checkout fails.
-      - name: Checkout commaai/openpilot
-        id: checkout_upstream
-        continue-on-error: true
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          repository: commaai/openpilot
-          ref: ${{ inputs.upstream_branch }}
+          repository: ${{ env.UPSTREAM_REPO }}
+          ref: ${{ github.event.inputs.upstream_branch }}
           submodules: recursive
-          path: openpilot
-
-      - name: Fallback to sunnypilot/sunnypilot
-        if: steps.checkout_upstream.outcome == 'failure'
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot
-          ref: ${{ inputs.upstream_branch }}
-          submodules: recursive
-          path: openpilot
       - name: Get commit date
         id: commit-date
         run: |
-          cd ${{ github.workspace }}/openpilot
+          # Get the commit date in YYYY-MM-DD format
           commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
           echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
-      - run: |
-          cd ${{ github.workspace }}/openpilot
-          git lfs pull
+      - run: git lfs pull
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v4
         with:
-          name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
-          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
+          name: models
+          path: ${{ github.workspace }}/selfdrive/modeld/models/*.onnx
 
   build_model:
-    runs-on: [self-hosted, tici]
+    runs-on: self-hosted
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
-      REF: ${{ inputs.upstream_branch }}
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -111,7 +71,7 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
@@ -154,10 +114,8 @@ jobs:
       - name: Download model artifacts
         uses: actions/download-artifact@v4
         with:
-          name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
+          name: models
           path: ${{ github.workspace }}/selfdrive/modeld/models
-      - run: |
-          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |
@@ -173,18 +131,9 @@ jobs:
 
             echo "Compiling: $onnx_file -> $output_file"
             QCOM=1 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file"
-            DEV=QCOM FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            QCOM=1 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
-      - name: Validate Model Outputs
-        run: |
-          source /etc/profile
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
-            --validate-only \
-            --model-dir "${{ env.MODELS_DIR }}"
-
       - name: Prepare Output
         run: |
           sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -193,6 +142,7 @@ jobs:
           # Copy the model files
           rsync -avm \
             --include='*.dlc' \
+            --include='*.thneed' \
             --include='*.pkl' \
             --include='*.onnx' \
             --exclude='*' \
@@ -207,22 +157,12 @@ jobs:
             --upstream-branch "${{ inputs.upstream_branch }}" \
             ${{ inputs.is_20hz && '--is-20hz' || '' }}
 
-      - name: Write artifact name to file
-        run: echo "model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}" > ${{ env.OUTPUT_DIR }}/artifact_name.txt
-
       - name: Upload Build Artifacts
-        id: upload-artifact
         uses: actions/upload-artifact@v4
         with:
-          name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
+          name: model-${{ env.MODEL_NAME }}-${{ github.run_number }}
           path: ${{ env.OUTPUT_DIR }}
 
-      - name: Upload artifact name file
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
-          path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
-
       - name: Re-enable powersave
         if: always()
         run: |

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -6,121 +6,57 @@ env:
   CI_DIR: ${{ github.workspace }}/release/ci
   SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
-
+  
   # Branch configurations
-  STAGING_SOURCE_BRANCH: 'master'
-
+  STAGING_C3_SOURCE_BRANCH: ${{ vars.STAGING_C3_SOURCE_BRANCH || 'master-new' }} # vars are set on repo settings.
+  DEV_C3_SOURCE_BRANCH: ${{ vars.DEV_C3_SOURCE_BRANCH || 'master-dev-c3-new' }} # vars are set on repo settings.
+  
+  # Target branch configurations
+  STAGING_TARGET_BRANCH: ${{ vars.STAGING_TARGET_BRANCH || 'staging-c3-new' }} # vars are set on repo settings.
+  DEV_TARGET_BRANCH: ${{ vars.DEV_TARGET_BRANCH || 'dev-c3-new' }} # vars are set on repo settings.
+  RELEASE_TARGET_BRANCH: ${{ vars.RELEASE_TARGET_BRANCH || 'release-c3-new' }} # vars are set on repo settings.
+  
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
 on:
   push:
-    branches: [ master, master-dev ]
-    tags: [ 'release/*' ]
+    branches: [ master, master-new, master-dev-c3-new ]
+    tags: [ '*' ]
   pull_request_target:
     types: [ labeled ]
   workflow_dispatch:
     inputs:
       wait_for_tests:
-        description: 'Wait for tests to finish'
+        description: 'Wait for selfdrive_tests to finish'
         required: false
         type: boolean
         default: false
 
 jobs:
-  prepare_strategy:
-    runs-on: ubuntu-24.04
-    if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-    outputs:
-      environment: ${{ steps.strategy.outputs.environment }}
-      new_branch: ${{ steps.strategy.outputs.new_branch }}
-      extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
-      version: ${{ steps.strategy.outputs.version }}
-      cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
-      publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
-      is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
-      build: ${{ steps.strategy.outputs.build }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Extract deploy strategy
-        id: strategy
-        run: |
-          echo '::group::Strategy Extraction'
-          BRANCH="${{ github.head_ref || github.ref_name }}"
-          echo "Current branch: $BRANCH"
-
-          STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
-          CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
-            .configs[] | select(.branch == $branch)
-          ')
-
-          BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
-          if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
-            echo "No exact strategy match found. Falling back to feature/fork logic."
-            IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
-            FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
-            NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
-
-            echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
-            echo "version=$BUILD" >> $GITHUB_OUTPUT
-            echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
-            echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
-            echo "environment=feature-branch" >> $GITHUB_OUTPUT
-            echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
-          else
-          echo "Matched config: $CONFIG"
-            environment=$(echo "$CONFIG" | jq -r '.environment')
-            echo "environment=$environment" >> $GITHUB_OUTPUT
-            echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
-            cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
-            echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
-            echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
-
-            is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
-            echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
-
-            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
-            echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
-            echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
-          fi
-          echo "build=$BUILD" >> $GITHUB_OUTPUT
-          cat $GITHUB_OUTPUT
-
   validate_tests:
     runs-on: ubuntu-24.04
-    needs: [ prepare_strategy ]
-    if: ${{
-      ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
-      (github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
-      contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      }}
+    if: ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) || contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
     steps:
       - uses: actions/checkout@v4
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
         with:
-          workflow: tests.yaml # The workflow file to monitor
+          workflow: selfdrive_tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
 
   build:
-    needs: [ validate_tests, prepare_strategy ]
+    needs: [ validate_tests ]
     concurrency:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
-    runs-on: [self-hosted, tici]
+    runs-on: self-hosted
     outputs:
-      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
-      version: ${{ needs.prepare_strategy.outputs.version }}
-      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
-      commit_sha: ${{ github.sha }}
-    if: ${{
-      (always() && !cancelled() && !failure()) &&
-      needs.prepare_strategy.result == 'success' &&
-      (needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
-      (!contains(github.event_name, 'pull_request') ||
-      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      }}
+      new_branch: ${{ steps.set-env.outputs.new_branch }}
+      version: ${{ steps.set-env.outputs.version }}
+      extra_version_identifier: ${{ steps.set-env.outputs.extra_version_identifier }}
+      commit_sha: ${{ steps.set-env.outputs.commit_sha }}
+    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -134,21 +70,70 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_C3_SOURCE_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}
 
+      - name: Set Feature Branch Prebuilt Configuration
+        id: set_feature_configuration
+        if: (
+              !(env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH) &&
+              !(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH) &&
+              !(startsWith(github.ref, 'refs/tags/'))
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.SOURCE_BRANCH }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-prebuilt" >> $GITHUB_ENV
+          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
+
+      - name: Set dev-c3-new prebuilt Configuration
+        id: set_dev_configuration
+        if: (
+                steps.set_feature_configuration.outcome == 'skipped' && 
+                env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV
+          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
+          echo "EXTRA_VERSION_IDENTIFIER=${{ github.run_number }}" >> $GITHUB_ENV
+
+      - name: Set staging-c3-new prebuilt Configuration
+        id: set_staging_configuration
+        if: (
+                steps.set_feature_configuration.outcome == 'skipped' &&
+                !contains(github.event_name, 'pull_request') &&
+                steps.set_dev_configuration.outcome == 'skipped' && 
+                (env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH)
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV
+          echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV
+          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-staging" >> $GITHUB_ENV
+
+      - name: Set release-c3-new prebuilt Configuration
+        id: set_tag_configuration
+        if: (
+              steps.set_feature_configuration.outcome == 'skipped' &&
+              !contains(github.event_name, 'pull_request') &&
+              steps.set_staging_configuration.outcome == 'skipped' &&
+              startsWith(github.ref, 'refs/tags/')
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV
+          echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV
+          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-release" >> $GITHUB_ENV
+
       - name: Set environment variables
         id: set-env
         run: |
-          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
-          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
-          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
+          # Write to GITHUB_OUTPUT from environment variables
+          echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
+          [[ ! -z "$EXTRA_VERSION_IDENTIFIER" ]] && echo "extra_version_identifier=$EXTRA_VERSION_IDENTIFIER" >> $GITHUB_OUTPUT
+          [[ ! -z "$VERSION" ]] && echo "version=$VERSION" >> $GITHUB_OUTPUT
           echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
-
+          
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
@@ -180,13 +165,6 @@ jobs:
           ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
           cd $BUILD_DIR
           sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
-          echo "Building sunnypilot's modeld_v2..."
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
-          echo "Building sunnypilot's locationd..."
-          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
-          echo "Building openpilot's locationd..."
-          scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
-          echo "Building rest of sunnypilot"
           scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
@@ -198,27 +176,37 @@ jobs:
           sudo rm -rf ${OUTPUT_DIR}
           mkdir -p ${OUTPUT_DIR}
           rsync -am${RUNNER_DEBUG:+v} \
+            --include='**/panda/board/' \
+            --include='**/panda/board/obj' \
+            --include='**/panda/board/obj/panda.bin.signed' \
+            --include='**/panda/board/obj/panda_h7.bin.signed' \
+            --include='**/panda/board/obj/bootstub.panda.bin' \
+            --include='**/panda/board/obj/bootstub.panda_h7.bin' \
             --exclude='.sconsign.dblite' \
             --exclude='*.a' \
             --exclude='*.o' \
             --exclude='*.os' \
             --exclude='*.pyc' \
             --exclude='moc_*' \
-            --exclude='__pycache__' \
+            --exclude='*.cc' \
             --exclude='Jenkinsfile' \
+            --exclude='supercombo.onnx' \
+            --exclude='**/panda/board/*' \
+            --exclude='**/panda/board/obj/**' \
+            --exclude='**/panda/certs/' \
+            --exclude='**/panda/crypto/' \
             --exclude='**/release/' \
             --exclude='**/.github/' \
             --exclude='**/selfdrive/ui/replay/' \
             --exclude='**/__pycache__/' \
+            --exclude='**/selfdrive/ui/*.h' \
+            --exclude='**/selfdrive/ui/**/*.h' \
+            --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' \
             --exclude='${{env.SCONS_CACHE_DIR}}' \
             --exclude='**/.git/' \
             --exclude='**/SConstruct' \
             --exclude='**/SConscript' \
             --exclude='**/.venv/' \
-            --exclude='selfdrive/modeld/models/driving_vision.onnx' \
-            --exclude='selfdrive/modeld/models/driving_policy.onnx' \
-            --exclude='third_party/*x86*' \
-            --exclude='third_party/*Darwin*' \
             --delete-excluded \
             --chown=comma:comma \
             ${BUILD_DIR}/ ${OUTPUT_DIR}/
@@ -242,15 +230,12 @@ jobs:
 
   publish:
     concurrency:
-      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
-      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
-      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
-      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
-      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
-    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build, prepare_strategy ]
+      group: publish-${{ github.head_ref || github.ref_name }}
+      cancel-in-progress: true
+    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+    needs: [ build ]
     runs-on: ubuntu-24.04
-    environment: ${{ needs.prepare_strategy.outputs.environment }}
+    environment: ${{ (contains(fromJSON(vars.AUTO_DEPLOY_PREBUILT_BRANCHES), github.head_ref || github.ref_name) || contains(github.event.pull_request.labels.*.name, 'prebuilt')) && 'auto-deploy' || 'feature-branch' }}
     steps:
       - uses: actions/checkout@v4
 
@@ -282,8 +267,8 @@ jobs:
             "${{ needs.build.outputs.new_branch }}" \
             "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.build.outputs.extra_version_identifier }}"
-
+            "-${{ needs.build.outputs.extra_version_identifier }}"
+          
           echo ""
           echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
           echo ""
@@ -291,60 +276,38 @@ jobs:
           echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
           echo "3. Update as needed (JSON array with no spaces)"
 
-      - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
-        if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
-        run: |
-            TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
-            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
-            git push -f origin ${TAG}
-
   notify:
-    needs:
-    - prepare_strategy
-    - build
-    - publish
+    needs: [ build, publish ]
     runs-on: ubuntu-24.04
-    if: ${{ (always() && !cancelled() && !failure())
-      && needs.publish.result == 'success'
-      && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
+    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
     steps:
       - uses: actions/checkout@v4
-
-      - name: Prepare notification message
-        id: message
-        run: |
-          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
-          export VERSION="${{ needs.prepare_strategy.outputs.version }}"
-          export branch_name="${{ env.SOURCE_BRANCH }}"
-          export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
-          export commit_sha="${{ github.sha }}"
-          export commit_short_sha="${{ github.sha }}"
-          export commit_short_sha="${commit_short_sha:0:7}"
-          export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
-          export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
-
-          MESSAGE=$(cat << 'EOF' | envsubst
-          ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
-          EOF
-          )
-
-          {
-            echo 'content<<EOFMARKER'
-            echo "$MESSAGE"
-            echo 'EOFMARKER'
-          } >> $GITHUB_OUTPUT
-        shell: bash
-
-      - name: Post to Discourse
-        uses: ./.github/workflows/post-to-discourse
+      - name: Setup Alpine Linux environment
+        uses: jirutka/setup-alpine@v1.2.0
         with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: "system"
-          topic-id: ${{ fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name].topic_id }}
-          message: ${{ steps.message.outputs.content }}
+          packages: 'jq gettext curl'
 
+      - name: Send Discord Notification
+        env:
+          DISCORD_WEBHOOK: ${{ contains(fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES), env.SOURCE_BRANCH) && secrets.DISCORD_DEV_FEEDBACK_CHANNEL_WEBHOOK || secrets.DISCORD_DEV_PRIVATE_CHANNEL_WEBHOOK }}
+        run: |
+          TEMPLATE='${{ vars.DISCORD_GENERAL_UPDATE_NOTICE }}'
+          export EXTRA_VERSION_IDENTIFIER="${{ needs.build.outputs.extra_version_identifier }}"
+          export VERSION="${{ needs.build.outputs.version }}"
+          export branch_name=${{ env.SOURCE_BRANCH }}
+          export new_branch=${{ needs.build.outputs.new_branch }}
+          export extra_version_identifier=${{ needs.build.outputs.extra_version_identifier || github.run_number}}
+          echo ${TEMPLATE} | envsubst | jq -c '.' | tee payload.json
+          curl -X POST -H "Content-Type: application/json" -d @payload.json $DISCORD_WEBHOOK
+          
+          echo ""
+          echo "----   ℹ️ To update the list of branches that notify to dev-feedback  -----"
+          echo ""
+          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/DEV_FEEDBACK_NOTIFICATION_BRANCHES"
+          echo "2. Current value: ${{ vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES }}"
+          echo "3. Update as needed (JSON array with no spaces)"
+        shell: alpine.sh {0}
+  
   manage-pr-labels:
     name: Remove prebuilt label
     runs-on: ubuntu-latest

.github/workflows/sunnypilot-master-dev-c3-prep.yaml

@@ -1,8 +1,9 @@
-name: Build dev
+name: Build dev-c3-new
 
 env:
-  DEFAULT_SOURCE_BRANCH: "master"
-  DEFAULT_TARGET_BRANCH: "master-dev"
+  DEFAULT_SOURCE_BRANCH: "master-new"
+  DEFAULT_TARGET_BRANCH: "master-dev-c3-new"
+  PR_LABEL: "dev-c3"
   LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
   LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
 
@@ -10,21 +11,23 @@ on:
   push:
     branches:
       - master
+      - master-new    
   pull_request_target:
-    types: [ labeled ]
-    branches:
-      - 'master'
+    types: [ labeled ]  
+    branches:    
+      - 'master'        
+      - 'master-new'            
   workflow_dispatch:
     inputs:
       source_branch:
         description: 'Source branch to reset from'
         required: true
-        default: 'master'
+        default: 'master-new'
         type: string
       target_branch:
         description: 'Target branch to reset and squash into'
         required: true
-        default: 'master-dev'
+        default: 'master-dev-c3-new'
         type: string
       cancel_in_progress:
         description: 'Cancel any in-progress runs of this workflow'
@@ -40,25 +43,24 @@ jobs:
   reset-and-squash:
     runs-on: ubuntu-latest
     if: (
-      (github.event_name == 'workflow_dispatch')
-      || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-      || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
-      )
+            (github.event_name == 'workflow_dispatch')
+            || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
+            || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
+        )
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Fetch all history for all branches
           token: ${{ secrets.GITHUB_TOKEN }}
-          persist-credentials: false
 
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
         if: (
-          (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-          || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
-          )
+                (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
+                || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
+            )
         with:
-          workflow: tests.yaml # The workflow file to monitor
+          workflow: selfdrive_tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Configure Git
@@ -118,8 +120,8 @@ jobs:
         run: |
           # Use GitHub API to get PRs with specific label, ordered by creation date
           PR_LIST=$(gh api graphql -f query='
-            query($search_query:String!) {
-              search(query: $search_query, type:ISSUE, first:40) {
+            query($label:String!) {
+              search(query: $label, type:ISSUE, first:100) {
                 nodes {
                   ... on PullRequest {
                     number
@@ -149,9 +151,8 @@ jobs:
                   }
                 }
               }
-            }' -F search_query="repo:${{ github.repository }} is:pr is:open label:${{ vars.PREBUILT_PR_LABEL }},${{ vars.PREBUILT_PR_LABEL }}-c3 draft:false sort:created-asc")
+            }' -F label="is:pr is:open label:${PR_LABEL} draft:false sort:created-asc")
 
-          PR_LIST=${PR_LIST//\'/}
           echo "PR_LIST=${PR_LIST}" >> $GITHUB_OUTPUT
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -174,38 +175,11 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Restore workflows from source
-        run: |
-          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-          SOURCE_BRANCH="${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-
-          # Ensure we are on the target branch
-          git checkout $TARGET_BRANCH
-
-          echo "Restoring .github/workflows from $SOURCE_BRANCH"
-          git checkout origin/$SOURCE_BRANCH -- .github/workflows
-
-          if ! git diff --cached --quiet; then
-            echo "Workflows differ. Committing restoration."
-            git commit -m "chore: restore .github/workflows from $SOURCE_BRANCH"
-          else
-            echo "Workflows match $SOURCE_BRANCH."
-          fi
-
-      - uses: actions/create-github-app-token@v2
-        id: ci-token
-        with:
-          app-id: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_ID }}
-          private-key: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_PRIVATE_KEY }}
-
       - name: Push changes if there are diffs
-        id: push-changes
+        id: push-changes  # Add an id so we can reference this step
         run: |
           TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
 
-          # Use the App Token to set the remote URL with authentication
-          git remote set-url origin "https://x-access-token:${{ steps.ci-token.outputs.token }}@github.com/${{ github.repository }}.git"
-
           # Fetch the latest from remote
           git fetch origin $TARGET_BRANCH
 
@@ -216,7 +190,7 @@ jobs:
             exit 0
           fi
 
-          # Push with the authenticated origin
+          # If we get here, there are diffs, so push
           if ! git push origin $TARGET_BRANCH --force; then
             echo "Failed to push changes to $TARGET_BRANCH"
             exit 1
@@ -229,15 +203,22 @@ jobs:
         if: steps.push-changes.outputs.has_changes == 'true'
         run: |
           echo "Triggering selfdrive tests..."
-          gh workflow run tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+          gh workflow run selfdrive_tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
 
           echo "Sleeping for 120s to give plenty of time for the action to start and then we wait"
           sleep 120
 
           echo "Getting latest run ID..."
-          RUN_ID=$(gh run list --workflow=tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+          RUN_ID=$(gh run list --workflow=selfdrive_tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
 
           echo "Watching run ID: $RUN_ID"
           gh run watch "$RUN_ID"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Trigger prebuilt workflow
+        if: success() && steps.push-changes.outputs.has_changes == 'true'
+        run: |
+          gh workflow run sunnypilot-build-prebuilt.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-discourse.yaml.yml

@@ -1,78 +0,0 @@
-name: Debug Discourse Posting
-
-on:
-  push:
-
-jobs:
-  test-discourse-post:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Post test message to Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
-          topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
-          message: |
-            ## 🧪 Test Post from GitHub Actions
-            
-            **This is a test post to verify Discourse integration**
-            
-            - **Workflow**: ${{ github.workflow }}
-            - **Run Number**: #${{ github.run_number }}
-            - **Branch**: `${{ github.ref_name }}`
-            - **Commit**: ${{ github.sha }}
-            - **Actor**: @${{ github.actor }}
-            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
-            
-            ---
-            
-            ### Fake Build Info (for testing)
-            - **Version**: 0.9.8-test
-            - **Build**: #42
-            - **Branch**: release-test
-            
-            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            
-            *This is an automated test message. Drive safe! 🚗💨*
-      
-      
-      - name: Create topic on Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
-          #topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
-          category-id: 4
-          title: "This is a test of a new topic instead of a reply"
-          message: |
-            ## 🧪 Test Post from GitHub Actions
-            
-            **This is a test post to verify Discourse integration**
-            
-            - **Workflow**: ${{ github.workflow }}
-            - **Run Number**: #${{ github.run_number }}
-            - **Branch**: `${{ github.ref_name }}`
-            - **Commit**: ${{ github.sha }}
-            - **Actor**: @${{ github.actor }}
-            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
-            
-            ---
-            
-            ### Fake Build Info (for testing)
-            - **Version**: 0.9.8-test
-            - **Build**: #42
-            - **Branch**: release-test
-            
-            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            
-            *This is an automated test message. Drive safe! 🚗💨*
-      - name: Display results
-        if: always()
-        run: |
-          echo "::notice::Discourse post test completed"
-          echo "Check your Discourse topic to verify the post appeared correctly"

.github/workflows/tests.yaml

@@ -1,248 +0,0 @@
-name: tests
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  CI: 1
-  PYTHONPATH: ${{ github.workspace }}
-  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
-
-jobs:
-  build_release:
-    name: build release
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-      PYTHONPATH: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - run: ./tools/op.sh setup
-    - name: Build openpilot and run checks
-      timeout-minutes: 30
-      working-directory: ${{ env.STRIPPED_DIR }}
-      run: python3 system/manager/build.py
-    - name: Run tests
-      timeout-minutes: 1
-      working-directory: ${{ env.STRIPPED_DIR }}
-      run: release/check-dirty.sh
-    - name: Check submodules
-      if: github.repository == 'sunnypilot/sunnypilot'
-      timeout-minutes: 3
-      run: |
-        if [ "${{ github.ref }}" != "refs/heads/master" ]; then
-          git fetch  origin master:refs/remotes/origin/master
-
-          SUBMODULE_PATHS=$(git diff origin/master HEAD --name-only | grep -E '^[^/]+$' | while read path; do
-            if git ls-files --stage "$path" | grep -q "^160000"; then
-              echo "$path"
-            fi
-          done | tr '\n' ' ')
-
-          if [ -n "$SUBMODULE_PATHS" ]; then
-            echo "Changed submodule paths: $SUBMODULE_PATHS"
-            export SUBMODULE_PATHS="$SUBMODULE_PATHS"
-            export CHECK_PR_REFS=true
-          fi
-        fi
-        release/check-submodules.sh
-
-  build_mac:
-    name: build macOS
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Remove Homebrew from environment
-      run: |
-        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
-        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
-    - run: ./tools/op.sh setup
-    - name: Building openpilot
-      run: scons
-
-  static_analysis:
-    name: static analysis
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Static analysis
-      timeout-minutes: 1
-      run: scripts/lint/lint.sh
-
-  unit_tests:
-    name: unit tests
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
-      run: |
-        source selfdrive/test/setup_xvfb.sh
-        # Pre-compile Python bytecode so each pytest worker doesn't need to
-        $PYTEST --collect-only -m 'not slow' -qq
-        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
-
-  process_replay:
-    name: process replay
-    if: false  # disable process_replay for forks
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
-      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
-      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - name: Print diff report
-      if: always()
-      run: cat selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v6
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload diff report
-      uses: actions/upload-artifact@v6
-      if: always() && github.event_name == 'pull_request'
-      continue-on-error: true
-      with:
-        name: diff_report_${{ github.event.number }}
-        path: selfdrive/test/process_replay/diff_report.txt
-    - name: Checkout ci-artifacts
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v4
-      with:
-        repository: commaai/ci-artifacts
-        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-        path: ${{ github.workspace }}/ci-artifacts
-    - name: Push refs
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      working-directory: ${{ github.workspace }}/ci-artifacts
-      run: |
-        git config user.name "GitHub Actions Bot"
-        git config user.email "<>"
-        git fetch origin process-replay || true
-        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
-        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
-        echo "${{ github.sha }}" > ref_commit
-        git add .
-        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
-        git push origin process-replay --force
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      env:
-        ONNXCPU: 1
-      run: $PYTEST selfdrive/test/process_replay/test_regen.py
-
-  simulator_driving:
-    name: simulator driving
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: false  # FIXME: Started to timeout recently
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Driving test
-      timeout-minutes: 2
-      run: |
-        source selfdrive/test/setup_xvfb.sh
-        pytest -s tools/sim/tests/test_metadrive_bridge.py
-
-  create_ui_report:
-    name: Create UI Report
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-      - run: ./tools/op.sh setup
-      - name: Build openpilot
-        run: scons -j$(nproc)
-      - name: Create UI Report
-        run: |
-          source selfdrive/test/setup_xvfb.sh
-          python3 selfdrive/ui/tests/diff/replay.py
-          python3 selfdrive/ui/tests/diff/replay.py --big
-      - name: Upload UI Report
-        uses: actions/upload-artifact@v6
-        with:
-          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -3,25 +3,21 @@ on:
   push:
     branches:
       - master
+      - master-new
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
     branches:
       - 'master'
+      - 'master-new'
     paths:
-      - 'selfdrive/assets/**'
       - 'selfdrive/ui/**'
-      - 'system/ui/**'
   workflow_dispatch:
 
 env:
   UI_JOB_NAME: "Create UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
-  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
-
-  # variant:video_prefix:master_branch
-  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
+  REPORT_NAME: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
 
 jobs:
   preview:
@@ -34,9 +30,8 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
+      - name: Waiting for ui generation to start
+        run: sleep 30
 
       - name: Waiting for ui generation to end
         uses: lewagon/wait-on-check-action@v1.3.4
@@ -53,93 +48,110 @@ jobs:
           echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
       - name: Getting proposed ui
+        id: download-artifact
         uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: ui-report-1-${{ env.REPORT_NAME }}
+          name: report-1-${{ env.REPORT_NAME }}
           path: ${{ github.workspace }}/pr_ui
 
-      - name: Getting mici master ui
-        uses: actions/checkout@v6
+      - name: Getting master ui
+        uses: actions/checkout@v4
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_mici
-          ref: openpilot_master_ui_mici_raylib
-
-      - name: Getting big master ui
-        uses: actions/checkout@v6
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_big
-          ref: openpilot_master_ui_big_raylib
+          path: ${{ github.workspace }}/master_ui
+          ref: openpilot_master_ui
 
       - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video branch <<< "$variant"
-            master_dir="${{ github.workspace }}/master_${name}"
-            cd "$master_dir"
-            git checkout --orphan=new_branch
-            git rm -rf *
-            git branch -D "$branch"
-            git branch -m "$branch"
-            git config user.name "GitHub Actions Bot"
-            git config user.email "<>"
-            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
-            git add .
-            git commit -m "${name} video for commit ${{ env.SHA }}"
-            git push origin "$branch" --force
-          done
+          git checkout --orphan=new_master_ui
+          git rm -rf *
+          git branch -D openpilot_master_ui
+          git branch -m openpilot_master_ui
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui --force
 
-      - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
-
-      - name: Finding diffs
+      - name: Finding diff
         if: github.event_name == 'pull_request_target'
         id: find_diff
-        run: |
-          export PYTHONPATH=${{ github.workspace }}
-          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
+        run: >-
+          sudo apt-get update && sudo apt-get install -y imagemagick
 
-          COMMENT=""
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video _ <<< "$variant"
-            diff_name="${name}_diff"
+          scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device')
+          A=($scenes)
 
-            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
-            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
+          DIFF=""
+          TABLE="<details><summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
 
-            diff_exit_code=0
-            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
-              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
-              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
-              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+            # Check if the master file exists
+            if [ ! -f "${{ github.workspace }}/master_ui/${A[$i]}.png" ]; then
+              # This is a new file in PR UI that doesn't exist in master
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
 
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
 
-            REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
-            if [ $diff_exit_code -eq 0 ]; then
-              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
             else
-              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
             fi
           done
 
-          {
-            echo "COMMENT<<EOF"
-            echo "$COMMENT"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
+          TABLE="${TABLE}</table></details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
 
       - name: Saving proposed ui
         if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_mici
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
@@ -147,29 +159,17 @@ jobs:
           git rm -rf *
           mv ${{ github.workspace }}/pr_ui/* .
           git add .
-          git commit -m "ui videos for PR #${{ github.event.number }}"
+          git commit -m "screenshots for PR #${{ github.event.number }}"
           git push origin ${{ env.BRANCH_NAME }} --force
 
-          # Append diff reports to report files branch
-          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
-          for variant in $VARIANTS; do
-            IFS=':' read -r name _ _ <<< "$variant"
-            diff_name="${name}_diff"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
-            git add "${diff_name}_pr_${{ github.event.number }}.html"
-          done
-          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
-          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-
-      - name: Comment on PR
+      - name: Comment Screenshots on PR
         if: github.event_name == 'pull_request_target'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
-            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
+            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
             ## UI Preview
-            ${{ steps.find_diff.outputs.COMMENT }}
-          comment_tag: run_id_ui_preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_screenshots
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/wait-for-action/action.yaml

@@ -4,7 +4,7 @@ inputs:
   workflow:
     description: 'The workflow file name to monitor'
     required: true
-    default: 'tests.yaml'
+    default: 'selfdrive_tests.yaml'
   branch:
     description: 'The branch to monitor (defaults to current branch)'
     required: false

.gitignore

@@ -10,16 +10,15 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
+model2.png
 a.out
 .hypothesis
-.cache/
-bin/
 
-*.mp4
+/docs_site/
+
 *.dylib
 *.DSYM
 *.d
-*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -36,60 +35,74 @@ bin/
 *.class
 *.pyxbldc
 *.vcd
-*.mo
+*.qm
 *_pyx.cpp
-*.stats
-*.pkl
-*.pkl*
 config.json
+clcache
 compile_commands.json
 compare_runtime*.html
-selfdrive/modeld/models/tg_compiled_flags.json
 
-# build artifacts
-docs_site/
+persist
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
 cereal/messaging/bridge
+selfdrive/mapd/default_speeds_by_region.json
+system/proclogd/proclogd
 selfdrive/ui/translations/tmp
+selfdrive/test/longitudinal_maneuvers/out
 selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 
+notebooks
+hyperthneed
+provisioning
+
 .coverage*
 coverage.xml
 htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/models/*.pkl
+sunnypilot/modeld*/thneed/compile
+sunnypilot/modeld*/models/*.thneed
+sunnypilot/modeld*/models/*.pkl
 
-# openpilot log files
 *.bz2
 *.zst
-*.rlog
 
 build/
 
 !**/.gitkeep
 
+poetry.toml
+Pipfile
 
 ### VisualStudioCode ###
-*.vsix
-.history
-.ionide
 .vscode/*
-.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# agents
-.claude/
-.context/
-PLAN.md
-TASK.md
-CLAUDE.md
-SKILL.md
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
 
 ### JetBrains ###
 !.idea/customTargets.xml

.gitmodules

@@ -6,7 +6,7 @@
   url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/commaai/msgq.git
+  url = https://github.com/sunnypilot/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git
@@ -15,7 +15,7 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/sunnypilot/tinygrad.git
+  url = https://github.com/tinygrad/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
   path = sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.idea/tools/External Tools.xml

@@ -2,7 +2,7 @@
   <tool name="uv Scons Build Debug" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
     <exec>
       <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --ccflags=\&quot;-fno-inline\&quot;&quot;" />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db --ccflags=\&quot;-fno-inline\&quot;&quot;" />
       <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
     </exec>
   </tool>
@@ -16,7 +16,7 @@
   <tool name="uv Scons Build Release" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
     <exec>
       <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc)&quot; " />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db&quot; " />
       <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
     </exec>
   </tool>

.python-version

@@ -1 +0,0 @@
-3.12.13

.run/Build_BIG_UI.run.xml

@@ -1,26 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build_BIG_UI" type="PythonConfigurationType" factoryName="Python">
-    <module name="sunnypilot" />
-    <option name="ENV_FILES" value="" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <envs>
-      <env name="BIG" value="1" />
-    </envs>
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
-    <option name="IS_MODULE_SDK" value="true" />
-    <option name="ADD_CONTENT_ROOTS" value="true" />
-    <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2">
-      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
-    </method>
-  </configuration>
-</component>

.run/Build_SMALL_UI.run.xml

@@ -1,23 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build_SMALL_UI" type="PythonConfigurationType" factoryName="Python">
-    <module name="sunnypilot" />
-    <option name="ENV_FILES" value="" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
-    <option name="IS_MODULE_SDK" value="true" />
-    <option name="ADD_CONTENT_ROOTS" value="true" />
-    <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2">
-      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
-    </method>
-  </configuration>
-</component>

.vscode/launch.json

@@ -23,11 +23,6 @@
       "id": "args",
       "description": "Arguments to pass to the process",
       "type": "promptString"
-    },
-    {
-      "id": "replayArg",
-      "type": "promptString",
-      "description": "Enter route or segment to replay."
     }
   ],
   "configurations": [
@@ -45,44 +40,7 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}"
-    },
-    {
-      "name": "Attach LLDB to Replay drive",
-      "type": "lldb",
-      "request": "attach",
-      "pid": "${command:pickMyProcess}",
-      "sourceMap": {
-        ".": "${workspaceFolder}/opendbc/safety"
-      },
-      "initCommands": [
-        "script import time; time.sleep(3)"
-      ]
-    },
-    {
-      "name": "Replay drive",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
-      "args": [
-        "${input:replayArg}"
-      ],
-      "console": "integratedTerminal",
-      "justMyCode": false,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}"
-      },
-      "subProcess": true,
-      "stopOnEntry": false
-    }
-  ],
-  "compounds": [
-    {
-      "name": "Replay drive + Safety LLDB",
-      "configurations": [
-        "Replay drive",
-        "Attach LLDB to Replay drive"
-      ]
+      "cwd": "${workspaceFolder}",
     }
   ]
 }

Dockerfile.openpilot

@@ -1,38 +1,12 @@
-FROM ubuntu:24.04
+FROM ghcr.io/commaai/openpilot-base:latest
 
 ENV PYTHONUNBUFFERED=1
 
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales && \
-    rm -rf /var/lib/apt/lists/*
+ENV OPENPILOT_PATH=/home/batman/openpilot
 
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-ENV OPENPILOT_PATH=/home/$USER/openpilot
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY --chown=$USER . ${OPENPILOT_PATH}/
+COPY . ${OPENPILOT_PATH}/
 
-ENV UV_BIN="/home/$USER/.local/bin/"
-ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
-ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
-RUN tools/setup_dependencies.sh && \
-    sudo rm -rf /var/lib/apt/lists/*
-
-USER root
-RUN git config --global --add safe.directory '*'
+RUN scons --cache-readonly -j$(nproc)

Dockerfile.openpilot_base

@@ -0,0 +1,82 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED=1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX=1
+
+RUN dbus-uuidgen > /etc/machine-id
+RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /home/$USER
+COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /home/$USER && \
+    tools/install_python_dependencies.sh && \
+    rm -rf tools/ pyproject.toml uv.lock .cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -22,7 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-#export LOGPRINT=debug # this has gotten too spammy...
+export LOGPRINT=debug
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -167,7 +167,7 @@ node {
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
   def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tici', 'release-tizi', 'release-tizi-staging', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
+                         'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
   if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
@@ -178,20 +178,20 @@ node {
 
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh && git push -f origin release-tizi-staging:release-mici-staging"),
+      deviceStage("build release3-staging", "tici-needs-can", [], [
+        step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
     if (env.BRANCH_NAME == '__nightly') {
       parallel (
         'nightly': {
-          deviceStage("build nightly", "tizi-needs-can", [], [
+          deviceStage("build nightly", "tici-needs-can", [], [
             step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
-          deviceStage("build nightly-dev", "tizi-needs-can", [], [
+          deviceStage("build nightly-dev", "tici-needs-can", [], [
             step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
@@ -200,43 +200,63 @@ node {
 
     if (!env.BRANCH_NAME.matches(excludeRegex)) {
     parallel (
+      // tici tests
       'onroad tests': {
-        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
+        deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
           step("check dirty", "release/check-dirty.sh"),
           step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
-        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
+        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
           step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
+          step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py", [diffPaths: ["system/ubloxd/"]]),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
-      'camerad OX03C10': {
-        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
+      'loopback': {
+        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+        ])
+      },
+      'camerad AR0231': {
+        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+      },
+      'camerad OX03C10': {
+        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'sensord': {
-        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
+        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+        ])
+        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
-        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
+        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
           step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
         ])
@@ -244,9 +264,12 @@ node {
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          // TODO: enable once new AGNOS is available
+          // step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),
           step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
         ])
       },

README.md

@@ -3,23 +3,68 @@
 ## 🌞 What is sunnypilot?
 [sunnypilot](https://github.com/sunnyhaibin/sunnypilot) is a fork of comma.ai's openpilot, an open source driver assistance system. sunnypilot offers the user a unique driving experience for over 300+ supported car makes and models with modified behaviors of driving assist engagements. sunnypilot complies with comma.ai's safety rules as accurately as possible.
 
-## 💭 Join our Community Forum
-Join the official sunnypilot community forum to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
-* https://community.sunnypilot.ai/
+## 💭 Join our Discord
+Join the official sunnypilot Discord server to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
+* https://discord.gg/sunnypilot
+
+  ![](https://dcbadge.vercel.app/api/server/wRW3meAgtx?style=flat) ![Discord Shield](https://discordapp.com/api/guilds/880416502577266699/widget.png?style=shield)
 
 ## Documentation
 https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
 
 ## 🚘 Running on a dedicated device in a car
-First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
+* A supported device to run this software
+    * a [comma three](https://comma.ai/shop/products/three) or a [C3X](https://comma.ai/shop/comma-3x)
+* This software
+* One of [the 300+ supported cars](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
+* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
+
+Detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
 
 ## Installation
-Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
+Please refer to [Recommended Branches](#-recommended-branches) to find your preferred/supported branch. This guide will assume you want to install the latest `release-c3` branch.
+
+* sunnypilot not installed or you installed a version before 0.8.17?
+  1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
+  2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
+  3. Input the installation URL per [Recommended Branches](#-recommended-branches). Example: ```release-c3.sunnypilot.ai```.
+  4. Complete the rest of the installation following the onscreen instructions.
+
+* sunnypilot already installed and you installed a version after 0.8.17?
+  1. On the comma three, go to `Settings` ▶️ `Software`.
+  2. At the `Download` option, press `CHECK`. This will fetch the list of latest branches from sunnypilot.
+  3. At the `Target Branch` option, press `SELECT` to open the Target Branch selector.
+  4. Scroll to select the desired branch per  Recommended Branches (see below). Example: `release-c3`
+
+|    Branch    |         Installation URL         |
+|:------------:|:--------------------------------:|
+| `release-c3` | https://release-c3.sunnypilot.ai |
+| `staging-c3` | https://staging-c3.sunnypilot.ai |
+|   `dev-c3`   | https://dev-c3.sunnypilot.ai     |
+
+### If you want to use our newest branches (our rewrite)
+> [!TIP]
+>You can see the rewrite state on our [rewrite project board](https://github.com/orgs/sunnypilot/projects/2), and to install the new branches, you can use the following links
+
+
+> [!IMPORTANT]
+> It is recommended to [re-flash AGNOS](https://flash.comma.ai/) if you intend to downgrade from the new branches.
+> You can still restore the latest sunnylink backup made on the old branches.
+
+|      Branch      |                 Installation URL              |
+|:----------------:|:---------------------------------------------:|
+| `staging-c3-new` | `https://staging-c3-new.sunnypilot.ai`        |
+|   `dev-c3-new`   | `https://dev-c3-new.sunnypilot.ai`            |
+| `custom-branch`  | `https://install.sunnypilot.ai/{branch_name}` |
+| `release-c3-new` |            **Not yet available**.             |
+
+> [!TIP]
+> Do you require further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
 
 ## 🎆 Pull Requests
 We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.
 
-Pull requests should be against the most current `master` branch.
+Pull requests should be against the most current `master-new` branch.
 
 ## 📊 User Data
 

RELEASES.md

@@ -1,60 +1,11 @@
-Version 0.11.1 (2026-04-22)
-========================
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-Version 0.11.0 (2026-03-17)
-========================
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-
-Version 0.10.3 (2025-12-17)
-========================
-* New driving model #36249
-  * New temporal policy architecture
-  * New on-policy training physics noise model
-* New driver monitoring model #36409
-  * Trained on a new dataset, including comma four data
-* Improved inter-process communication memory efficiency
-
-Version 0.10.2 (2025-11-19)
-========================
-* comma four support
-
-Version 0.10.1 (2025-09-08)
-========================
-* New driving model #36276
-  * World Model: removed global localization inputs
-  * World Model: 2x the number of parameters
-  * World Model: trained on 4x the number of segments
-  * VAE Compression Model: new architecture and training objective
-  * Driving Vision Model: trained on 4x the number of segments
-* New Driver Monitoring model #36198
-* Acura TLX 2021 support thanks to MVL!
-* Honda City 2023 support thanks to vanillagorillaa and drFritz!
-* Honda N-Box 2018 support thanks to miettal!
-* Honda Odyssey 2021-25 support thanks to csouers and MVL!
-* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
-
-Version 0.10.0 (2025-08-05)
+Version 0.10.0 (2025-07-07)
 ========================
 * New driving model
-  * New training architecture
-     * Described in our CVPR paper: "Learning to Drive from a World Model"
-     * Longitudinal MPC replaced by E2E planning from World Model in Experimental Mode
-     * Action from lateral MPC as training objective replaced by E2E planning from World Model
-  * Low-speed lead car ground-truth fixes
+  * Lead car ground-truth fixes
+  * Ported over VAE from the MLSIM stack
+  * New training architecture described in CVPR paper
 * Enable live-learned steering actuation delay
 * Opt-in audio recording for dashcam video
-* Acura MDX 2025 support thanks to vanillagorillaa and MVL!
-* Honda Accord 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda CR-V 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda Pilot 2023-25 support thanks to vanillagorillaa and MVL!
 
 Version 0.9.9 (2025-05-23)
 ========================

SConstruct

@@ -3,223 +3,345 @@ import subprocess
 import sys
 import sysconfig
 import platform
-import shlex
-import importlib
 import numpy as np
 
 import SCons.Errors
-from SCons.Defaults import _stripixes
 
 SCons.Warnings.warningAsException(True)
 
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
+TICI = os.path.isfile('/TICI')
+AGNOS = TICI
+
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+SetOption('num_jobs', int(os.cpu_count()/2))
+
+AddOption('--kaitai',
+          action='store_true',
+          help='Regenerate kaitai struct parsers')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
 
-AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
-AddOption('--verbose', action='store_true', default=False, help='show full build commands')
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-# Detect platform
-arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+AddOption('--stock-ui',
+          action='store_true',
+          dest='stock_ui',
+          default=False,
+          help='Build stock openpilot UI instead of sunnypilot UI')
+
+## Architecture name breakdown (arch)
+## - larch64: linux tici aarch64
+## - aarch64: linux pc aarch64
+## - x86_64:  linux pc x64
+## - Darwin:  mac x64 or arm64
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and os.path.isfile('/TICI'):
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and AGNOS:
   arch = "larch64"
-assert arch in [
-  "larch64",  # linux tici arm64
-  "aarch64",  # linux pc arm64
-  "x86_64",   # linux pc x64
-  "Darwin",   # macOS arm64 (x86 not supported)
-]
+assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
 
-pkg_names = ['bzip2', 'capnproto', 'eigen', 'ffmpeg', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
-pkgs = [importlib.import_module(name) for name in pkg_names]
+lenv = {
+  "PATH": os.environ['PATH'],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
 
-
-# ***** enforce a whitelist of system libraries *****
-# this prevents silently relying on a 3rd party package,
-# e.g. apt-installed libusb. all libraries should either
-# be distributed with all Linux distros and macOS, or
-# vendored in commaai/dependencies.
-allowed_system_libs = {
-  "EGL", "GLESv2", "GL",
-  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
-  "dl", "drm", "gbm", "m", "pthread",
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
 
-def _resolve_lib(env, name):
-  for d in env.Flatten(env.get('LIBPATH', [])):
-    p = Dir(str(d)).abspath
-    for ext in ('.a', '.so', '.dylib'):
-      f = File(os.path.join(p, f'lib{name}{ext}'))
-      if f.exists() or f.has_builder():
-        return name
-  if name in allowed_system_libs:
-    return name
-  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
+rpath = lenv["LD_LIBRARY_PATH"].copy()
 
-def _libflags(target, source, env, for_signature):
-  libs = []
-  lp = env.subst('$LIBLITERALPREFIX')
-  for lib in env.Flatten(env.get('LIBS', [])):
-    if isinstance(lib, str):
-      if os.sep in lib or lib.startswith('#'):
-        libs.append(File(lib))
-      elif lib.startswith('-') or (lp and lib.startswith(lp)):
-        libs.append(lib)
-      else:
-        libs.append(_resolve_lib(env, lib))
-    else:
-      libs.append(lib)
-  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
-                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
+if arch == "larch64":
+  cpppath = [
+    "#third_party/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/local/lib",
+    "/system/vendor/lib64",
+    f"#third_party/acados/{arch}/lib",
+  ]
+
+  libpath += [
+    "#third_party/snpe/larch64",
+    "#third_party/libyuv/larch64/lib",
+    "/usr/lib/aarch64-linux-gnu"
+  ]
+  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  rpath += ["/usr/local/lib"]
+else:
+  cflags = []
+  cxxflags = []
+  cpppath = []
+  rpath += []
+
+  # MacOS
+  if arch == "Darwin":
+    libpath = [
+      f"#third_party/libyuv/{arch}/lib",
+      f"#third_party/acados/{arch}/lib",
+      f"{brew_prefix}/lib",
+      f"{brew_prefix}/opt/openssl@3.0/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += [
+      f"{brew_prefix}/include",
+      f"{brew_prefix}/opt/openssl@3.0/include",
+    ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
+  # Linux
+  else:
+    libpath = [
+      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{arch}/lib",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
+
+if GetOption('asan'):
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
+else:
+  ccflags = []
+  ldflags = []
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
+
+if not GetOption('stock_ui'):
+  cflags += ["-DSUNNYPILOT"]
+  cxxflags += ["-DSUNNYPILOT"]
+
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
 
 env = Environment(
-  ENV={
-    "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
-  },
+  ENV=lenv,
   CCFLAGS=[
     "-g",
     "-fPIC",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
+    "-Wshadow",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
     "-Wno-vla-cxx-extension",
-  ],
-  CFLAGS=["-std=gnu11"],
-  CXXFLAGS=["-std=c++1z"],
-  CPPPATH=[
+  ] + cflags + ccflags,
+
+  CPPPATH=cpppath + [
     "#",
-    "#msgq",
-    "#third_party",
-    "#third_party/json11",
-    "#third_party/linux/include",
     "#third_party/acados/include",
     "#third_party/acados/include/blasfeo/include",
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
-    [x.INCLUDE_DIR for x in pkgs],
+    "#third_party/libyuv/include",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/snpe/include",
+    "#third_party",
+    "#msgq",
   ],
-  LIBPATH=[
-    "#common",
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
+  LIBPATH=libpath + [
     "#msgq_repo",
     "#third_party",
     "#selfdrive/pandad",
+    "#common",
     "#rednose/helpers",
-    f"#third_party/acados/{arch}/lib",
-    [x.LIB_DIR for x in pkgs],
   ],
-  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
-if arch != "larch64":
-  env['_LIBFLAGS'] = _libflags
 
-# Arch-specific flags and paths
-if arch == "larch64":
-  env["CC"] = "clang"
-  env["CXX"] = "clang++"
-  env.Append(LIBPATH=[
-    "/usr/lib/aarch64-linux-gnu",
-  ])
-  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
-  env.Append(CCFLAGS=arch_flags)
-  env.Append(CXXFLAGS=arch_flags)
-elif arch == "Darwin":
-  env.Append(LIBPATH=[
-    "/System/Library/Frameworks/OpenGL.framework/Libraries",
-  ])
-  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
-  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+if arch == "Darwin":
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
 
-_extra_cc = shlex.split(GetOption('ccflags') or '')
-if _extra_cc:
-  env.Append(CCFLAGS=_extra_cc)
+env.CompilationDatabase('compile_commands.json')
 
-# no --as-needed on mac linker
-if arch != "Darwin":
-  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
+# Setup cache dir
+default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
 
-# Shorter build output: show brief descriptions instead of full commands.
-# Full command lines are still printed on failure by scons.
-if not GetOption('verbose'):
-  for action, short in (
-    ("CC",     "CC"),
-    ("CXX",    "CXX"),
-    ("LINK",   "LINK"),
-    ("SHCC",   "CC"),
-    ("SHCXX",  "CXX"),
-    ("SHLINK", "LINK"),
-    ("AR",     "AR"),
-    ("RANLIB", "RANLIB"),
-    ("AS",     "AS"),
-  ):
-    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
-
-# progress output
 node_interval = 5
 node_count = 0
 def progress_function(node):
   global node_count
   node_count += node_interval
   sys.stderr.write("progress: %d\n" % node_count)
+
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-# ********** Cython build environment **********
+# Cython build environment
+py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
-envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch')
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
 
-# Setup cache dir
-default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
-cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+qt_libs = []
+if arch == "Darwin":
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
+else:
+  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
+  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
 
-# ********** start building stuff **********
+  qt_env['QTDIR'] = qt_install_prefix
+  qt_dirs = [
+    f"{qt_install_headers}",
+  ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
+  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+qt_env['QT3DIR'] = qt_env['QTDIR']
+qt_env.Tool('qt3')
+
+qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_CORE_LIB",
+  "-DQT_MESSAGELOGCONTEXT",
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui', ]
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+
+Export('env', 'qt_env', 'arch', 'real_arch')
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common')
+Import('_common', '_gpucommon')
+
 common = [_common, 'json11', 'zmq']
-Export('common')
+gpucommon = [_gpucommon]
+
+Export('common', 'gpucommon')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
 # Enable swaglog include in submodules
 env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
 
 SConscript(['cereal/SConscript'])
 
@@ -236,8 +358,14 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
+  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
+if arch != "Darwin":
+  SConscript([
+    'system/logcatd/SConscript',
+    'system/proclogd/SConscript',
+  ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
@@ -245,25 +373,15 @@ if arch == "larch64":
 # Build openpilot
 SConscript(['third_party/SConscript'])
 
-# Build selfdrive
-SConscript([
-  'selfdrive/pandad/SConscript',
-  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
-  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'selfdrive/locationd/SConscript',
-  'selfdrive/modeld/SConscript',
-  'selfdrive/ui/SConscript',
-])
+SConscript(['selfdrive/SConscript'])
 
 SConscript(['sunnypilot/SConscript'])
 
-# Build tools
-if arch != "larch64":
-  SConscript([
-    'tools/replay/SConscript',
-    'tools/cabana/SConscript',
-    'tools/jotpluggler/SConscript',
-  ])
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
+  SConscript(['tools/replay/SConscript'])
+  if arch != "larch64":
+    SConscript(['tools/cabana/SConscript'])
 
-
-env.CompilationDatabase('compile_commands.json')
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

cereal/SConscript

@@ -4,7 +4,7 @@ cereal_dir = Dir('.')
 gen_dir = Dir('gen')
 
 # Build cereal
-schema_files = ['log.capnp', 'car.capnp', 'deprecated.capnp', 'custom.capnp']
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
 env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
             schema_files,
             f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
@@ -13,7 +13,7 @@ cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
 socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 

cereal/custom.capnp

@@ -25,92 +25,8 @@ struct ModularAssistiveDrivingSystem {
   }
 }
 
-struct IntelligentCruiseButtonManagement {
-  state @0 :IntelligentCruiseButtonManagementState;
-  sendButton @1 :SendButtonState;
-  vTarget @2 :Float32;
-
-  enum IntelligentCruiseButtonManagementState {
-    inactive @0;      # No button press or default state
-    preActive @1;     # Pre-active state before transitioning to increasing or decreasing
-    increasing @2;    # Increasing speed
-    decreasing @3;    # Decreasing speed
-    holding @4;       # Holding steady speed
-  }
-
-  enum SendButtonState {
-    none @0;
-    increase @1;
-    decrease @2;
-  }
-}
-
-# Same struct as Log.RadarState.LeadData
-struct LeadData {
-  dRel @0 :Float32;
-  yRel @1 :Float32;
-  vRel @2 :Float32;
-  aRel @3 :Float32;
-  vLead @4 :Float32;
-  dPath @6 :Float32;
-  vLat @7 :Float32;
-  vLeadK @8 :Float32;
-  aLeadK @9 :Float32;
-  fcw @10 :Bool;
-  status @11 :Bool;
-  aLeadTau @12 :Float32;
-  modelProb @13 :Float32;
-  radar @14 :Bool;
-  radarTrackId @15 :Int32 = -1;
-
-  aLeadDEPRECATED @5 :Float32;
-}
-
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
   mads @0 :ModularAssistiveDrivingSystem;
-  intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
-
-  enum AudibleAlert {
-    none @0;
-
-    engage @1;
-    disengage @2;
-    refuse @3;
-
-    warningSoft @4;
-    warningImmediate @5;
-
-    prompt @6;
-    promptRepeat @7;
-    promptDistracted @8;
-
-    # unused, these are reserved for upstream events so we don't collide
-    reserved9 @9;
-    reserved10 @10;
-    reserved11 @11;
-    reserved12 @12;
-    reserved13 @13;
-    reserved14 @14;
-    reserved15 @15;
-    reserved16 @16;
-    reserved17 @17;
-    reserved18 @18;
-    reserved19 @19;
-    reserved20 @20;
-    reserved21 @21;
-    reserved22 @22;
-    reserved23 @23;
-    reserved24 @24;
-    reserved25 @25;
-    reserved26 @26;
-    reserved27 @27;
-    reserved28 @28;
-    reserved29 @29;
-    reserved30 @30;
-
-    promptSingleLow @31;
-    promptSingleHigh @32;
-  }
 }
 
 struct ModelManagerSP @0xaedffd8f31e7b55d {
@@ -153,8 +69,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
       navigation @1;
       vision @2;
       policy @3;
-      offPolicy @4;
-      onPolicy @5;
     }
   }
 
@@ -187,20 +101,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
 
 struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
   dec @0 :DynamicExperimentalControl;
-  longitudinalPlanSource @1 :LongitudinalPlanSource;
-  smartCruiseControl @2 :SmartCruiseControl;
-  speedLimit @3 :SpeedLimit;
-  vTarget @4 :Float32;
-  aTarget @5 :Float32;
-  events @6 :List(OnroadEventSP.Event);
-  e2eAlerts @7 :E2eAlerts;
-  accelPersonality @8 :AccelerationPersonality;
-
-  enum AccelerationPersonality {
-    sport @0;
-    normal @1;
-    eco @2;
-  }
 
   struct DynamicExperimentalControl {
     state @0 :DynamicExperimentalControlState;
@@ -212,97 +112,6 @@ struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
       blended @1;
     }
   }
-
-  struct SmartCruiseControl {
-    vision @0 :Vision;
-    map @1 :Map;
-
-    struct Vision {
-      state @0 :VisionState;
-      vTarget @1 :Float32;
-      aTarget @2 :Float32;
-      currentLateralAccel @3 :Float32;
-      maxPredictedLateralAccel @4 :Float32;
-      enabled @5 :Bool;
-      active @6 :Bool;
-    }
-
-    struct Map {
-      state @0 :MapState;
-      vTarget @1 :Float32;
-      aTarget @2 :Float32;
-      enabled @3 :Bool;
-      active @4 :Bool;
-    }
-
-    enum VisionState {
-      disabled @0; # System disabled or inactive.
-      enabled @1; # No predicted substantial turn on vision range.
-      entering @2; # A substantial turn is predicted ahead, adapting speed to turn comfort levels.
-      turning @3; # Actively turning. Managing acceleration to provide a roll on turn feeling.
-      leaving @4; # Road ahead straightens. Start to allow positive acceleration.
-      overriding @5; # System overriding with manual control.
-    }
-
-    enum MapState {
-      disabled @0; # System disabled or inactive.
-      enabled @1; # No predicted substantial turn on map range.
-      turning @2; # Actively turning. Managing acceleration to provide a roll on turn feeling.
-      overriding @3; # System overriding with manual control.
-    }
-  }
-
-  struct SpeedLimit {
-    resolver @0 :Resolver;
-    assist @1 :Assist;
-
-    struct Resolver {
-      speedLimit @0 :Float32;
-      distToSpeedLimit @1 :Float32;
-      source @2 :Source;
-      speedLimitOffset @3 :Float32;
-      speedLimitLast @4 :Float32;
-      speedLimitFinal @5 :Float32;
-      speedLimitFinalLast @6 :Float32;
-      speedLimitValid @7 :Bool;
-      speedLimitLastValid @8 :Bool;
-    }
-
-    struct Assist {
-      state @0 :AssistState;
-      enabled @1 :Bool;
-      active @2 :Bool;
-      vTarget @3 :Float32;
-      aTarget @4 :Float32;
-    }
-
-    enum Source {
-      none @0;
-      car @1;
-      map @2;
-    }
-
-    enum AssistState {
-      disabled @0;
-      inactive @1; # No speed limit set or not enabled by parameter.
-      preActive @2;
-      pending @3; # Awaiting new speed limit.
-      adapting @4; # Reducing speed to match new speed limit.
-      active @5; # Cruising at speed limit.
-    }
-  }
-
-  enum LongitudinalPlanSource {
-    cruise @0;
-    sccVision @1;
-    sccMap @2;
-    speedLimitAssist @3;
-  }
-
-  struct E2eAlerts {
-    greenLightAlert @0 :Bool;
-    leadDepartAlert @1 :Bool;
-  }
 }
 
 struct OnroadEventSP @0xda96579883444c35 {
@@ -342,22 +151,12 @@ struct OnroadEventSP @0xda96579883444c35 {
     experimentalModeSwitched @14;
     wrongCarModeAlertOnly @15;
     pedalPressedAlertOnly @16;
-    laneTurnLeft @17;
-    laneTurnRight @18;
-    speedLimitPreActive @19;
-    speedLimitActive @20;
-    speedLimitChanged @21;
-    speedLimitPending @22;
-    e2eChime @23;
   }
 }
 
 struct CarParamsSP @0x80ae746ee2596b11 {
   flags @0 :UInt32;        # flags for car specific quirks in sunnypilot
   safetyParam @1 : Int16;  # flags for sunnypilot's custom safety flags
-  pcmCruiseSpeed @3 :Bool;
-  intelligentCruiseButtonManagementAvailable @4 :Bool;
-  enableGasInterceptor @5 :Bool;
 
   neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
 
@@ -375,26 +174,10 @@ struct CarParamsSP @0x80ae746ee2596b11 {
 struct CarControlSP @0xa5cd762cd951a455 {
   mads @0 :ModularAssistiveDrivingSystem;
   params @1 :List(Param);
-  leadOne @2 :LeadData;
-  leadTwo @3 :LeadData;
-  intelligentCruiseButtonManagement @4 :IntelligentCruiseButtonManagement;
 
   struct Param {
     key @0 :Text;
-    type @2 :ParamType;
-    value @3 :Data;
-
-    valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
-  }
-
-  enum ParamType {
-    string @0;
-    bool @1;
-    int @2;
-    float @3;
-    time @4;
-    json @5;
-    bytes @6;
+    value @1 :Text;
   }
 }
 
@@ -441,7 +224,6 @@ struct BackupManagerSP @0xf98d843bfd7004a3 {
 }
 
 struct CarStateSP @0xb86e6369214c01c8 {
-  speedLimit @0 :Float32;
 }
 
 struct LiveMapDataSP @0xf416ec09499d9d19 {
@@ -453,14 +235,7 @@ struct LiveMapDataSP @0xf416ec09499d9d19 {
   roadName @5 :Text;
 }
 
-struct ModelDataV2SP @0xa1680744031fdb2d {
-  laneTurnDirection @0 :TurnDirection;
-
-  enum TurnDirection {
-    none @0;
-    turnLeft @1;
-    turnRight @2;
-  }
+struct CustomReserved9 @0xa1680744031fdb2d {
 }
 
 struct CustomReserved10 @0xcb9fd56c7057593a {

cereal/legacy.capnp

@@ -3,7 +3,7 @@ $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
 
-# deprecated.capnp: a home for deprecated structs
+# legacy.capnp: a home for deprecated structs
 
 struct LogRotate @0x9811e1f38f62f2d1 {
   segmentNum @0 :Int32;
@@ -571,219 +571,4 @@ struct LidarPts @0xe3d6685d4e9d8f7a {
   pkt @4 :Data;
 }
 
-struct LiveTracksDEPRECATED @0xb16f60103159415a {
-  trackId @0 :Int32;
-  dRel @1 :Float32;
-  yRel @2 :Float32;
-  vRel @3 :Float32;
-  aRel @4 :Float32;
-  timeStamp @5 :Float32;
-  status @6 :Float32;
-  currentTime @7 :Float32;
-  stationary @8 :Bool;
-  oncoming @9 :Bool;
-}
 
-struct LiveMpcData @0x92a5e332a85f32a0 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  psi @2 :List(Float32);
-  curvature @3 :List(Float32);
-  qpIterations @4 :UInt32;
-  calculationTime @5 :UInt64;
-  cost @6 :Float64;
-}
-
-struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
-  xEgo @0 :List(Float32);
-  vEgo @1 :List(Float32);
-  aEgo @2 :List(Float32);
-  xLead @3 :List(Float32);
-  vLead @4 :List(Float32);
-  aLead @5 :List(Float32);
-  aLeadTau @6 :Float32;    # lead accel time constant
-  qpIterations @7 :UInt32;
-  mpcId @8 :UInt32;
-  calculationTime @9 :UInt64;
-  cost @10 :Float64;
-}
-
-struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
-  frameId @0 :UInt32;
-  modelExecutionTime @14 :Float32;
-  dspExecutionTime @16 :Float32;
-  rawPredictions @15 :Data;
-
-  faceOrientation @3 :List(Float32);
-  facePosition @4 :List(Float32);
-  faceProb @5 :Float32;
-  leftEyeProb @6 :Float32;
-  rightEyeProb @7 :Float32;
-  leftBlinkProb @8 :Float32;
-  rightBlinkProb @9 :Float32;
-  faceOrientationStd @11 :List(Float32);
-  facePositionStd @12 :List(Float32);
-  sunglassesProb @13 :Float32;
-  poorVision @17 :Float32;
-  partialFace @18 :Float32;
-  distractedPose @19 :Float32;
-  distractedEyes @20 :Float32;
-  eyesOnRoad @21 :Float32;
-  phoneUse @22 :Float32;
-  occludedProb @23 :Float32;
-
-  readyProb @24 :List(Float32);
-  notReadyProb @25 :List(Float32);
-
-  irPwrDEPRECATED @10 :Float32;
-  descriptorDEPRECATED @1 :List(Float32);
-  stdDEPRECATED @2 :Float32;
-}
-
-struct NavModelData @0xac3de5c437be057a {
-  frameId @0 :UInt32;
-  locationMonoTime @6 :UInt64;
-  modelExecutionTime @1 :Float32;
-  dspExecutionTime @2 :Float32;
-  features @3 :List(Float32);
-  # predicted future position
-  position @4 :XYData;
-  desirePrediction @5 :List(Float32);
-
-  # All SI units and in device frame
-  struct XYData @0xbe09e615b2507e26 {
-    x @0 :List(Float32);
-    y @1 :List(Float32);
-    xStd @2 :List(Float32);
-    yStd @3 :List(Float32);
-  }
-}
-
-struct AndroidBuildInfo @0xfe2919d5c21f426c {
-  board @0 :Text;
-  bootloader @1 :Text;
-  brand @2 :Text;
-  device @3 :Text;
-  display @4 :Text;
-  fingerprint @5 :Text;
-  hardware @6 :Text;
-  host @7 :Text;
-  id @8 :Text;
-  manufacturer @9 :Text;
-  model @10 :Text;
-  product @11 :Text;
-  radioVersion @12 :Text;
-  serial @13 :Text;
-  supportedAbis @14 :List(Text);
-  tags @15 :Text;
-  time @16 :Int64;
-  type @17 :Text;
-  user @18 :Text;
-
-  versionCodename @19 :Text;
-  versionRelease @20 :Text;
-  versionSdk @21 :Int32;
-  versionSecurityPatch @22 :Text;
-}
-
-struct AndroidSensor @0x9b513b93a887dbcd {
-  id @0 :Int32;
-  name @1 :Text;
-  vendor @2 :Text;
-  version @3 :Int32;
-  handle @4 :Int32;
-  type @5 :Int32;
-  maxRange @6 :Float32;
-  resolution @7 :Float32;
-  power @8 :Float32;
-  minDelay @9 :Int32;
-  fifoReservedEventCount @10 :UInt32;
-  fifoMaxEventCount @11 :UInt32;
-  stringType @12 :Text;
-  maxDelay @13 :Int32;
-}
-
-struct IosBuildInfo @0xd97e3b28239f5580 {
-  appVersion @0 :Text;
-  appBuild @1 :UInt32;
-  osVersion @2 :Text;
-  deviceModel @3 :Text;
-}
-
-enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
-  unknown @0;
-  neo @1;
-  chffrAndroid @2;
-  front @3;
-}
-
-struct AndroidCaptureResult @0xbcc3efbac41d2048 {
-  sensitivity @0 :Int32;
-  frameDuration @1 :Int64;
-  exposureTime @2 :Int64;
-  rollingShutterSkew @3 :UInt64;
-  colorCorrectionTransform @4 :List(Int32);
-  colorCorrectionGains @5 :List(Float32);
-  displayRotation @6 :Int8;
-}
-
-enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
-  none @0;
-  client @1;
-  cdp @2;
-  dcp @3;
-}
-
-struct LateralINDIState @0x939463348632375e {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  steeringRateDeg @2 :Float32;
-  steeringAccelDeg @3 :Float32;
-  rateSetPoint @4 :Float32;
-  accelSetPoint @5 :Float32;
-  accelError @6 :Float32;
-  delayedOutput @7 :Float32;
-  delta @8 :Float32;
-  output @9 :Float32;
-  saturated @10 :Bool;
-  steeringAngleDesiredDeg @11 :Float32;
-  steeringRateDesiredDeg @12 :Float32;
-}
-
-struct LateralLQRState @0x9024e2d790c82ade {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  i @2 :Float32;
-  output @3 :Float32;
-  lqrOutput @4 :Float32;
-  saturated @5 :Bool;
-  steeringAngleDesiredDeg @6 :Float32;
-}
-
-struct LateralCurvatureState @0xad9d8095c06f7c61 {
-  active @0 :Bool;
-  actualCurvature @1 :Float32;
-  desiredCurvature @2 :Float32;
-  error @3 :Float32;
-  p @4 :Float32;
-  i @5 :Float32;
-  f @6 :Float32;
-  output @7 :Float32;
-  saturated @8 :Bool;
-}
-
-struct LateralPlannerSolution @0x84caeca5a6b4acfe {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  yaw @2 :List(Float32);
-  yawRate @3 :List(Float32);
-  xStd @4 :List(Float32);
-  yStd @5 :List(Float32);
-  yawStd @6 :List(Float32);
-  yawRateStd @7 :List(Float32);
-}
-
-struct GpsTrajectory @0x8cfeb072f5301000 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-}

cereal/messaging/__init__.py

@@ -1,8 +1,10 @@
 # must be built with scons
-from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
-                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
 import msgq
+
 import os
 import capnp
 import time
@@ -11,25 +13,11 @@ from typing import Optional, List, Union, Dict
 
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def pub_sock(endpoint: str) -> PubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.pub_sock(endpoint, segment_size)
-
-
-def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
-                       timeout=timeout, segment_size=segment_size)
-
-
 def reset_context():
   msgq.context = Context()
 

cereal/messaging/bridge.cc

@@ -25,16 +25,15 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<BridgeZmqPoller>();
-  auto pub_context = std::make_unique<Context>();
-  auto sub_context = std::make_unique<BridgeZmqContext>();
-  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
+  auto poller = std::make_unique<ZMQPoller>();
+  auto pub_context = std::make_unique<MSGQContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
+  std::map<SubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new PubSocket();
-    auto sub_sock = new BridgeZmqSubSocket();
-    size_t queue_size = services.at(endpoint).queue_size;
-    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
+    auto pub_sock = new MSGQPubSocket();
+    auto sub_sock = new ZMQSubSocket();
+    pub_sock->connect(pub_context.get(), endpoint);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/bridge_zmq.cc

@@ -1,170 +0,0 @@
-#include "cereal/messaging/bridge_zmq.h"
-
-#include <cassert>
-#include <cstring>
-#include <unistd.h>
-
-static size_t fnv1a_hash(const std::string &str) {
-  const size_t fnv_prime = 0x100000001b3;
-  size_t hash_value = 0xcbf29ce484222325;
-  for (char c : str) {
-    hash_value ^= (unsigned char)c;
-    hash_value *= fnv_prime;
-  }
-  return hash_value;
-}
-
-// FIXME: This is a hack to get the port number from the socket name, might have collisions.
-static int get_port(std::string endpoint) {
-  size_t hash_value = fnv1a_hash(endpoint);
-  int start_port = 8023;
-  int max_port = 65535;
-  return start_port + (hash_value % (max_port - start_port));
-}
-
-BridgeZmqContext::BridgeZmqContext() {
-  context = zmq_ctx_new();
-}
-
-BridgeZmqContext::~BridgeZmqContext() {
-  if (context != nullptr) {
-    zmq_ctx_term(context);
-  }
-}
-
-void BridgeZmqMessage::init(size_t sz) {
-  size = sz;
-  data = new char[size];
-}
-
-void BridgeZmqMessage::init(char *d, size_t sz) {
-  size = sz;
-  data = new char[size];
-  memcpy(data, d, size);
-}
-
-void BridgeZmqMessage::close() {
-  if (size > 0) {
-    delete[] data;
-  }
-  data = nullptr;
-  size = 0;
-}
-
-BridgeZmqMessage::~BridgeZmqMessage() {
-  close();
-}
-
-int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
-
-  if (conflate) {
-    int arg = 1;
-    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
-  }
-
-  int reconnect_ivl = 500;
-  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
-
-  full_endpoint = "tcp://" + address + ":";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  return zmq_connect(sock, full_endpoint.c_str());
-}
-
-void BridgeZmqSubSocket::setTimeout(int timeout) {
-  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
-}
-
-Message *BridgeZmqSubSocket::receive(bool non_blocking) {
-  zmq_msg_t msg;
-  assert(zmq_msg_init(&msg) == 0);
-
-  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
-  int rc = zmq_msg_recv(&msg, sock, flags);
-
-  Message *ret = nullptr;
-  if (rc >= 0) {
-    ret = new BridgeZmqMessage;
-    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
-  }
-
-  zmq_msg_close(&msg);
-  return ret;
-}
-
-BridgeZmqSubSocket::~BridgeZmqSubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  full_endpoint = "tcp://*:";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
-  pid = getpid();
-
-  return zmq_bind(sock, full_endpoint.c_str());
-}
-
-int BridgeZmqPubSocket::sendMessage(Message *message) {
-  assert(pid == getpid());
-  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
-}
-
-int BridgeZmqPubSocket::send(char *data, size_t size) {
-  assert(pid == getpid());
-  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
-}
-
-BridgeZmqPubSocket::~BridgeZmqPubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
-  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
-  polls[num_polls].socket = socket->getRawSocket();
-  polls[num_polls].events = ZMQ_POLLIN;
-
-  sockets.push_back(socket);
-  num_polls++;
-}
-
-std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
-  std::vector<BridgeZmqSubSocket *> ret;
-
-  int rc = zmq_poll(polls, num_polls, timeout);
-  if (rc < 0) {
-    return ret;
-  }
-
-  for (size_t i = 0; i < num_polls; i++) {
-    if (polls[i].revents) {
-      ret.push_back(sockets[i]);
-    }
-  }
-
-  return ret;
-}

cereal/messaging/bridge_zmq.h

@@ -1,72 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <string>
-#include <vector>
-
-#include <zmq.h>
-
-#include "msgq/ipc.h"
-
-class BridgeZmqContext {
-public:
-  BridgeZmqContext();
-  void *getRawContext() { return context; }
-  ~BridgeZmqContext();
-
-private:
-  void *context = nullptr;
-};
-
-class BridgeZmqMessage : public Message {
-public:
-  void init(size_t size);
-  void init(char *data, size_t size);
-  void close();
-  size_t getSize() { return size; }
-  char *getData() { return data; }
-  ~BridgeZmqMessage();
-
-private:
-  char *data = nullptr;
-  size_t size = 0;
-};
-
-class BridgeZmqSubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
-  void setTimeout(int timeout);
-  Message *receive(bool non_blocking = false);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqSubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-};
-
-class BridgeZmqPubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
-  int sendMessage(Message *message);
-  int send(char *data, size_t size);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqPubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-  int pid = -1;
-};
-
-class BridgeZmqPoller {
-public:
-  void registerSocket(BridgeZmqSubSocket *socket);
-  std::vector<BridgeZmqSubSocket *> poll(int timeout);
-
-private:
-  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
-  std::vector<BridgeZmqSubSocket *> sockets;
-  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
-  size_t num_polls = 0;
-};

cereal/messaging/msgq_to_zmq.cc

@@ -2,7 +2,6 @@
 
 #include <cassert>
 
-#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -22,14 +21,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
-  msgq_context = std::make_unique<Context>();
+  zmq_context = std::make_unique<ZMQContext>();
+  msgq_context = std::make_unique<MSGQContext>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -49,7 +48,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -72,7 +71,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -109,8 +108,7 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -130,7 +128,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,8 +7,9 @@
 #include <string>
 #include <vector>
 
+#define private public
 #include "msgq/impl_msgq.h"
-#include "cereal/messaging/bridge_zmq.h"
+#include "msgq/impl_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -21,16 +22,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<Context> msgq_context;
-  std::unique_ptr<BridgeZmqContext> zmq_context;
+  std::unique_ptr<MSGQContext> msgq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -33,7 +33,7 @@ MessageContext message_context;
 struct SubMaster::SubMessage {
   std::string name;
   SubSocket *socket = nullptr;
-  float freq = 0.0f;
+  int freq = 0;
   bool updated = false, alive = false, valid = false, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,8 +187,7 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    service serv = services.at(std::string(name));
-    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
+    PubSocket *socket = PubSocket::create(message_context.context(), name);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/messaging/tests/test_messaging.py

@@ -5,7 +5,7 @@ import numbers
 import random
 import threading
 import time
-from openpilot.common.parameterized import parameterized
+from parameterized import parameterized
 import pytest
 
 from cereal import log, car
@@ -30,7 +30,7 @@ def zmq_sleep(t=1):
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:
@@ -177,8 +177,8 @@ class TestMessaging:
 
     # wait 5 socket timeouts before sending
     msg = random_carstate()
-    start_time = time.monotonic()
     delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
+    start_time = time.monotonic()
     recvd = messaging.recv_one_retry(sub_sock)
     assert (time.monotonic() - start_time) >= sock_timeout*5
     assert isinstance(recvd, capnp._DynamicStructReader)

cereal/messaging/tests/test_pub_sub_master.py

@@ -86,7 +86,7 @@ class TestSubMaster:
         "cameraOdometry": (20, 10),
         "liveCalibration": (4, 4),
         "carParams": (None, None),
-        "userBookmark": (None, None),
+        "userFlag": (None, None),
       }
 
       for service, (max_freq, min_freq) in checks.items():

cereal/messaging/tests/test_services.py

@@ -1,7 +1,7 @@
 import os
 import tempfile
 from typing import Dict
-from openpilot.common.parameterized import parameterized
+from parameterized import parameterized
 
 import cereal.services as services
 from cereal.services import SERVICE_LIST

cereal/messaging/tests/validate_sp_cereal_upstream.py

@@ -1,256 +1,222 @@
 #!/usr/bin/env python3
-"""Schema-level cereal compat check between sunnypilot and upstream openpilot.
-
-Rules (per struct matched across sides by typeId):
-  R1  shared ordinal must reference the same type.
-  R2  sunnypilot-only ordinal in a union -> FAIL (unknown discriminant upstream).
-  R3  sunnypilot-only ordinal on a regular field -> OK (additive struct evolution).
-  R4  upstream-only ordinal -> OK.
-  R5  sunnypilot-only struct referenced via an upstream-shared field -> FAIL.
-"""
-
-from __future__ import annotations
-
 import argparse
-import json
-import os
 import sys
-from typing import Any
+from typing import Any, List, Tuple
 
-NO_DISCRIMINANT = 0xFFFF
+DEBUG = False
 
 
-def hex_id(value: int) -> str:
-  return f"0x{value:016x}"
+def print_debug(string: str) -> None:
+  if DEBUG:
+    print(string)
 
 
-def encode_type(type_node: Any) -> dict:
-  which = type_node.which()
-  if which == "struct":
-    return {"kind": "struct", "typeId": hex_id(type_node.struct.typeId)}
-  if which == "enum":
-    return {"kind": "enum", "typeId": hex_id(type_node.enum.typeId)}
-  if which == "interface":
-    return {"kind": "interface", "typeId": hex_id(type_node.interface.typeId)}
-  if which == "list":
-    return {"kind": "list", "element": encode_type(type_node.list.elementType)}
-  if which == "anyPointer":
-    return {"kind": "anyPointer"}
-  return {"kind": which}
+def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
+  """
+  Create a new instance of a schema type, handling different field types.
+
+  Args:
+      struct: The Cap'n Proto schema structure
+      prop: A tuple containing the field name and field metadata
+
+  Returns:
+      A new initialized schema instance
+  """
+  struct_instance = struct.new_message()
+  field_name, field_metadata = prop
+
+  try:
+    field_type = field_metadata.proto.slot.type.which()
+
+    # Initialize different types of fields
+    if field_type in ('list', 'text', 'data'):
+      struct_instance.init(field_name, 1)
+      print_debug(f"Initialized list/text/data field: {field_name}")
+    elif field_type in ('struct', 'object'):
+      struct_instance.init(field_name)
+      print_debug(f"Initialized struct/object field: {field_name}")
+
+    return struct_instance
+
+  except Exception as e:
+    print(f"Error creating instance for {field_name}: {e}")
+    return None
 
 
-def encode_field(name: str, field: Any) -> dict:
-  proto = field.proto
-  ordinal = proto.ordinal.explicit if proto.ordinal.which() == "explicit" else None
-  discriminant = proto.discriminantValue if proto.discriminantValue != NO_DISCRIMINANT else None
+def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
+  """
+  Retrieve all fields from a given schema structure.
 
-  if proto.which() == "group":
-    type_desc = {"kind": "group", "typeId": hex_id(proto.group.typeId)}
-  else:
-    type_desc = encode_type(proto.slot.type)
+  Args:
+      schema_struct: The Cap'n Proto schema structure
 
-  return {
-    "name": name,
-    "ordinal": ordinal,
-    "discriminant": discriminant,
-    "type": type_desc,
-  }
+  Returns:
+      A list of field names and their metadata
+  """
+  try:
+    # Get all fields from the schema
+    schema_fields = list(schema_struct.schema.fields.items())
+
+    print_debug("Discovered schema fields:")
+    for field_name, field_metadata in schema_fields:
+      print_debug(f"- {field_name}")
+
+    return schema_fields
+
+  except Exception as e:
+    print(f"Error retrieving schema fields: {e}")
+    return []
 
 
-def encode_struct(schema: Any) -> dict:
-  node = schema.node
-  return {
-    "typeId": hex_id(node.id),
-    "displayName": node.displayName,
-    "hasUnion": node.struct.discriminantCount > 0,
-    "fields": [encode_field(name, field) for name, field in schema.fields.items()],
-  }
+def generate_schema_instances(schema_struct: Any) -> List[Any]:
+  """
+  Generate instances for all fields in a given schema.
+
+  Args:
+      schema_struct: The Cap'n Proto schema structure
+
+  Returns:
+      A list of schema instances
+  """
+  schema_fields = get_schema_fields(schema_struct)
+  instances = []
+
+  for field_prop in schema_fields:
+    try:
+      instance = create_schema_instance(schema_struct, field_prop)
+      if instance is not None:
+        instances.append(instance)
+    except Exception as e:
+      print(f"Skipping field due to error: {e}")
+
+  print(f"Generated {len(instances)} schema instances")
+  return instances
 
 
-def _child_struct_schema(field: Any) -> Any:
-  proto = field.proto
-  if proto.which() == "group":
-    return field.schema
-  type_node = proto.slot.type
-  which = type_node.which()
-  if which == "struct":
-    return field.schema
-  if which == "list":
-    container = field.schema
-    element_type = type_node.list.elementType
-    while element_type.which() == "list":
-      container = container.elementType
-      element_type = element_type.list.elementType
-    if element_type.which() == "struct":
-      return container.elementType
-  return None
+def persist_instances(instances: List[Any], filename: str) -> None:
+  """
+  Write schema instances to a binary file.
+
+  Args:
+      instances: List of schema instances
+      filename: Output file path
+  """
+  try:
+    with open(filename, 'wb') as f:
+      for instance in instances:
+        f.write(instance.to_bytes())
+
+    print(f"Successfully wrote {len(instances)} instances to {filename}")
+
+  except Exception as e:
+    print(f"Error persisting instances: {e}")
+    sys.exit(1)
 
 
-def collect_schema(root: Any) -> dict[str, dict]:
-  structs: dict[str, dict] = {}
-  stack = [root]
-  while stack:
-    schema = stack.pop()
-    type_id = hex_id(schema.node.id)
-    if type_id in structs:
-      continue
-    structs[type_id] = encode_struct(schema)
-    for _name, field in schema.fields.items():
-      try:
-        child = _child_struct_schema(field)
-      except Exception:
-        child = None
-      if child is not None:
-        stack.append(child)
-  return structs
+def read_instances(filename: str, schema_type: Any) -> List[Any]:
+  """
+  Read schema instances from a binary file.
+
+  Args:
+      filename: Input file path
+      schema_type: The schema type to use for reading
+
+  Returns:
+      A list of read schema instances
+  """
+  try:
+    with open(filename, 'rb') as f:
+      data = f.read()
+
+    instances = list(schema_type.read_multiple_bytes(data))
+
+    print(f"Read {len(instances)} instances from {filename}")
+    return instances
+
+  except Exception as e:
+    print(f"Error reading instances: {e}")
+    sys.exit(1)
 
 
-def load_log(cereal_dir: str) -> Any:
-  import capnp
-  cereal_dir = os.path.abspath(cereal_dir)
-  capnp.remove_import_hook()
-  return capnp.load(os.path.join(cereal_dir, "log.capnp"), imports=[cereal_dir])
+def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
+  """
+  Compare original and read-back instances to detect potential breaking changes.
 
+  Args:
+      original_instances: List of originally generated instances
+      read_instances: List of instances read back from file
 
-def dump_schema(cereal_dir: str, path: str) -> None:
-  log = load_log(cereal_dir)
-  payload = {
-    "root": hex_id(log.Event.schema.node.id),
-    "structs": collect_schema(log.Event.schema),
-  }
-  with open(path, "w", encoding="utf-8") as handle:
-    json.dump(payload, handle, indent=2, sort_keys=True)
-  print(f"wrote schema dump with {len(payload['structs'])} structs to {path}")
-
-
-def types_equal(a: dict, b: dict) -> bool:
-  if a.get("kind") != b.get("kind"):
+  Returns:
+      Boolean indicating whether schemas appear compatible
+  """
+  if len(original_instances) != len(read_instances):
+    print("❌ Schema Compatibility Warning: Instance count mismatch")
     return False
-  kind = a["kind"]
-  if kind in ("struct", "enum", "interface", "group"):
-    return a.get("typeId") == b.get("typeId")
-  if kind == "list":
-    return types_equal(a["element"], b["element"])
-  return True
+
+  compatible = True
+  for struct in read_instances:
+    try:
+      getattr(struct, struct.which())  # Attempting to access the field to validate readability
+    except Exception as e:
+      print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
+      compatible = False
+
+  return compatible
 
 
-def type_repr(t: dict) -> str:
-  kind = t.get("kind", "?")
-  if kind in ("struct", "enum", "interface", "group"):
-    return f"{kind}({t.get('typeId')})"
-  if kind == "list":
-    return f"list<{type_repr(t['element'])}>"
-  return kind
-
-
-def field_is_union_variant(field: dict) -> bool:
-  return field.get("discriminant") is not None
-
-
-def index_fields_by_ordinal(struct: dict) -> dict[int, dict]:
-  indexed: dict[int, dict] = {}
-  for field in struct["fields"]:
-    ordinal = field.get("ordinal")
-    if ordinal is None:
-      continue
-    indexed[ordinal] = field
-  return indexed
-
-
-def compare(sunnypilot_dump: dict, upstream_dump: dict) -> list[str]:
-  violations: list[str] = []
-  sunnypilot_structs: dict[str, dict] = sunnypilot_dump["structs"]
-  upstream_structs: dict[str, dict] = upstream_dump["structs"]
-
-  sunnypilot_struct_referenced_from_shared: set[str] = set()
-
-  for type_id, sunnypilot_struct in sunnypilot_structs.items():
-    upstream_struct = upstream_structs.get(type_id)
-    if upstream_struct is None:
-      continue
-
-    sunnypilot_fields = index_fields_by_ordinal(sunnypilot_struct)
-    upstream_fields = index_fields_by_ordinal(upstream_struct)
-    display = sunnypilot_struct["displayName"]
-
-    for ordinal, sunnypilot_field in sunnypilot_fields.items():
-      upstream_field = upstream_fields.get(ordinal)
-      if upstream_field is None:
-        if field_is_union_variant(sunnypilot_field):
-          violations.append(
-            f"[R2] {display} @{ordinal} ('{sunnypilot_field['name']}', {type_repr(sunnypilot_field['type'])}): "
-            f"union variant not present upstream. upstream cannot parse this discriminant."
-          )
-        continue
-
-      if not types_equal(sunnypilot_field["type"], upstream_field["type"]):
-        violations.append(
-          f"[R1] {display} @{ordinal}: type mismatch. "
-          f"sunnypilot='{sunnypilot_field['name']}' {type_repr(sunnypilot_field['type'])} vs "
-          f"upstream='{upstream_field['name']}' {type_repr(upstream_field['type'])}."
-        )
-        continue
-
-      cursor = sunnypilot_field["type"]
-      while cursor.get("kind") == "list":
-        cursor = cursor["element"]
-      if cursor.get("kind") in ("struct", "group", "interface") and cursor.get("typeId"):
-        sunnypilot_struct_referenced_from_shared.add(cursor["typeId"])
-
-  for type_id, sunnypilot_struct in sunnypilot_structs.items():
-    if type_id in upstream_structs:
-      continue
-    if type_id in sunnypilot_struct_referenced_from_shared:
-      violations.append(
-        f"[R5] struct {sunnypilot_struct['displayName']} ({type_id}) exists only on sunnypilot "
-        f"but is referenced from an upstream-shared field. upstream cannot resolve this type."
-      )
-
-  return violations
-
-
-def load_peer(path: str) -> dict:
-  with open(path, "r", encoding="utf-8") as handle:
-    return json.load(handle)
-
-
-def run_read(cereal_dir: str, peer_path: str) -> int:
-  log = load_log(cereal_dir)
-  peer_dump = load_peer(peer_path)
-  local_dump = {
-    "root": hex_id(log.Event.schema.node.id),
-    "structs": collect_schema(log.Event.schema),
-  }
-  violations = compare(sunnypilot_dump=peer_dump, upstream_dump=local_dump)
-
-  if not violations:
-    print("cereal compat OK: upstream openpilot can parse sunnypilot routes "
-          "(no leaked structs, no ordinal collisions).")
-    return 0
-
-  print(f"cereal compat FAIL: upstream openpilot would misparse sunnypilot routes "
-        f"({len(violations)} violation(s)):")
-  for v in violations:
-    print(f"  {v}")
-  return 1
-
-
-def main() -> int:
+def main():
+  """
+  CLI entry point for schema compatibility testing.
+  """
+  # Setup argument parser
   parser = argparse.ArgumentParser(
-    description="sunnypilot <-> upstream cereal compatibility validator (schema-level)."
+    description='Cap\'n Proto Schema Compatibility Testing Tool',
+    epilog='Test schema compatibility by generating and reading back instances.'
   )
-  mode = parser.add_mutually_exclusive_group(required=True)
-  mode.add_argument("-g", "--generate", action="store_true", help="dump local schema to JSON")
-  mode.add_argument("-r", "--read", action="store_true", help="load peer JSON and diff against local")
-  parser.add_argument("-f", "--file", default="schema.json", help="JSON file path (default: schema.json)")
-  parser.add_argument("--cereal-dir", required=True, help="path to cereal directory containing log.capnp")
+
+  # Add mutually exclusive group for generation or reading mode
+  mode_group = parser.add_mutually_exclusive_group(required=True)
+  mode_group.add_argument('-g', '--generate', action='store_true',
+                          help='Generate schema instances')
+  mode_group.add_argument('-r', '--read', action='store_true',
+                          help='Read and validate schema instances')
+
+  # Common arguments
+  parser.add_argument('-f', '--file',
+                      default='schema_instances.bin',
+                      help='Output/input binary file (default: schema_instances.bin)')
+
+  # Parse arguments
   args = parser.parse_args()
 
+  # Import the schema dynamically 
+  try:
+    from cereal import log
+    schema_type = log.Event
+  except ImportError:
+    print("Error: Unable to import schema. Ensure 'cereal' is installed.")
+    sys.exit(1)
+
+  # Execute based on mode
   if args.generate:
-    dump_schema(args.cereal_dir, args.file)
-    return 0
-  return run_read(args.cereal_dir, args.file)
+    print("🔧 Generating Schema Instances")
+    instances = generate_schema_instances(schema_type)
+    persist_instances(instances, args.file)
+    print("✅ Instance generation complete")
+
+  elif args.read:
+    print("🔍 Reading and Validating Schema Instances")
+    generated_instances = generate_schema_instances(schema_type)
+    read_back_instances = read_instances(args.file, schema_type)
+
+    # Compare schemas
+    if compare_schemas(generated_instances, read_back_instances):
+      print("✅ Schema Compatibility: No breaking changes detected")
+      sys.exit(0)
+    else:
+      print("❌ Potential Schema Breaking Changes Detected")
+      sys.exit(1)
 
 
 if __name__ == "__main__":
-  sys.exit(main())
+  main()

cereal/services.py

@@ -1,46 +1,39 @@
 #!/usr/bin/env python3
-from enum import IntEnum
 from typing import Optional
 
 
-# TODO: this should be automatically determined using the capnp schema
-class QueueSize(IntEnum):
-  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
-  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
-  SMALL = 250 * 1024          # 250KB - most services
-
-
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
-               queue_size: QueueSize = QueueSize.SMALL):
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
-    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
+  "accelerometer2": (True, 100., 100),
   "magnetometer": (True, 25.),
   "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
   "touch": (True, 20., 1),
-  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10),
   "selfdriveState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
-  "logMessage": (True, 0., None, QueueSize.BIG),
-  "errorLogMessage": (True, 0., 1, QueueSize.BIG),
+  "sendcan": (True, 100., 139),
+  "logMessage": (True, 0.),
+  "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
   "liveDelay": (True, 4., 1),
@@ -49,9 +42,8 @@ _services: dict[str, tuple] = {
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
-  "lateralManeuverPlan": (True, 20.),
   "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15, QueueSize.BIG),
+  "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
@@ -73,26 +65,20 @@ _services: dict[str, tuple] = {
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., None, QueueSize.BIG),
+  "modelV2": (True, 20.),
   "managerState": (True, 2., 1),
   "uploaderState": (True, 0., 1),
   "navInstruction": (True, 1., 10),
   "navRoute": (True, 0.),
   "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
-  "userBookmark": (True, 0., 1),
+  "userFlag": (True, 0., 1),
   "soundPressure": (True, 10., 10),
   "rawAudioData": (False, 20.),
-  "bookmarkButton": (True, 0., 1),
-  "audioFeedback": (True, 0., 1),
-  "roadEncodeData": (False, 20., None, QueueSize.BIG),
-  "driverEncodeData": (False, 20., None, QueueSize.BIG),
-  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
 
   # sunnypilot
-  "modelManagerSP": (False, 1., 1, QueueSize.BIG),
-  "backupManagerSP": (False, 1., 1, QueueSize.BIG),
+  "modelManagerSP": (False, 1., 1),
+  "backupManagerSP": (False, 1., 1),
   "selfdriveStateSP": (True, 100., 10),
   "longitudinalPlanSP": (True, 20., 10),
   "onroadEventsSP": (True, 1., 1),
@@ -100,19 +86,21 @@ _services: dict[str, tuple] = {
   "carControlSP": (True, 100., 10),
   "carStateSP": (True, 100., 10),
   "liveMapDataSP": (True, 1., 1),
-  "modelDataV2SP": (True, 20., None, QueueSize.BIG),
-  "liveLocationKalman": (True, 20.),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamWideRoadEncodeData": (False, 20.),
+  "livestreamRoadEncodeData": (False, 20.),
+  "livestreamDriverEncodeData": (False, 20.),
   "customReservedRawData0": (True, 0.),
   "customReservedRawData1": (True, 0.),
   "customReservedRawData2": (True, 0.),
@@ -130,13 +118,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
+  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation, v.queue_size)
+    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation)
   h += "};\n"
 
   h += "#endif\n"

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -1,14 +1,21 @@
-Import('env', 'envCython')
+Import('env', 'envCython', 'arch')
 
 common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
-  'ratekeeper.cc',
+  'watchdog.cc',
+  'ratekeeper.cc'
 ]
 
 _common = env.Library('common', common_libs, LIBS="json11")
-Export('_common')
+
+files = [
+  'clutil.cc',
+]
+
+_gpucommon = env.Library('gpucommon', files)
+Export('_common', '_gpucommon')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
@@ -18,6 +25,11 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-common_python = [params_python]
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
 
 Export('common_python')

common/api/__init__.py

@@ -14,13 +14,9 @@ class Api:
   def post(self, *args, **kwargs):
     return self.service.post(*args, **kwargs)
 
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    return self.service.get_token(payload_extra, expiry_hours)
+  def get_token(self, expiry_hours=1):
+    return self.service.get_token(expiry_hours)
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
-  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
-
-
-def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-  return CommaConnectApi(None).get_key_pair()
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, **params)

common/api/base.py

@@ -1,22 +1,18 @@
 import jwt
-import os
 import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
 from openpilot.system.hardware.hw import Paths
 from openpilot.system.version import get_version
 
-# name: jwt signature algorithm
-KEYS = {"id_rsa": "RS256",
-        "id_ecdsa": "ES256"}
-
 
 class BaseApi:
   def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
     self.dongle_id = dongle_id
     self.api_host = api_host
     self.user_agent = user_agent
-    self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
+    with open(f'{Paths.persist_root()}/comma/id_rsa') as f:
+      self.private_key = f.read()
 
   def get(self, *args, **kwargs):
     return self.request('GET', *args, **kwargs)
@@ -27,7 +23,7 @@ class BaseApi:
   def request(self, method, endpoint, timeout=None, access_token=None, **params):
     return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
 
-  def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
+  def _get_token(self, expiry_hours=1, **extra_payload):
     now = datetime.now(UTC).replace(tzinfo=None)
     payload = {
       'identity': self.dongle_id,
@@ -36,22 +32,20 @@ class BaseApi:
       'exp': now + timedelta(hours=expiry_hours),
       **extra_payload
     }
-    if payload_extra is not None:
-      payload.update(payload_extra)
-    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
     if isinstance(token, bytes):
       token = token.decode('utf8')
     return token
 
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    return self._get_token(payload_extra, expiry_hours)
+  def get_token(self, expiry_hours=1):
+    return self._get_token(expiry_hours)
 
   def remove_non_ascii_chars(self, text):
     normalized_text = unicodedata.normalize('NFD', text)
     ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
     return ascii_encoded_text.decode()
 
-  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, json=None, **params):
     headers = {}
     if access_token is not None:
       headers['Authorization'] = "JWT " + access_token
@@ -59,14 +53,4 @@ class BaseApi:
     version = self.remove_non_ascii_chars(get_version())
     headers['User-Agent'] = self.user_agent + version
 
-    # TODO: add session to Api
-    req = requests if session is None else session
-    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
-
-  @staticmethod
-  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-    for key in KEYS:
-      if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
-        with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
-          return KEYS[key], private.read(), public.read()
-    return None, None, None
+    return requests.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)

common/clutil.cc

@@ -0,0 +1,98 @@
+#include "common/clutil.h"
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+
+#include "common/util.h"
+#include "common/swaglog.h"
+
+namespace {  // helper functions
+
+template <typename Func, typename Id, typename Name>
+std::string get_info(Func get_info_func, Id id, Name param_name) {
+  size_t size = 0;
+  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
+  std::string info(size, '\0');
+  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
+  return info;
+}
+inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
+inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
+
+void cl_print_info(cl_platform_id platform, cl_device_id device) {
+  size_t work_group_size = 0;
+  cl_device_type device_type = 0;
+  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
+  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
+  const char *type_str = "Other...";
+  switch (device_type) {
+    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
+    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
+    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
+  }
+
+  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
+  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
+  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
+  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
+  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
+  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
+  LOGD("max work group size: %zu", work_group_size);
+  LOGD("type = %d, %s", (int)device_type, type_str);
+}
+
+void cl_print_build_errors(cl_program program, cl_device_id device) {
+  cl_build_status status;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
+  size_t log_size;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
+  std::string log(log_size, '\0');
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
+
+  LOGE("build failed; status=%d, log: %s", status, log.c_str());
+}
+
+}  // namespace
+
+cl_device_id cl_get_device_id(cl_device_type device_type) {
+  cl_uint num_platforms = 0;
+  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
+  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
+  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
+
+  for (size_t i = 0; i < num_platforms; ++i) {
+    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
+
+    // Get first device
+    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
+      cl_print_info(platform_ids[i], device_id);
+      return device_id;
+    }
+  }
+  LOGE("No valid openCL platform found");
+  assert(0);
+  return nullptr;
+}
+
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
+void cl_release_context(cl_context context) {
+  clReleaseContext(context);
+}
+
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
+  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
+}
+
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
+  const char *csrc = src.c_str();
+  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
+  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
+    cl_print_build_errors(prg, device_id);
+    assert(0);
+  }
+  return prg;
+}

common/clutil.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#include <string>
+
+#define CL_CHECK(_expr)          \
+  do {                           \
+    assert(CL_SUCCESS == (_expr)); \
+  } while (0)
+
+#define CL_CHECK_ERR(_expr)           \
+  ({                                  \
+    cl_int err = CL_INVALID_VALUE;    \
+    __typeof__(_expr) _ret = _expr;   \
+    assert(_ret&& err == CL_SUCCESS); \
+    _ret;                             \
+  })
+
+cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
+void cl_release_context(cl_context context);
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/conversions.py

@@ -1,7 +1,6 @@
 import numpy as np
 
-# conversions
-class CV:
+class Conversions:
   # Speed
   MPH_TO_KPH = 1.609344
   KPH_TO_MPH = 1. / MPH_TO_KPH
@@ -18,6 +17,3 @@ class CV:
 
   # Mass
   LB_TO_KG = 0.453592
-
-
-ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/file_chunker.py

@@ -1,37 +0,0 @@
-import math
-import os
-from pathlib import Path
-
-CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
-
-def get_chunk_name(name, idx, num_chunks):
-  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
-
-def get_manifest_path(name):
-  return f"{name}.chunkmanifest"
-
-def get_chunk_paths(path, file_size):
-  num_chunks = math.ceil(file_size / CHUNK_SIZE)
-  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
-
-def chunk_file(path, targets):
-  manifest_path, *chunk_paths = targets
-  with open(path, 'rb') as f:
-    data = f.read()
-  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
-  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
-  for i, chunk_path in enumerate(chunk_paths):
-    with open(chunk_path, 'wb') as f:
-      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
-  Path(manifest_path).write_text(str(len(chunk_paths)))
-  os.remove(path)
-
-
-def read_file_chunked(path):
-  manifest_path = get_manifest_path(path)
-  if os.path.isfile(manifest_path):
-    num_chunks = int(Path(manifest_path).read_text().strip())
-    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
-  if os.path.isfile(path):
-    return Path(path).read_bytes()
-  raise FileNotFoundError(path)

common/file_helpers.py

@@ -0,0 +1,58 @@
+import io
+import os
+import tempfile
+import contextlib
+import zstandard as zstd
+
+LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+                        overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)
+
+
+def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
+  if not should_compress:
+    file_size = os.path.getsize(filepath)
+    file_stream = open(filepath, "rb")
+    return file_stream, file_size
+
+  # Compress the file on the fly
+  compressed_stream = io.BytesIO()
+  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
+
+  with open(filepath, "rb") as f:
+    compressor.copy_stream(f, compressed_stream)
+    compressed_size = compressed_stream.tell()
+    compressed_stream.seek(0)
+    return compressed_stream, compressed_size

common/filter_simple.py

@@ -15,20 +15,3 @@ class FirstOrderFilter:
       self.initialized = True
       self.x = x
     return self.x
-
-
-class BounceFilter(FirstOrderFilter):
-  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
-    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
-    self.bounce = bounce
-    super().__init__(x0, rc, dt, initialized)
-
-  def update(self, x):
-    super().update(x)
-    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
-    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
-    self.velocity.update(0.0)
-    if abs(self.velocity.x) < 1e-3:
-      self.velocity.x = 0.0
-    self.x += self.velocity.x
-    return self.x

common/git.py

@@ -1,30 +1,30 @@
 from functools import cache
 import subprocess
-from openpilot.common.utils import run_cmd, run_cmd_default
+from openpilot.common.run import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str | None = None) -> str:
+def get_short_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str | None = None) -> str:
+def get_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str | None = None) -> str:
+def get_origin(cwd: str = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
@@ -34,7 +34,7 @@ def get_origin(cwd: str | None = None) -> str:
 
 
 @cache
-def get_normalized_origin(cwd: str | None = None) -> str:
+def get_normalized_origin(cwd: str = None) -> str:
   return get_origin(cwd) \
     .replace("git@", "", 1) \
     .replace(".git", "", 1) \

common/i2c.py

@@ -1,81 +0,0 @@
-import os
-import fcntl
-import ctypes
-
-# I2C constants from /usr/include/linux/i2c-dev.h
-I2C_SLAVE = 0x0703
-I2C_SLAVE_FORCE = 0x0706
-I2C_SMBUS = 0x0720
-
-# SMBus transfer types
-I2C_SMBUS_READ = 1
-I2C_SMBUS_WRITE = 0
-I2C_SMBUS_BYTE_DATA = 2
-I2C_SMBUS_I2C_BLOCK_DATA = 8
-
-I2C_SMBUS_BLOCK_MAX = 32
-
-
-class _I2cSmbusData(ctypes.Union):
-  _fields_ = [
-    ("byte", ctypes.c_uint8),
-    ("word", ctypes.c_uint16),
-    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
-  ]
-
-
-class _I2cSmbusIoctlData(ctypes.Structure):
-  _fields_ = [
-    ("read_write", ctypes.c_uint8),
-    ("command", ctypes.c_uint8),
-    ("size", ctypes.c_uint32),
-    ("data", ctypes.POINTER(_I2cSmbusData)),
-  ]
-
-
-class SMBus:
-  def __init__(self, bus: int):
-    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
-
-  def __enter__(self) -> 'SMBus':
-    return self
-
-  def __exit__(self, *args) -> None:
-    self.close()
-
-  def close(self) -> None:
-    if hasattr(self, '_fd') and self._fd >= 0:
-      os.close(self._fd)
-      self._fd = -1
-
-  def _set_address(self, addr: int, force: bool = False) -> None:
-    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
-    fcntl.ioctl(self._fd, ioctl_arg, addr)
-
-  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
-    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
-    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
-
-  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
-    return int(data.byte)
-
-  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    data.byte = value & 0xFF
-    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
-
-  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
-    self._set_address(addr, force)
-    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
-      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
-
-    data = _I2cSmbusData()
-    data.block[0] = length
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
-    read_len = int(data.block[0]) or length
-    read_len = min(read_len, length)
-    return [int(b) for b in data.block[1 : read_len + 1]]

common/mat.h

@@ -0,0 +1,85 @@
+#pragma once
+
+typedef struct vec3 {
+  float v[3];
+} vec3;
+
+typedef struct vec4 {
+  float v[4];
+} vec4;
+
+typedef struct mat3 {
+  float v[3*3];
+} mat3;
+
+typedef struct mat4 {
+  float v[4*4];
+} mat4;
+
+static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
+  mat3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      float v = 0.0;
+      for (int k=0; k<3; k++) {
+        v += a.v[r*3+k] * b.v[k*3+c];
+      }
+      ret.v[r*3+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
+  vec3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      ret.v[r] += a.v[r*3+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+static inline mat4 matmul(const mat4 &a, const mat4 &b) {
+  mat4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      float v = 0.0;
+      for (int k=0; k<4; k++) {
+        v += a.v[r*4+k] * b.v[k*4+c];
+      }
+      ret.v[r*4+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
+  vec4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      ret.v[r] += a.v[r*4+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+// scales the input and output space of a transformation matrix
+// that assumes pixel-center origin.
+static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
+  // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
+
+  mat3 transform_out = {{
+    1.0f/s, 0.0f, 0.5f,
+    0.0f, 1.0f/s, 0.5f,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  mat3 transform_in = {{
+    s,  0.0f, -0.5f*s,
+    0.0f, s, -0.5f*s,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  return matmul3(transform_in, matmul3(in, transform_out));
+}

common/model.h

@@ -0,0 +1 @@
+#define DEFAULT_MODEL "Tomb Raider 14 (Default)"

common/parameterized.py

@@ -1,47 +0,0 @@
-import sys
-import pytest
-import inspect
-
-
-class parameterized:
-  @staticmethod
-  def expand(cases):
-    cases = list(cases)
-
-    if not cases:
-      return lambda func: pytest.mark.skip("no parameterized cases")(func)
-
-    def decorator(func):
-      params = [p for p in inspect.signature(func).parameters if p != 'self']
-      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
-      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
-      expand_params = params[: len(normalized[0])]
-      if len(expand_params) == 1:
-        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
-      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
-
-    return decorator
-
-
-def parameterized_class(attrs, input_list=None):
-  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
-    params_list = attrs
-  else:
-    assert input_list is not None
-    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
-    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
-
-  def decorator(cls):
-    globs = sys._getframe(1).f_globals
-    for i, params in enumerate(params_list):
-      name = f"{cls.__name__}_{i}"
-      new_cls = type(name, (cls,), dict(params))
-      new_cls.__module__ = cls.__module__
-      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
-      globs[name] = new_cls
-    # Don't collect the un-parametrised base, but return it so outer decorators
-    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
-    cls.__test__ = False
-    return cls
-
-  return decorator

common/params.cc

@@ -103,10 +103,10 @@ Params::~Params() {
   assert(queue.empty());
 }
 
-std::vector<std::string> Params::allKeys(ParamKeyFlag flag) const {
+std::vector<std::string> Params::allKeys(ParamKeyType type) const {
   std::vector<std::string> ret;
   for (auto &p : keys) {
-    if (flag == ALL || (p.second.flags & flag)) {
+    if (type == ALL || (p.second & type)) {
       ret.push_back(p.first);
     }
   }
@@ -117,16 +117,8 @@ bool Params::checkKey(const std::string &key) {
   return keys.find(key) != keys.end();
 }
 
-ParamKeyFlag Params::getKeyFlag(const std::string &key) {
-  return static_cast<ParamKeyFlag>(keys[key].flags);
-}
-
 ParamKeyType Params::getKeyType(const std::string &key) {
-  return keys[key].type;
-}
-
-std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
-  return keys[key].default_value;
+  return static_cast<ParamKeyType>(keys[key]);
 }
 
 int Params::put(const char* key, const char* value, size_t value_size) {
@@ -205,17 +197,17 @@ std::map<std::string, std::string> Params::readAll() {
   return util::read_files_in_dir(getParamPath());
 }
 
-void Params::clearAll(ParamKeyFlag key_flag) {
+void Params::clearAll(ParamKeyType key_type) {
   FileLock file_lock(params_path + "/.lock");
 
-  // 1) delete params of key_flag
+  // 1) delete params of key_type
   // 2) delete files that are not defined in the keys.
   if (DIR *d = opendir(getParamPath().c_str())) {
     struct dirent *de = NULL;
     while ((de = readdir(d))) {
       if (de->d_type != DT_DIR) {
         auto it = keys.find(de->d_name);
-        if (it == keys.end() || (it->second.flags & key_flag)) {
+        if (it == keys.end() || (it->second & key_type)) {
           unlink(getParamPath(de->d_name).c_str());
         }
       }

common/params.h

@@ -2,7 +2,6 @@
 
 #include <future>
 #include <map>
-#include <optional>
 #include <string>
 #include <tuple>
 #include <utility>
@@ -10,34 +9,17 @@
 
 #include "common/queue.h"
 
-enum ParamKeyFlag {
+enum ParamKeyType {
   PERSISTENT = 0x02,
   CLEAR_ON_MANAGER_START = 0x04,
   CLEAR_ON_ONROAD_TRANSITION = 0x08,
   CLEAR_ON_OFFROAD_TRANSITION = 0x10,
   DONT_LOG = 0x20,
   DEVELOPMENT_ONLY = 0x40,
-  CLEAR_ON_IGNITION_ON = 0x80,
-  BACKUP = 0x100,
+  BACKUP = 0x80,
   ALL = 0xFFFFFFFF
 };
 
-enum ParamKeyType {
-  STRING = 0, // must be utf-8 decodable
-  BOOL = 1,
-  INT = 2,
-  FLOAT = 3,
-  TIME = 4, // ISO 8601
-  JSON = 5,
-  BYTES = 6
-};
-
-struct ParamKeyAttributes {
-  uint32_t flags;
-  ParamKeyType type;
-  std::optional<std::string> default_value = std::nullopt;
-};
-
 class Params {
 public:
   explicit Params(const std::string &path = {});
@@ -46,18 +28,16 @@ public:
   Params(const Params&) = delete;
   Params& operator=(const Params&) = delete;
 
-  std::vector<std::string> allKeys(ParamKeyFlag flag = ALL) const;
+  std::vector<std::string> allKeys(ParamKeyType type = ALL) const;
   bool checkKey(const std::string &key);
-  ParamKeyFlag getKeyFlag(const std::string &key);
   ParamKeyType getKeyType(const std::string &key);
-  std::optional<std::string> getKeyDefaultValue(const std::string &key);
   inline std::string getParamPath(const std::string &key = {}) {
     return params_path + params_prefix + (key.empty() ? "" : "/" + key);
   }
 
   // Delete a value
   int remove(const std::string &key);
-  void clearAll(ParamKeyFlag flag);
+  void clearAll(ParamKeyType type);
 
   // helpers for reading values
   std::string get(const std::string &key, bool block = false);

common/params.py

@@ -1,6 +1,5 @@
-from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
 assert Params
-assert ParamKeyFlag
 assert ParamKeyType
 assert UnknownKeyName
 

common/params_keys.h

@@ -3,280 +3,204 @@
 #include <string>
 #include <unordered_map>
 
-#include "cereal/gen/cpp/log.capnp.h"
-
-inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
-    {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
-    {"AdbEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"AlwaysOnDM", {PERSISTENT | BACKUP, BOOL}},
-    {"ApiCache_Device", {PERSISTENT, STRING}},
-    {"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
-    {"AssistNowToken", {PERSISTENT, STRING}},
-    {"AthenadPid", {PERSISTENT, INT}},
-    {"AthenadUploadQueue", {PERSISTENT, JSON}},
-    {"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
-    {"BootCount", {PERSISTENT, INT}},
-    {"CalibrationParams", {PERSISTENT, BYTES}},
-    {"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
-    {"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
-    {"CarBatteryCapacity", {PERSISTENT, INT}},
-    {"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
-    {"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"CarParamsPersistent", {PERSISTENT, BYTES}},
-    {"CarParamsPrevRoute", {PERSISTENT, BYTES}},
-    {"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
-    {"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"CurrentBootlog", {PERSISTENT, STRING}},
-    {"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"DisablePowerDown", {PERSISTENT | BACKUP, BOOL}},
-    {"DisableUpdates", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"DisengageOnAccelerator", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"DongleId", {PERSISTENT, STRING}},
-    {"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
-    {"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
-    {"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
-    {"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"ForcePowerDown", {PERSISTENT, BOOL}},
-    {"GitBranch", {PERSISTENT, STRING}},
-    {"GitCommit", {PERSISTENT, STRING}},
-    {"GitCommitDate", {PERSISTENT, STRING}},
-    {"GitDiff", {PERSISTENT, STRING}},
-    {"GithubSshKeys", {PERSISTENT | BACKUP, STRING}},
-    {"GithubUsername", {PERSISTENT | BACKUP, STRING}},
-    {"GitRemote", {PERSISTENT, STRING}},
-    {"GsmApn", {PERSISTENT | BACKUP, STRING}},
-    {"GsmMetered", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"GsmRoaming", {PERSISTENT | BACKUP, BOOL}},
-    {"HardwareSerial", {PERSISTENT, STRING}},
-    {"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
-    {"InstallDate", {PERSISTENT, TIME}},
-    {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsEngaged", {PERSISTENT, BOOL}},
-    {"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"IsMetric", {PERSISTENT | BACKUP, BOOL}},
-    {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsOnroad", {PERSISTENT, BOOL}},
-    {"IsRhdDetected", {PERSISTENT, BOOL}},
-    {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
-    {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"LastGPSPosition", {PERSISTENT, STRING}},
-    {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
-    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
-    {"LastUpdateTime", {PERSISTENT, TIME}},
-    {"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
-    {"LiveParameters", {PERSISTENT, JSON}},
-    {"LiveParametersV2", {PERSISTENT, BYTES}},
-    {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
-    {"LocationFilterInitialState", {PERSISTENT, BYTES}},
-    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
-    {"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
-    {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
-    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"PrimeType", {PERSISTENT, INT}},
-    {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
-    {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RecordFront", {PERSISTENT | BACKUP, BOOL}},
-    {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
-    {"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
-    {"ShowDebugInfo", {PERSISTENT, BOOL}},
-    {"RouteCount", {PERSISTENT, INT, "0"}},
-    {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"TermsVersion", {PERSISTENT, STRING}},
-    {"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TrainingVersion", {PERSISTENT, STRING}},
-    {"UbloxAvailable", {PERSISTENT, BOOL}},
-    {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
-    {"UpdaterAvailableBranches", {PERSISTENT, STRING}},
-    {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
-    {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"Version", {PERSISTENT, STRING}},
+inline static std::unordered_map<std::string, uint32_t> keys = {
+    {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
+    {"AdbEnabled", PERSISTENT},
+    {"AlwaysOnDM", PERSISTENT},
+    {"ApiCache_Device", PERSISTENT},
+    {"ApiCache_FirehoseStats", PERSISTENT},
+    {"AssistNowToken", PERSISTENT},
+    {"AthenadPid", PERSISTENT},
+    {"AthenadUploadQueue", PERSISTENT},
+    {"AthenadRecentlyViewedRoutes", PERSISTENT},
+    {"BootCount", PERSISTENT},
+    {"CalibrationParams", PERSISTENT},
+    {"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
+    {"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
+    {"CarBatteryCapacity", PERSISTENT},
+    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsPersistent", PERSISTENT},
+    {"CarParamsPrevRoute", PERSISTENT},
+    {"CompletedTrainingVersion", PERSISTENT},
+    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CurrentBootlog", PERSISTENT},
+    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisablePowerDown", PERSISTENT | BACKUP},
+    {"DisableUpdates", PERSISTENT | BACKUP},
+    {"DisengageOnAccelerator", PERSISTENT | BACKUP},
+    {"DongleId", PERSISTENT},
+    {"DoReboot", CLEAR_ON_MANAGER_START},
+    {"DoShutdown", CLEAR_ON_MANAGER_START},
+    {"DoUninstall", CLEAR_ON_MANAGER_START},
+    {"AlphaLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY | BACKUP},
+    {"ExperimentalMode", PERSISTENT | BACKUP},
+    {"ExperimentalModeConfirmed", PERSISTENT | BACKUP},
+    {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ForcePowerDown", PERSISTENT},
+    {"GitBranch", PERSISTENT},
+    {"GitCommit", PERSISTENT},
+    {"GitCommitDate", PERSISTENT},
+    {"GitDiff", PERSISTENT},
+    {"GithubSshKeys", PERSISTENT | BACKUP},
+    {"GithubUsername", PERSISTENT | BACKUP},
+    {"GitRemote", PERSISTENT},
+    {"GsmApn", PERSISTENT | BACKUP},
+    {"GsmMetered", PERSISTENT | BACKUP},
+    {"GsmRoaming", PERSISTENT | BACKUP},
+    {"HardwareSerial", PERSISTENT},
+    {"HasAcceptedTerms", PERSISTENT},
+    {"InstallDate", PERSISTENT},
+    {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
+    {"IsEngaged", PERSISTENT},
+    {"IsLdwEnabled", PERSISTENT | BACKUP},
+    {"IsMetric", PERSISTENT | BACKUP},
+    {"IsOffroad", CLEAR_ON_MANAGER_START},
+    {"IsOnroad", PERSISTENT},
+    {"IsRhdDetected", PERSISTENT},
+    {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
+    {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
+    {"IsTestedBranch", CLEAR_ON_MANAGER_START},
+    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LanguageSetting", PERSISTENT | BACKUP},
+    {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
+    {"LastGPSPosition", PERSISTENT},
+    {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
+    {"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
+    {"LastUpdateException", CLEAR_ON_MANAGER_START},
+    {"LastUpdateTime", PERSISTENT},
+    {"LiveDelay", PERSISTENT | BACKUP},
+    {"LiveParameters", PERSISTENT},
+    {"LiveParametersV2", PERSISTENT},
+    {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
+    {"LocationFilterInitialState", PERSISTENT},
+    {"LongitudinalManeuverMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LongitudinalPersonality", PERSISTENT | BACKUP},
+    {"NetworkMetered", PERSISTENT},
+    {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
+    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
+    {"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
+    {"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
+    {"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
+    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
+    {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
+    {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
+    {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
+    {"OnroadCycleRequested", CLEAR_ON_MANAGER_START},
+    {"OpenpilotEnabledToggle", PERSISTENT | BACKUP},
+    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSignatures", CLEAR_ON_MANAGER_START},
+    {"PrimeType", PERSISTENT},
+    {"RecordAudio", PERSISTENT | BACKUP},
+    {"RecordFront", PERSISTENT | BACKUP},
+    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
+    {"SecOCKey", PERSISTENT | DONT_LOG | BACKUP},
+    {"RouteCount", PERSISTENT},
+    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"SshEnabled", PERSISTENT | BACKUP},
+    {"TermsVersion", PERSISTENT},
+    {"TrainingVersion", PERSISTENT},
+    {"UbloxAvailable", PERSISTENT},
+    {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
+    {"UpdaterAvailableBranches", PERSISTENT},
+    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterState", CLEAR_ON_MANAGER_START},
+    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
+    {"UpdaterLastFetchTime", PERSISTENT},
+    {"Version", PERSISTENT},
 
     // --- sunnypilot params --- //
-    {"AccelPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPlanSP::AccelerationPersonality::NORMAL))}},
-    {"AccelPersonalityEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ApiCache_DriveStats", {PERSISTENT, JSON}},
-    {"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
-    {"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}},  // seconds
-    {"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}},  // MPH or km/h
-    {"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
-    {"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"CarList", {PERSISTENT, JSON}},
-    {"CarParamsSP", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
-    {"CarParamsSPCache", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"CarParamsSPPersistent", {PERSISTENT, BYTES}},
-    {"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
-    {"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
-    {"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
-    {"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
-    {"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
-    {"DeviceBootMode", {PERSISTENT | BACKUP, INT, "0"}},
-    {"DevUIInfo", {PERSISTENT | BACKUP, INT, "0"}},
-    {"EnableCopyparty", {PERSISTENT | BACKUP, BOOL}},
-    {"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
-    {"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
-    {"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
-    {"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
-    {"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
-    {"IsDevelopmentBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsReleaseSpBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"LastGPSPositionLLK", {PERSISTENT, STRING}},
-    {"LeadDepartAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}},
-    {"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}},
-    {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
-    {"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
-    {"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
-    {"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
-    {"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ApiCache_DriveStats", PERSISTENT},
+    {"AutoLaneChangeBsmDelay", PERSISTENT},
+    {"AutoLaneChangeTimer", PERSISTENT},
+    {"BlinkerMinLateralControlSpeed", PERSISTENT | BACKUP},
+    {"BlinkerPauseLateralControl", PERSISTENT | BACKUP},
+    {"CarParamsSP", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsSPCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsSPPersistent", PERSISTENT},
+    {"CarPlatformBundle", PERSISTENT},
+    {"CustomAccIncrementsEnabled", PERSISTENT | BACKUP},
+    {"CustomAccLongPressIncrement", PERSISTENT | BACKUP},
+    {"CustomAccShortPressIncrement", PERSISTENT | BACKUP},
+    {"DeviceBootMode", PERSISTENT | BACKUP},
+    {"EnableGithubRunner", PERSISTENT | BACKUP},
+    {"InteractivityTimeout", PERSISTENT | BACKUP},
+    {"MaxTimeOffroad", PERSISTENT | BACKUP},
+    {"Brightness", PERSISTENT | BACKUP},
+    {"ModelRunnerTypeCache", CLEAR_ON_ONROAD_TRANSITION},
+    {"OffroadMode", CLEAR_ON_MANAGER_START},
+    {"QuietMode", PERSISTENT | BACKUP},
 
     // MADS params
-    {"Mads", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"MadsMainCruiseAllowed", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"MadsSteeringMode", {PERSISTENT | BACKUP, INT, "0"}},
-    {"MadsUnifiedEngagementMode", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"Mads", PERSISTENT | BACKUP},
+    {"MadsMainCruiseAllowed", PERSISTENT | BACKUP},
+    {"MadsSteeringMode", PERSISTENT | BACKUP},
+    {"MadsUnifiedEngagementMode", PERSISTENT | BACKUP},
 
     // Model Manager params
-    {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
-    {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
-    {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
-    {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
-    {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
+    {"DynamicModeldOutputs", PERSISTENT | BACKUP},
+    {"ModelManager_ActiveBundle", PERSISTENT},
+    {"ModelManager_DownloadIndex", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ModelManager_LastSyncTime", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"ModelManager_ModelsCache", PERSISTENT | BACKUP},
 
     // Neural Network Lateral Control
-    {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"NeuralNetworkLateralControl", PERSISTENT | BACKUP},
 
     // sunnylink params
-    {"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
-    {"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"ParamsVersion", {PERSISTENT, INT}},
-    {"SunnylinkCache_Roles", {PERSISTENT, STRING}},
-    {"SunnylinkCache_Users", {PERSISTENT, STRING}},
-    {"SunnylinkDongleId", {PERSISTENT, STRING}},
-    {"SunnylinkdPid", {PERSISTENT, INT}},
-    {"SunnylinkEnabled", {PERSISTENT, BOOL, "1"}},
-    {"SunnylinkTempFault", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL, "0"}},
+    {"EnableSunnylinkUploader", PERSISTENT | BACKUP},
+    {"LastSunnylinkPingTime", CLEAR_ON_MANAGER_START},
+    {"SunnylinkCache_Roles", PERSISTENT},
+    {"SunnylinkCache_Users", PERSISTENT},
+    {"SunnylinkDongleId", PERSISTENT},
+    {"SunnylinkdPid", PERSISTENT},
+    {"SunnylinkEnabled", PERSISTENT},
 
     // Backup Manager params
-    {"BackupManager_CreateBackup", {PERSISTENT, BOOL}},
-    {"BackupManager_RestoreVersion", {PERSISTENT, STRING}},
+    {"BackupManager_CreateBackup", PERSISTENT},
+    {"BackupManager_RestoreVersion", PERSISTENT},
 
     // sunnypilot car specific params
-    {"HyundaiLongitudinalTuning", {PERSISTENT | BACKUP, INT, "0"}},
-    {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"HyundaiLongitudinalTuning", PERSISTENT},
 
-    {"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"DynamicExperimentalControl", PERSISTENT},
+    {"BlindSpot", PERSISTENT | BACKUP},
 
-    // sunnypilot model params
-    {"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
-    {"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
-    {"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
-    {"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
-    {"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
+    // model panel params
+    {"LagdToggle", PERSISTENT | BACKUP},
+    {"LagdToggleDesc", PERSISTENT},
+    {"LagdToggledelay", PERSISTENT | BACKUP},
 
     // mapd
-    {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
-    {"MapdVersion", {PERSISTENT, STRING}},
-    {"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}},
-    {"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_OSMUpdateRequired", {CLEAR_ON_MANAGER_START, JSON}},
-    {"OsmDbUpdatesCheck", {CLEAR_ON_MANAGER_START, BOOL}},  // mapd database update happens with device ON, reset on boot
-    {"OSMDownloadBounds", {PERSISTENT, STRING}},
-    {"OsmDownloadedDate", {PERSISTENT, STRING, "0.0"}},
-    {"OSMDownloadLocations", {PERSISTENT, JSON}},
-    {"OSMDownloadProgress", {CLEAR_ON_MANAGER_START, JSON}},
-    {"OsmLocal", {PERSISTENT, BOOL}},
-    {"OsmLocationName", {PERSISTENT, STRING}},
-    {"OsmLocationTitle", {PERSISTENT, STRING}},
-    {"OsmLocationUrl", {PERSISTENT, STRING}},
-    {"OsmStateName", {PERSISTENT, STRING, "All"}},
-    {"OsmStateTitle", {PERSISTENT, STRING}},
-    {"OsmWayTest", {PERSISTENT, STRING}},
-    {"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
-
-    // Speed Limit
-    {"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
-    {"SpeedLimitOffsetType", {PERSISTENT | BACKUP, INT, "0"}},
-    {"SpeedLimitPolicy", {PERSISTENT | BACKUP, INT, "3"}},
-    {"SpeedLimitValueOffset", {PERSISTENT | BACKUP, INT, "0"}},
-
-    // Smart Cruise Control
-    {"MapTargetVelocities", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"SmartCruiseControlMap", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"SmartCruiseControlVision", {PERSISTENT | BACKUP, BOOL, "0"}},
-
-    // Torque lateral control custom params
-    {"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
-    {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
-    {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
-    {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
-    {"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
-    {"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
-    {"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
+    {"MapAdvisorySpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"MapdVersion", PERSISTENT},
+    {"MapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"NextMapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_OSMUpdateRequired", CLEAR_ON_MANAGER_START},
+    {"OsmDbUpdatesCheck", CLEAR_ON_MANAGER_START},  // mapd database update happens with device ON, reset on boot
+    {"OSMDownloadBounds", PERSISTENT},
+    {"OsmDownloadedDate", PERSISTENT},
+    {"OSMDownloadLocations", PERSISTENT},
+    {"OSMDownloadProgress", CLEAR_ON_MANAGER_START},
+    {"OsmLocal", PERSISTENT},
+    {"OsmLocationName", PERSISTENT},
+    {"OsmLocationTitle", PERSISTENT},
+    {"OsmLocationUrl", PERSISTENT},
+    {"OsmStateName", PERSISTENT},
+    {"OsmStateTitle", PERSISTENT},
+    {"OsmWayTest", PERSISTENT},
+    {"RoadName", CLEAR_ON_ONROAD_TRANSITION},
 };

common/params_pyx.pyx

@@ -1,35 +1,19 @@
 # distutils: language = c++
 # cython: language_level = 3
-import builtins
-import datetime
-import json
 from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
-from libcpp.optional cimport optional
-
-from openpilot.common.swaglog import cloudlog
 
 cdef extern from "common/params.h":
-  cpdef enum ParamKeyFlag:
+  cpdef enum ParamKeyType:
     PERSISTENT
     CLEAR_ON_MANAGER_START
     CLEAR_ON_ONROAD_TRANSITION
     CLEAR_ON_OFFROAD_TRANSITION
     DEVELOPMENT_ONLY
-    CLEAR_ON_IGNITION_ON
     BACKUP
     ALL
 
-  cpdef enum ParamKeyType:
-    STRING
-    BOOL
-    INT
-    FLOAT
-    TIME
-    JSON
-    BYTES
-
   cdef cppclass c_Params "Params":
     c_Params(string) except + nogil
     string get(string, bool) nogil
@@ -40,31 +24,10 @@ cdef extern from "common/params.h":
     void putBoolNonBlocking(string, bool) nogil
     int putBool(string, bool) nogil
     bool checkKey(string) nogil
-    ParamKeyType getKeyType(string) nogil
-    optional[string] getKeyDefaultValue(string) nogil
     string getParamPath(string) nogil
-    void clearAll(ParamKeyFlag)
-    vector[string] allKeys(ParamKeyFlag)
+    void clearAll(ParamKeyType)
+    vector[string] allKeys(ParamKeyType)
 
-PYTHON_2_CPP = {
-  (str, STRING): lambda v: v,
-  (builtins.bool, BOOL): lambda v: "1" if v else "0",
-  (int, INT): str,
-  (float, FLOAT): str,
-  (datetime.datetime, TIME): lambda v: v.isoformat(),
-  (dict, JSON): json.dumps,
-  (list, JSON): json.dumps,
-  (bytes, BYTES): lambda v: v,
-}
-CPP_2_PYTHON = {
-  STRING: lambda v: v.decode("utf-8"),
-  BOOL: lambda v: v == b"1",
-  INT: int,
-  FLOAT: float,
-  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
-  JSON: json.loads,
-  BYTES: lambda v: v,
-}
 
 def ensure_bytes(v):
   return v.encode() if isinstance(v, str) else v
@@ -88,8 +51,8 @@ cdef class Params:
   def __dealloc__(self):
     del self.p
 
-  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
-    self.p.clearAll(tx_flag)
+  def clear_all(self, tx_type=ParamKeyType.ALL):
+    self.p.clearAll(tx_type)
 
   def check_key(self, key):
     key = ensure_bytes(key)
@@ -97,38 +60,21 @@ cdef class Params:
       raise UnknownKeyName(key)
     return key
 
-  def python2cpp(self, proposed_type, expected_type, value, key):
-    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
-    if cast:
-      return cast(value)
-    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
-
-  def _cpp2python(self, t, value, default, key):
-    if value is None:
-      return None
-    try:
-      return CPP_2_PYTHON[t](value)
-    except (KeyError, TypeError, ValueError):
-      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
-      return self._cpp2python(t, default, None, key)
-
-  def get(self, key, bool block=False, bool return_default=False):
+  def get(self, key, bool block=False, encoding=None):
     cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    cdef optional[string] default = self.p.getKeyDefaultValue(k)
     cdef string val
     with nogil:
       val = self.p.get(k, block)
 
-    default_val = (default.value() if default.has_value() else None) if return_default else None
     if val == b"":
       if block:
         # If we got no value while running in blocked mode
         # it means we got an interrupt while waiting
         raise KeyboardInterrupt
       else:
-        return self._cpp2python(t, default_val, None, key)
-    return self._cpp2python(t, val, default_val, key)
+        return None
+
+    return val if encoding is None else val.decode(encoding)
 
   def get_bool(self, key, bool block=False):
     cdef string k = self.check_key(key)
@@ -137,11 +83,6 @@ cdef class Params:
       r = self.p.getBool(k, block)
     return r
 
-  def _put_cast(self, key, dat):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
-
   def put(self, key, dat):
     """
     Warning: This function blocks until the param is written to disk!
@@ -150,7 +91,7 @@ cdef class Params:
     in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
+    cdef string dat_bytes = ensure_bytes(dat)
     with nogil:
       self.p.put(k, dat_bytes)
 
@@ -161,7 +102,7 @@ cdef class Params:
 
   def put_nonblocking(self, key, dat):
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
+    cdef string dat_bytes = ensure_bytes(dat)
     with nogil:
       self.p.putNonBlocking(k, dat_bytes)
 
@@ -179,19 +120,5 @@ cdef class Params:
     cdef string key_bytes = ensure_bytes(key)
     return self.p.getParamPath(key_bytes).decode("utf-8")
 
-  def get_type(self, key):
-    return self.p.getKeyType(self.check_key(key))
-
-  def all_keys(self, flag=ParamKeyFlag.ALL):
-    return self.p.allKeys(flag)
-
-  def get_default_value(self, key):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    cdef optional[string] default = self.p.getKeyDefaultValue(k)
-    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
-
-  def cpp2python(self, key, value):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    return self._cpp2python(t, value, None, key)
+  def all_keys(self, type=ParamKeyType.ALL):
+    return self.p.allKeys(type)

common/pid.py

@@ -2,14 +2,23 @@ import numpy as np
 from numbers import Number
 
 class PIDController:
-  def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
-    self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
-    self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
-    self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
+  def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p = k_p
+    self._k_i = k_i
+    self._k_d = k_d
+    self.k_f = k_f   # feedforward gain
+    if isinstance(self._k_p, Number):
+      self._k_p = [[0], [self._k_p]]
+    if isinstance(self._k_i, Number):
+      self._k_i = [[0], [self._k_i]]
+    if isinstance(self._k_d, Number):
+      self._k_d = [[0], [self._k_d]]
 
-    self.set_limits(pos_limit, neg_limit)
+    self.pos_limit = pos_limit
+    self.neg_limit = neg_limit
 
-    self.i_dt = 1.0 / rate
+    self.i_unwind_rate = 0.3 / rate
+    self.i_rate = 1.0 / rate
     self.speed = 0.0
 
     self.reset()
@@ -26,6 +35,10 @@ class PIDController:
   def k_d(self):
     return np.interp(self.speed, self._k_d[0], self._k_d[1])
 
+  @property
+  def error_integral(self):
+    return self.i/self.k_i
+
   def reset(self):
     self.p = 0.0
     self.i = 0.0
@@ -33,25 +46,25 @@ class PIDController:
     self.f = 0.0
     self.control = 0
 
-  def set_limits(self, pos_limit, neg_limit):
-    self.pos_limit = pos_limit
-    self.neg_limit = neg_limit
-
-  def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
+  def update(self, error, error_rate=0.0, speed=0.0, override=False, feedforward=0., freeze_integrator=False):
     self.speed = speed
-    self.p = self.k_p * float(error)
-    self.d = self.k_d * error_rate
-    self.f = feedforward
 
-    if not freeze_integrator:
-      i = self.i + self.k_i * self.i_dt * error
+    self.p = float(error) * self.k_p
+    self.f = feedforward * self.k_f
+    self.d = error_rate * self.k_d
 
-      # Don't allow windup if already clipping
-      test_control = self.p + i + self.d + self.f
-      i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
-      i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
-      self.i = np.clip(i, i_lowerbound, i_upperbound)
+    if override:
+      self.i -= self.i_unwind_rate * float(np.sign(self.i))
+    else:
+      if not freeze_integrator:
+        self.i = self.i + error * self.k_i * self.i_rate
+
+        # Clip i to prevent exceeding control limits
+        control_no_i = self.p + self.d + self.f
+        control_no_i = np.clip(control_no_i, self.neg_limit, self.pos_limit)
+        self.i = np.clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
 
     control = self.p + self.i + self.d + self.f
+
     self.control = np.clip(control, self.neg_limit, self.pos_limit)
     return self.control

common/prefix.h

@@ -13,11 +13,7 @@ public:
     if (prefix.empty()) {
       prefix = util::random_string(15);
     }
-#ifdef __APPLE__
-    msgq_path = "/tmp/msgq_" + prefix;
-#else
-    msgq_path = "/dev/shm/msgq_" + prefix;
-#endif
+    msgq_path = Path::shm_path() + "/" + prefix;
     bool ret = util::create_directories(msgq_path, 0777);
     assert(ret);
     setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
@@ -27,14 +23,14 @@ public:
     auto param_path = Params().getParamPath();
     if (util::file_exists(param_path)) {
       std::string real_path = util::readlink(param_path);
-      util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
+      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
       unlink(param_path.c_str());
     }
     if (getenv("COMMA_CACHE") == nullptr) {
-      util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
+      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
     }
-    util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
-    util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
+    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
+    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
     unsetenv("OPENPILOT_PREFIX");
   }
 

common/prefix.py

@@ -1,5 +1,4 @@
 import os
-import platform
 import shutil
 import uuid
 
@@ -10,20 +9,20 @@ from openpilot.system.hardware.hw import Paths
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
-  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
     self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
-    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
-    self.create_dirs_on_enter = create_dirs_on_enter
+    self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
     self.clean_dirs_on_exit = clean_dirs_on_exit
     self.shared_download_cache = shared_download_cache
 
   def __enter__(self):
     self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
     os.environ['OPENPILOT_PREFIX'] = self.prefix
-
-    if self.create_dirs_on_enter:
-      self.create_dirs()
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
 
     if self.shared_download_cache:
       os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
@@ -41,13 +40,6 @@ class OpenpilotPrefix:
       pass
     return False
 
-  def create_dirs(self):
-    try:
-      os.mkdir(self.msgq_path)
-    except FileExistsError:
-      pass
-    os.makedirs(Paths.log_root(), exist_ok=True)
-
   def clean_dirs(self):
     symlink_path = Params().get_param_path()
     if os.path.exists(symlink_path):

common/ratekeeper.cc

@@ -6,9 +6,9 @@
 #include "common/timing.h"
 #include "common/util.h"
 
-RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
-    : name(name_),
-      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
+    : name(name),
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
   interval = 1 / rate;
   last_monitor_time = seconds_since_boot();
   next_frame_time = last_monitor_time + interval;

common/realtime.py

@@ -6,7 +6,7 @@ import time
 
 from setproctitle import getproctitle
 
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
 from openpilot.system.hardware import PC
 
 

common/retry.py

@@ -0,0 +1,30 @@
+import time
+import functools
+
+from openpilot.common.swaglog import cloudlog
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          cloudlog.exception(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        cloudlog.error(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator
+
+
+if __name__ == "__main__":
+  @retry(attempts=10)
+  def abc():
+    raise ValueError("abc failed :(")
+  abc()

common/run.py

@@ -0,0 +1,13 @@
+import subprocess
+
+
+def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
+  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
+
+
+def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
+  try:
+    return run_cmd(cmd, cwd=cwd, env=env)
+  except subprocess.CalledProcessError:
+    return default
+

common/swaglog.cc

@@ -15,8 +15,6 @@
 #include "common/version.h"
 #include "system/hardware/hw.h"
 
-#include "sunnypilot/common/version.h"
-
 class SwaglogState {
 public:
   SwaglogState() {
@@ -58,7 +56,7 @@ public:
     if (char* daemon_name = getenv("MANAGER_DAEMON")) {
       ctx_j["daemon"] = daemon_name;
     }
-    ctx_j["version"] = SUNNYPILOT_VERSION;
+    ctx_j["version"] = COMMA_VERSION;
     ctx_j["dirty"] = !getenv("CLEAN");
     ctx_j["device"] = Hardware::get_name();
   }

common/tests/test_file_helpers.py

@@ -1,7 +1,7 @@
 import os
 from uuid import uuid4
 
-from openpilot.common.utils import atomic_write
+from openpilot.common.file_helpers import atomic_write_in_dir
 
 
 class TestFileHelpers:
@@ -15,5 +15,5 @@ class TestFileHelpers:
       assert f.read() == "test"
     os.remove(path)
 
-  def test_atomic_write(self):
-    self.run_atomic_write_func(atomic_write)
+  def test_atomic_write_in_dir(self):
+    self.run_atomic_write_func(atomic_write_in_dir)

common/tests/test_markdown.py

@@ -6,7 +6,7 @@ from openpilot.common.markdown import parse_markdown
 
 class TestMarkdown:
   def test_all_release_notes(self):
-    with open(os.path.join(BASEDIR, "CHANGELOG.md")) as f:
+    with open(os.path.join(BASEDIR, "RELEASES.md")) as f:
       release_notes = f.read().split("\n\n")
       assert len(release_notes) > 10
 

common/tests/test_params.py

@@ -1,11 +1,10 @@
 import pytest
-import datetime
 import os
 import threading
 import time
 import uuid
 
-from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
+from openpilot.common.params import Params, ParamKeyType, UnknownKeyName
 
 class TestParams:
   def setup_method(self):
@@ -13,7 +12,7 @@ class TestParams:
 
   def test_params_put_and_get(self):
     self.params.put("DongleId", "cb38263377b873ee")
-    assert self.params.get("DongleId") == "cb38263377b873ee"
+    assert self.params.get("DongleId") == b"cb38263377b873ee"
 
   def test_params_non_ascii(self):
     st = b"\xe1\x90\xff"
@@ -21,7 +20,7 @@ class TestParams:
     assert self.params.get("CarParams") == st
 
   def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", b"test")
+    self.params.put("CarParams", "test")
     self.params.put("DongleId", "cb38263377b873ee")
     assert self.params.get("CarParams") == b"test"
 
@@ -30,24 +29,24 @@ class TestParams:
       f.write("test")
     assert os.path.isfile(undefined_param)
 
-    self.params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
+    self.params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
     assert self.params.get("CarParams") is None
     assert self.params.get("DongleId") is not None
     assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
     self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", 123)
-    assert self.params.get("DongleId") == "bob"
-    assert self.params.get("AthenadPid") == 123
+    self.params.put("AthenadPid", "123")
+    assert self.params.get("DongleId") == b"bob"
+    assert self.params.get("AthenadPid") == b"123"
 
   def test_params_get_block(self):
     def _delayed_writer():
       time.sleep(0.1)
-      self.params.put("CarParams", b"test")
+      self.params.put("CarParams", "test")
     threading.Thread(target=_delayed_writer).start()
     assert self.params.get("CarParams") is None
-    assert self.params.get("CarParams", block=True) == b"test"
+    assert self.params.get("CarParams", True) == b"test"
 
   def test_params_unknown_key_fails(self):
     with pytest.raises(UnknownKeyName):
@@ -77,17 +76,17 @@ class TestParams:
     self.params.put_bool("IsMetric", False)
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", True)
+    self.params.put("IsMetric", "1")
     assert self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", False)
+    self.params.put("IsMetric", "0")
     assert not self.params.get_bool("IsMetric")
 
   def test_put_non_blocking_with_get_block(self):
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_nonblocking("CarParams", b"test")
+      Params().put_nonblocking("CarParams", "test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
@@ -108,34 +107,3 @@ class TestParams:
     assert len(keys) > 20
     assert len(keys) == len(set(keys))
     assert b"CarParams" in keys
-
-  def test_params_default_value(self):
-    self.params.remove("LanguageSetting")
-    self.params.remove("LongitudinalPersonality")
-    self.params.remove("LiveParameters")
-
-    assert self.params.get("LanguageSetting") is None
-    assert self.params.get("LanguageSetting", return_default=False) is None
-    assert isinstance(self.params.get("LanguageSetting", return_default=True), str)
-    assert isinstance(self.params.get("LongitudinalPersonality", return_default=True), int)
-    assert self.params.get("LiveParameters") is None
-    assert self.params.get("LiveParameters", return_default=True) is None
-
-  def test_params_get_type(self):
-    # json
-    self.params.put("ApiCache_FirehoseStats", {"a": 0})
-    assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
-
-    # int
-    self.params.put("BootCount", 1441)
-    assert self.params.get("BootCount") == 1441
-
-    # bool
-    self.params.put("AdbEnabled", True)
-    assert self.params.get("AdbEnabled")
-    assert isinstance(self.params.get("AdbEnabled"), bool)
-
-    # time
-    now = datetime.datetime.now(datetime.UTC)
-    self.params.put("InstallDate", now)
-    assert self.params.get("InstallDate") == now

common/tests/test_swaglog.cc

@@ -9,8 +9,6 @@
 #include "system/hardware/hw.h"
 #include "third_party/json11/json11.hpp"
 
-#include "sunnypilot/common/version.h"
-
 std::string daemon_name = "testy";
 std::string dongle_id = "test_dongle_id";
 int LINE_NO = 0;
@@ -55,7 +53,7 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {
     REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
     REQUIRE(ctx["dirty"].bool_value() == true);
 
-    REQUIRE(ctx["version"].string_value() == SUNNYPILOT_VERSION);
+    REQUIRE(ctx["version"].string_value() == COMMA_VERSION);
 
     std::string device = Hardware::get_name();
     REQUIRE(ctx["device"].string_value() == device);

common/tests/test_util.cc

@@ -36,7 +36,7 @@ TEST_CASE("util::read_file") {
     REQUIRE(util::read_file(filename).empty());
 
     std::string content = random_bytes(64 * 1024);
-    REQUIRE(write(fd, content.c_str(), content.size()) == (ssize_t)content.size());
+    write(fd, content.c_str(), content.size());
     std::string ret = util::read_file(filename);
     bool equal = (ret == content);
     REQUIRE(equal);
@@ -114,12 +114,12 @@ TEST_CASE("util::safe_fwrite") {
 }
 
 TEST_CASE("util::create_directories") {
-  REQUIRE(system("rm /tmp/test_create_directories -rf") == 0);
+  system("rm /tmp/test_create_directories -rf");
   std::string dir = "/tmp/test_create_directories/a/b/c/d/e/f";
 
-  auto check_dir_permissions = [](const std::string &path, mode_t mode) -> bool {
+  auto check_dir_permissions = [](const std::string &dir, mode_t mode) -> bool {
     struct stat st = {};
-    return stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
+    return stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
   };
 
   SECTION("create_directories") {
@@ -132,7 +132,7 @@ TEST_CASE("util::create_directories") {
   }
   SECTION("a file exists with the same name") {
     REQUIRE(util::create_directories(dir, 0755));
-    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT, 0644);
+    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT);
     REQUIRE(f != -1);
     close(f);
     REQUIRE(util::create_directories(dir + "/file", 0755) == false);

common/time_helpers.py

@@ -2,7 +2,6 @@ import datetime
 from pathlib import Path
 
 MIN_DATE = datetime.datetime(year=2025, month=2, day=21)
-MAX_DATE = datetime.datetime(year=2035, month=1, day=1)
 
 def min_date():
   # on systemd systems, the default time is the systemd build time
@@ -13,4 +12,4 @@ def min_date():
   return MIN_DATE
 
 def system_time_valid():
-  return min_date() < datetime.datetime.now() < MAX_DATE
+  return datetime.datetime.now() > min_date()