Add Dockerfile and related scripts for sunnypilot CI build pipeline

This commit introduces `Dockerfile.sunnypilot`, `.dockerignore` updates, and a new `docker_build_sp.sh` script to support building and testing sunnypilot in a Dockerized environment.

.clang-tidy

@@ -0,0 +1,19 @@
+---
+Checks: '
+bugprone-*,
+-bugprone-integer-division,
+-bugprone-narrowing-conversions,
+performance-*,
+clang-analyzer-*,
+misc-*,
+-misc-unused-parameters,
+modernize-*,
+-modernize-avoid-c-arrays,
+-modernize-deprecated-headers,
+-modernize-use-auto,
+-modernize-use-using,
+-modernize-use-nullptr,
+-modernize-use-trailing-return-type,
+'
+CheckOptions:
+...

.codespellignore

@@ -2,5 +2,3 @@ Wen
 REGIST
 PullRequest
 cancelled
-FOF
-NoO

.dockerignore

@@ -13,6 +13,40 @@
 *.o-*
 *.os
 *.os-*
+*.so
+*.a
 
 venv/
 .venv/
+**/.idea
+**/.hypothesis
+**/.mypy_cache
+
+**/.venv
+**/.venv/
+
+**/.ci_cache
+**/*.rlog
+
+**/Dockerfile*
+**/dockerfile*
+**/build_output
+
+notebooks
+phone
+massivemap
+neos
+installer
+chffr/app2
+chffr/backend/env
+selfdrive/nav
+selfdrive/baseui
+selfdrive/test/simulator2
+**/cache_data
+xx/plus
+xx/community
+xx/projects
+!xx/projects/eon_testing_master
+!xx/projects/map3d
+xx/ops
+xx/junk

.gitattributes

@@ -7,12 +7,10 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
-*.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text

.github/labeler.yaml

@@ -1,6 +1,6 @@
 ci:
   - changed-files:
-    - any-glob-to-all-files: "{.github/**,**/test_*,**/test/**,Jenkinsfile}"
+    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
 
 chore:
   - changed-files:
@@ -16,7 +16,7 @@ simulation:
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
+    - any-glob-to-all-files: '{selfdrive/ui/**,system/ui/**}'
 
 tools:
   - changed-files:

.github/workflows/auto_pr_review.yaml

@@ -1,7 +1,7 @@
 name: "PR review"
 on:
   pull_request_target:
-    types: [ opened, reopened, synchronize, edited ]
+    types: [opened, reopened, synchronize, edited]
 
 jobs:
   labeler:
@@ -12,12 +12,12 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v6
+    - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -29,21 +29,21 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
-        target: /^(?!master$).*/
+        target: /^(?!master-new$).*/
         exclude: /sunnypilot:.*/
         change-to: ${{ github.base_ref }}
         already-exists-action: close_this
-        already-exists-comment: "Your PR should be made against the `master` branch"
+        already-exists-comment: "Your PR should be made against the `master-new` branch"
 
   update-pr-labels:
     name: Update fork's PR Labels
     runs-on: ubuntu-latest
     if: (github.event.pull_request.head.repo.fork && (contains(github.event_name, 'pull_request') && github.event.action == 'synchronize'))
-    env:
-      PR_LABEL: 'dev'
+    env: 
+      PR_LABEL: 'dev-c3'
       TRUST_FORK_PR_LABEL: 'trust-fork-pr'
     steps:
-      - name: Check if PR has dev label
+      - name: Check if PR has dev-c3 label
         id: check-labels
         uses: actions/github-script@v7
         with:
@@ -55,33 +55,33 @@ jobs:
               repo: context.repo.repo,
               issue_number: prNumber
             });
-
+            
             const hasDevC3Label = labels.some(label => label.name === process.env.PR_LABEL);
             const hasTrustLabel = labels.some(label => label.name === process.env.TRUST_FORK_PR_LABEL);
-
+            
             console.log(`PR #${prNumber} has ${process.env.PR_LABEL} label: ${hasDevC3Label}`);
             console.log(`PR #${prNumber} has ${process.env.TRUST_FORK_PR_LABEL} label: ${hasTrustLabel}`);
-
-            core.setOutput('has-dev', hasDevC3Label ? 'true' : 'false');
+            
+            core.setOutput('has-dev-c3', hasDevC3Label ? 'true' : 'false');
             core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
-
+      
       - name: Remove trust-fork-pr label if present
-        if: steps.check-labels.outputs.has-dev == 'true' && steps.check-labels.outputs.has-trust == 'true'
+        if: steps.check-labels.outputs.has-dev-c3 == 'true' && steps.check-labels.outputs.has-trust == 'true'
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const prNumber = context.payload.pull_request.number;
-
+            
             await github.rest.issues.removeLabel({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: prNumber,
               name: process.env.TRUST_FORK_PR_LABEL
             });
-
+            
             console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
-
+            
             // Add a comment to the PR
             await github.rest.issues.createComment({
               owner: context.repo.owner,

.github/workflows/badges.yaml

@@ -5,8 +5,8 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: sunnypilot-base
-  DOCKER_REGISTRY: ghcr.io/sunnypilot
+  BASE_IMAGE: openpilot-base
+  DOCKER_REGISTRY: ghcr.io/commaai
   RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
@@ -17,13 +17,13 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
-        ${{ env.RUN }} "python3 selfdrive/ui/translations/create_badges.py"
+        ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py"
 
         rm .gitattributes
 

.github/workflows/build-all-tinygrad-models.yaml

@@ -1,304 +0,0 @@
-name: Build and push all tinygrad models
-
-on:
-  workflow_dispatch:
-    inputs:
-      set_min_version:
-        description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
-        required: true
-        type: string
-
-jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      json_version: ${{ steps.get-json.outputs.json_version }}
-      recompiled_dir: ${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
-      json_file: ${{ steps.get-json.outputs.json_file }}
-      model_matrix: ${{ steps.set-matrix.outputs.model_matrix }}
-      tinygrad_ref: ${{ steps.get-tinygrad-ref.outputs.tinygrad_ref }}
-    steps:
-      - name: Checkout sunnypilot repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot
-          path: sunnypilot
-          submodules: recursive
-
-      - name: Get tinygrad_repo ref
-        id: get-tinygrad-ref
-        run: |
-          cd sunnypilot
-          export PYTHONPATH=$(pwd)
-          ref=$(python3 sunnypilot/models/tinygrad_ref.py)
-          echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
-          echo "tinygrad_ref is $ref"
-
-      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-docs
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Get next JSON version to use (from GitHub docs repo)
-        id: get-json
-        run: |
-          cd docs/docs
-          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
-          next=$((latest+1))
-          json_file="driving_models_v${next}.json"
-          cp "driving_models_v${latest}.json" "$json_file"
-          echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
-          echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
-          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
-
-      - name: Extract tinygrad models
-        id: set-matrix
-        working-directory: docs/docs
-        run: |
-          jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
-          echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
-
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-      - run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo and create new recompiled dir
-        id: create-recompiled-dir
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          git checkout main
-          git sparse-checkout set --no-cone models/
-          cd models
-          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          if [[ -z "$latest_dir" ]]; then
-            next_dir=1
-          else
-            next_dir=$((latest_dir+1))
-          fi
-          recompiled_dir="${next_dir}"
-          mkdir -p "recompiled${recompiled_dir}"
-          touch "recompiled${recompiled_dir}/.gitkeep"
-          cd ../..
-          echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
-
-      - name: Push empty recompiled dir to GitLab
-        run: |
-          cd gitlab_docs
-          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
-          git push origin main
-
-      - name: Push new JSON to GitHub docs repo
-        run: |
-          cd docs
-          git pull origin gh-pages
-          git add docs/"$(basename ${{ steps.get-json.outputs.json_file }})"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add new ${{ steps.get-json.outputs.json_file }} for build-all" || echo "No changes to commit"
-          git push origin gh-pages
-
-  get_and_build:
-    needs: [setup]
-    strategy:
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/build-single-tinygrad-model.yaml
-    with:
-      upstream_branch: ${{ matrix.model.ref }}
-      custom_name: ${{ matrix.model.display_name }}
-      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
-      json_version: ${{ needs.setup.outputs.json_version }}
-    secrets: inherit
-
-  retry_failed_models:
-    needs: [setup, get_and_build]
-    runs-on: ubuntu-latest
-    if: ${{ needs.setup.result != 'failure' && !cancelled() }}
-    outputs:
-      retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: model-*
-          path: output
-
-      - id: set-retry-matrix
-        run: |
-          echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
-          built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
-          )
-          jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
-            'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
-          echo "retry_matrix=$(cat retry_matrix.json)" >> $GITHUB_OUTPUT
-
-  retry_get_and_build:
-    needs: [setup, get_and_build, retry_failed_models]
-    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
-    strategy:
-      matrix:
-        model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/build-single-tinygrad-model.yaml
-    with:
-      upstream_branch: ${{ matrix.model.ref }}
-      custom_name: ${{ matrix.model.display_name }}
-      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
-      json_version: ${{ needs.setup.outputs.json_version }}
-      artifact_suffix: -retry
-    secrets: inherit
-
-  publish_models:
-    name: Publish models sequentially
-    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
-    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      max-parallel: 1
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
-    env:
-      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
-      JSON_FILE: ${{ needs.setup.outputs.json_file }}
-      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
-    steps:
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab.com SSH key to known_hosts
-        run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
-          git checkout main
-          cd ..
-
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-docs
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Validate recompiled dir and JSON version
-        run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
-          if [ ! -f "$JSON_FILE" ]; then
-            echo "JSON file $JSON_FILE does not exist!"
-            exit 1
-          fi
-
-      - name: Download artifact name file
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: read-artifact-name
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
-        run: |
-          find output -type f -name '*.onnx' -delete
-          find output -type f -name 'big_*.pkl' -delete
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-
-      - name: Copy model artifacts to gitlab
-        env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
-        run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
-
-      - name: Push recompiled dir to GitLab
-        env:
-          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-        run: |
-          cd gitlab_docs
-          git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
-          git push origin main
-      - run: |
-          cd docs
-          git pull origin gh-pages
-
-      - name: update json
-        run: |
-          ARGS=""
-          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
-          ARGS="$ARGS --sort-by-date"
-          ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
-          eval python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            $ARGS
-
-      - name: Push updated json to GitHub
-        run: |
-          cd docs
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
-          git push origin gh-pages

.github/workflows/build-single-tinygrad-model.yaml

@@ -1,228 +0,0 @@
-name: Build Single Tinygrad Model and Push
-
-on:
-  workflow_call:
-    inputs:
-      upstream_branch:
-        description: 'Upstream commit to build from'
-        required: true
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
-        required: true
-        type: string
-      json_version:
-        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
-        required: true
-        type: string
-      artifact_suffix:
-        description: 'Suffix for artifact name'
-        required: false
-        type: string
-        default: ''
-      bypass_push:
-        description: 'Bypass pushing to GitLab for build-all'
-        required: false
-        default: true
-        type: boolean
-  workflow_dispatch:
-    inputs:
-      upstream_branch:
-        description: 'Upstream commit to build from'
-        required: true
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
-        required: true
-        type: string
-      json_version:
-        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
-        required: true
-        type: string
-      model_folder:
-        description: 'Model folder'
-        type: choice
-        default: 'None'
-        options:
-          - None
-          - Simple Plan Models
-          - Space Lab Models
-          - TR Models
-          - DTR Models
-          - Custom Merge Models
-          - FOF series models
-          - Other
-      custom_model_folder:
-        description: 'Custom model folder name (if "Other" selected)'
-        required: false
-        type: string
-      generation:
-        description: 'Model generation'
-        required: false
-        type: string
-      version:
-        description: 'Minimum selector version'
-        required: false
-        type: string
-env:
-  RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
-  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
-
-jobs:
-  build_model:
-    uses: ./.github/workflows/sunnypilot-build-model.yaml
-    with:
-      upstream_branch: ${{ inputs.upstream_branch }}
-      custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
-      is_20hz: true
-      artifact_suffix: ${{ inputs.artifact_suffix }}
-    secrets: inherit
-
-  publish_model:
-    if: ${{ !inputs.bypass_push && !cancelled() }}
-    concurrency:
-      group: gitlab-push-${{ inputs.recompiled_dir }}
-      cancel-in-progress: false
-    needs: build_model
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab.com SSH key to known_hosts
-        run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
-          git checkout main
-          cd ..
-
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-docs
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Validate recompiled dir and JSON version
-        run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
-          if [ ! -f "$JSON_FILE" ]; then
-            echo "JSON file $JSON_FILE does not exist!"
-            exit 1
-          fi
-
-      - name: Download artifact name file
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: read-artifact-name
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download and extract model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove unwanted files
-        run: |
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-          find output -type f -name 'dmonitoring_model.onnx' -delete
-
-      - name: Copy model artifact(s) to GitLab recompiled dir
-        env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
-        run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
-
-      - name: Push recompiled dir to GitLab
-        env:
-          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-        run: |
-          cd gitlab_docs
-          git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
-          git push origin main
-
-      - run: |
-          cd docs
-          git pull origin gh-pages
-
-      - name: Run json_parser.py to update JSON
-        run: |
-          FOLDER="${{ inputs.model_folder }}"
-          if [ "$FOLDER" = "Other" ]; then
-            FOLDER="${{ inputs.custom_model_folder }}"
-          fi
-          ARGS=""
-          if [ "$FOLDER" != "None" ] && [ -n "$FOLDER" ]; then
-            ARGS="$ARGS --model-folder \"$FOLDER\""
-          fi
-          [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
-          [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
-          eval python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            --sort-by-date \
-            $ARGS
-
-      - name: Push updated JSON to GitHub docs repo
-        run: |
-          cd docs
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
-          git push origin gh-pages

.github/workflows/cereal_validation.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - master-new
   pull_request:
     paths:
       - 'cereal/**'
@@ -16,7 +17,7 @@ on:
         type: string
 
 concurrency:
-  group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:

.github/workflows/ci_weekly_report.yaml

@@ -38,10 +38,10 @@ jobs:
   report:
     needs: [ci_matrix_run]
     runs-on: ubuntu-latest
-    if: always() && github.repository == 'commaai/openpilot'
+    if: always()
     steps:
       - name: Get job results
-        uses: actions/github-script@v8
+        uses: actions/github-script@v7
         id: get-job-results
         with:
           script: |

.github/workflows/ci_weekly_run.yaml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  tests:
-    uses: sunnypilot/sunnypilot/.github/workflows/tests.yaml@master
+  selfdrive_tests:
+    uses: sunnypilot/sunnypilot/.github/workflows/selfdrive_tests.yaml@master
     with:
       run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -15,7 +15,7 @@ runs:
                           scons -j$(nproc) --cache-populate"
     - name: Save scons cache
       uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
       with:
         path: .ci_cache/scons_cache
         key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/docs.yaml

@@ -22,7 +22,7 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
@@ -34,7 +34,7 @@ jobs:
         mkdocs build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,54 +5,14 @@ on:
     types: [created, edited]
 
 jobs:
-  cleanup-branches:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Delete stale Jenkins branches
-        uses: actions/github-script@v8
-        with:
-          script: |
-            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
-            const prefixes = ['tmp-jenkins', '__jenkins'];
-
-            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              per_page: 100,
-            })) {
-              for (const branch of response.data) {
-                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
-
-                const { data: commit } = await github.rest.repos.getCommit({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: branch.commit.sha,
-                });
-
-                const commitDate = new Date(commit.commit.committer.date).getTime();
-                if (commitDate < cutoff) {
-                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
-                  await github.rest.git.deleteRef({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                    ref: `heads/${branch.name}`,
-                  });
-                }
-              }
-            }
-
+  # TODO: gc old branches in a separate job in this workflow
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
-    permissions:
-      contents: write
-      issues: write
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v8
+      uses: actions/github-script@v7
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +32,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -83,14 +43,3 @@ jobs:
         git config --global user.email "github-actions[bot]@users.noreply.github.com"
         git checkout -b tmp-jenkins-${{ github.event.issue.number }}
         GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
-
-    - name: Delete trigger comment
-      if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v8
-      with:
-        script: |
-          await github.rest.issues.deleteComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            comment_id: context.payload.comment.id,
-          });

.github/workflows/lfs-maintenance.yaml

@@ -9,11 +9,11 @@ on:
     - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
   push:
     branches:
-      - 'master'
+      - 'master-new'
   pull_request:
     branches:
-      - 'master'
-  workflow_dispatch: # enables manual triggering
+      - 'master-new'
+  workflow_dispatch:  # enables manual triggering
     inputs:
       upstream_branch:
         default: 'master'
@@ -30,7 +30,7 @@ jobs:
         with:
           repository: 'commaai/openpilot'
           ref: ${{ inputs.upstream_branch }}
-
+      
       - name: LFS Fetch
         run: |
           git lfs fetch
@@ -48,7 +48,7 @@ jobs:
       - name: Add GitLab public keys
         run: |
           ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
+      
       - name: Ensure branch
         run: |
           if git symbolic-ref -q HEAD >/dev/null; then

.github/workflows/mici_raylib_ui_preview.yaml

@@ -1,151 +0,0 @@
-name: "mici raylib ui preview"
-on:
-  push:
-    branches:
-      - master
-  pull_request_target:
-    types: [assigned, opened, synchronize, reopened, edited]
-    branches:
-      - 'master'
-    paths:
-      - 'selfdrive/assets/**'
-      - 'selfdrive/ui/**'
-      - 'system/ui/**'
-  workflow_dispatch:
-
-env:
-  UI_JOB_NAME: "Create mici raylib UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-mici-raylib-ui"
-  MASTER_BRANCH_NAME: "openpilot_master_ui_mici_raylib"
-  # All report files are pushed here
-  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
-
-jobs:
-  preview:
-    if: github.repository == 'sunnypilot/sunnypilot'
-    name: preview
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-
-      - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@v1.3.4
-        with:
-          ref: ${{ env.SHA }}
-          check-name: ${{ env.UI_JOB_NAME }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allowed-conclusions: success
-          wait-interval: 20
-
-      - name: Getting workflow run ID
-        id: get_run_id
-        run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
-
-      - name: Getting proposed ui  # filename: pr_ui/mici_ui_replay.mp4
-        id: download-artifact
-        uses: dawidd6/action-download-artifact@v6
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ steps.get_run_id.outputs.run_id }}
-          search_artifacts: true
-          name: mici-raylib-report-1-${{ env.REPORT_NAME }}
-          path: ${{ github.workspace }}/pr_ui
-
-      - name: Getting master ui  # filename: master_ui_raylib/mici_ui_replay.mp4
-        uses: actions/checkout@v6
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_ui_raylib
-          ref: ${{ env.MASTER_BRANCH_NAME }}
-
-      - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        working-directory: ${{ github.workspace }}/master_ui_raylib
-        run: |
-          git checkout --orphan=new_master_ui_mici_raylib
-          git rm -rf *
-          git branch -D ${{ env.MASTER_BRANCH_NAME }}
-          git branch -m ${{ env.MASTER_BRANCH_NAME }}
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          mv ${{ github.workspace }}/pr_ui/* .
-          git add .
-          git commit -m "mici raylib video for commit ${{ env.SHA }}"
-          git push origin ${{ env.MASTER_BRANCH_NAME }} --force
-
-      - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
-
-      - name: Finding diff
-        if: github.event_name == 'pull_request_target'
-        id: find_diff
-        run: |
-          # Find the video file from PR
-          pr_video="${{ github.workspace }}/pr_ui/mici_ui_replay_proposed.mp4"
-          mv "${{ github.workspace }}/pr_ui/mici_ui_replay.mp4" "$pr_video"
-
-          master_video="${{ github.workspace }}/pr_ui/mici_ui_replay_master.mp4"
-          mv "${{ github.workspace }}/master_ui_raylib/mici_ui_replay.mp4" "$master_video"
-
-          # Run report
-          export PYTHONPATH=${{ github.workspace }}
-          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
-          diff_exit_code=0
-          python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py "${{ github.workspace }}/pr_ui/mici_ui_replay_master.mp4" "${{ github.workspace }}/pr_ui/mici_ui_replay_proposed.mp4" "diff.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
-
-          # Copy diff report files
-          cp ${{ github.workspace }}/selfdrive/ui/tests/diff/report/diff.html ${{ github.workspace }}/pr_ui/
-          cp ${{ github.workspace }}/selfdrive/ui/tests/diff/report/diff.mp4 ${{ github.workspace }}/pr_ui/
-
-          REPORT_URL="https://sunnypilot.github.io/ci-artifacts/diff_pr_${{ github.event.number }}.html"
-          if [ $diff_exit_code -eq 0 ]; then
-            DIFF="✅ Videos are identical! [View Diff Report]($REPORT_URL)"
-          else
-            DIFF="❌ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"
-          fi
-          echo "DIFF=$DIFF" >> "$GITHUB_OUTPUT"
-
-      - name: Saving proposed ui
-        if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_ui_raylib
-        run: |
-          # Overwrite PR branch w/ proposed ui, and master ui at this point in time for future reference
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          git checkout --orphan=${{ env.BRANCH_NAME }}
-          git rm -rf *
-          mv ${{ github.workspace }}/pr_ui/* .
-          git add .
-          git commit -m "mici raylib video for PR #${{ github.event.number }}"
-          git push origin ${{ env.BRANCH_NAME }} --force
-
-          # Append diff report to report files branch
-          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
-          cp ${{ github.workspace }}/selfdrive/ui/tests/diff/report/diff.html diff_pr_${{ github.event.number }}.html
-          git add diff_pr_${{ github.event.number }}.html
-          git commit -m "mici raylib ui diff report for PR #${{ github.event.number }}" || echo "No changes to commit"
-          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-
-      - name: Comment Video on PR
-        if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          message: |
-            <!-- _(run_id_video_mici_raylib **${{ github.run_id }}**)_ -->
-            ## mici raylib UI Preview
-            ${{ steps.find_diff.outputs.DIFF }}
-          comment_tag: run_id_video_mici_raylib
-          pr_number: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/model_review.yaml

@@ -16,9 +16,9 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
       - name: Checkout master
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           ref: master
           path: base

.github/workflows/post-to-discourse/action.yml

@@ -1,105 +0,0 @@
-name: 'Post to Discourse'
-description: 'Posts a message to a Discourse topic (existing or new)'
-
-inputs:
-  discourse-url:
-    description: 'Discourse instance URL (e.g., https://discourse.example.com)'
-    required: true
-  api-key:
-    description: 'Discourse API key'
-    required: true
-  api-username:
-    description: 'Discourse API username'
-    required: true
-  topic-id:
-    description: 'Discourse topic ID to post to (use this OR category-id + title)'
-    required: false
-  category-id:
-    description: 'Category ID for new topic (required if topic-id not provided)'
-    required: false
-  title:
-    description: 'Title for new topic (required if topic-id not provided)'
-    required: false
-  message:
-    description: 'Message content (markdown supported)'
-    required: true
-
-outputs:
-  post-number:
-    description: 'The post number in the topic'
-    value: ${{ steps.post.outputs.post_number }}
-  post-url:
-    description: 'Direct URL to the post'
-    value: ${{ steps.post.outputs.post_url }}
-  topic-id:
-    description: 'The topic ID (useful when creating a new topic)'
-    value: ${{ steps.post.outputs.topic_id }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: Post to Discourse
-      id: post
-      shell: bash
-      run: |
-        # Validate inputs
-        if [ -z "${{ inputs.topic-id }}" ] && ([ -z "${{ inputs.category-id }}" ] || [ -z "${{ inputs.title }}" ]); then
-          echo "❌ Error: Must provide either topic-id OR both category-id and title"
-          exit 1
-        fi
-        
-        if [ -n "${{ inputs.topic-id }}" ] && ([ -n "${{ inputs.category-id }}" ] || [ -n "${{ inputs.title }}" ]); then
-          echo "⚠️  Warning: Both topic-id and category-id/title provided. Will post to existing topic."
-        fi
-        
-        # Determine if creating new topic or posting to existing
-        if [ -n "${{ inputs.topic-id }}" ]; then
-          echo "📝 Posting to existing topic ID: ${{ inputs.topic-id }}"
-        
-          # Create JSON payload for posting to existing topic
-          PAYLOAD=$(jq -n \
-            --arg content '${{ inputs.message }}' \
-            --arg topic_id "${{ inputs.topic-id }}" \
-            '{topic_id: $topic_id, raw: $content}')
-        else
-          echo "✨ Creating new topic: ${{ inputs.title }}"
-        
-          # Create JSON payload for new topic
-          PAYLOAD=$(jq -n \
-            --arg content '${{ inputs.message }}' \
-            --arg title "${{ inputs.title }}" \
-            --arg category "${{ inputs.category-id }}" \
-            '{title: $title, category: ($category | tonumber), raw: $content}')
-        fi
-        
-        # Post to Discourse
-        RESPONSE=$(curl -s -w "\n%{http_code}" \
-          -X POST "${{ inputs.discourse-url }}/posts.json" \
-          -H "Content-Type: application/json" \
-          -H "Api-Key: ${{ inputs.api-key }}" \
-          -H "Api-Username: ${{ inputs.api-username }}" \
-          -d "$PAYLOAD")
-        
-        HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
-        BODY=$(echo "$RESPONSE" | sed '$d')
-        
-        if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
-          echo "✅ Successfully posted to Discourse!"
-        
-          POST_NUMBER=$(echo "$BODY" | jq -r '.post_number // "unknown"')
-          TOPIC_ID=$(echo "$BODY" | jq -r '.topic_id // "${{ inputs.topic-id }}"')
-          POST_URL="${{ inputs.discourse-url }}/t/${TOPIC_ID}/${POST_NUMBER}"
-        
-          echo "post_number=${POST_NUMBER}" >> $GITHUB_OUTPUT
-          echo "post_url=${POST_URL}" >> $GITHUB_OUTPUT
-          echo "topic_id=${TOPIC_ID}" >> $GITHUB_OUTPUT
-        
-          echo "Topic ID: ${TOPIC_ID}"
-          echo "Post number: ${POST_NUMBER}"
-          echo "URL: ${POST_URL}"
-        else
-          echo "❌ Failed to post to Discourse"
-          echo "HTTP Code: ${HTTP_CODE}"
-          echo "Response: ${BODY}"
-          exit 1
-        fi

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh prebuilt
+  BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
   build_prebuilt:
@@ -29,7 +29,7 @@ jobs:
         running-workflow-name: 'build prebuilt'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - run: git lfs pull

.github/workflows/release-drafter.yml

@@ -3,6 +3,7 @@ name: Release Drafter
 on:
   push:
     branches:
+      - master-new
       - master
     tags:
       - 'v*'

.github/workflows/release.yaml

@@ -5,12 +5,12 @@ on:
   workflow_dispatch:
 
 jobs:
-  build___nightly:
-    name: build __nightly
+  build_masterci:
+    name: build master-ci
     env:
       ImageOS: ubuntu24
     container:
-      image: ghcr.io/sunnypilot/sunnypilot-base:latest
+      image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     permissions:
@@ -27,10 +27,10 @@ jobs:
       with:
         ref: master
         wait-interval: 30
-        running-workflow-name: 'build __nightly'
+        running-workflow-name: 'build master-ci'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build prebuilt).*).)*$
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
@@ -38,5 +38,5 @@ jobs:
       run: |
         git config --global --add safe.directory '*'
         git lfs pull
-    - name: Push __nightly
-      run: BRANCH=__nightly release/build_stripped.sh
+    - name: Push master-ci
+      run: BRANCH=__nightly release/build_devel.sh

.github/workflows/repo-maintenance.yaml

@@ -6,24 +6,24 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: sunnypilot-base
-  BUILD: release/ci/docker_build_sp.sh base
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  BASE_IMAGE: openpilot-base
+  BUILD: selfdrive/test/docker_build.sh base
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
   update_translations:
     runs-on: ubuntu-latest
-    if: github.repository == 'sunnypilot/sunnypilot'
+    if: github.repository == 'commaai/openpilot'
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/workflows/setup-with-retry
       - name: Update translations
         run: |
           ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish"
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+        uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
         with:
-          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
+          author: Vehicle Researcher <user@comma.ai>
           commit-message: "Update translations"
           title: "[bot] Update translations"
           body: "Automatic PR from repo-maintenance -> update_translations"
@@ -36,10 +36,10 @@ jobs:
     name: package_updates
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/sunnypilot/sunnypilot-base:latest
+      image: ghcr.io/commaai/openpilot-base:latest
     if: github.repository == 'sunnypilot/sunnypilot'
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: uv lock
@@ -47,19 +47,9 @@ jobs:
         python3 -m ensurepip --upgrade
         pip3 install uv
         uv lock --upgrade
-    - name: uv pip tree
-      id: pip_tree
-      run: |
-        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
-        uv pip tree >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
     - name: bump submodules
       run: |
         git config --global --add safe.directory '*'
-        git config submodule.msgq.update none
-        git config submodule.rednose_repo.update none
-        git config submodule.teleoprtc_repo.update none
-        git config submodule.tinygrad.update none
         git submodule update --remote
         git add .
     - name: update car docs
@@ -69,19 +59,14 @@ jobs:
         python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:
-        author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
-        token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
+        author: Vehicle Researcher <user@comma.ai>
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
         commit-message: Update Python packages
         title: '[bot] Update Python packages'
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: |
-          Automatic PR from repo-maintenance -> package_updates
-
-          ```
-          ${{ steps.pip_tree.outputs.PIP_TREE }}
-          ```
+        body: 'Automatic PR from repo-maintenance -> package_updates'
         labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -0,0 +1,366 @@
+name: selfdrive
+
+on:
+  push:
+    branches:
+      - master
+      - master-new
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
+
+concurrency:
+  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  REPORT_NAME: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
+  PYTHONWARNINGS: error
+  BASE_IMAGE: openpilot-base
+  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
+
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
+
+jobs:
+  build_release:
+    if: github.repository == 'commaai/openpilot' # build_release blocked for the time being to only comma as we may have a different process.
+    name: build release
+    runs-on:
+      - 'ubuntu-24.04'
+    env:
+      STRIPPED_DIR: /tmp/releasepilot
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Getting LFS files
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: git lfs pull
+    - name: Build devel
+      timeout-minutes: 1
+      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot and run checks
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "python3 system/manager/build.py"
+    - name: Run tests
+      timeout-minutes: 1
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - name: Check submodules
+      if: github.repository == 'sunnypilot/sunnypilot'
+      timeout-minutes: 3
+      run: release/check-submodules.sh
+
+  build:
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup docker push
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
+    - uses: ./.github/workflows/setup-with-retry
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: 30
+
+  build_mac:
+    name: build macOS
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - name: Homebrew cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          brew-macos-${{ env.CACHE_COMMIT_DATE }}
+          brew-macos
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+      env:
+        # package install has DeprecationWarnings
+        PYTHONWARNINGS: default
+    - name: Save Homebrew cache
+      uses: actions/cache/save@v4
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+    - run: git lfs pull
+    - name: Getting scons cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}-macos
+    - name: Building openpilot
+      run: . .venv/bin/activate && scons -j$(nproc)
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+
+  static_analysis:
+    name: static analysis
+    runs-on:
+      - 'ubuntu-latest'
+    env:
+      PYTHONWARNINGS: default
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Static analysis
+      timeout-minutes: 1
+      run: ${{ env.RUN }} "scripts/lint/lint.sh"
+
+  unit_tests:
+    name: unit tests
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run unit tests
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \
+                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
+                        ./selfdrive/ui/tests/create_test_translations.sh && \
+                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
+                        chmod -R 777 /tmp/comma_download_cache"
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v4
+      with:
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  process_replay:
+    name: process replay
+    if: github.repository == 'commaai/openpilot'  # disable process_replay for forks
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run replay
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache && \
+                        coverage combine && \
+                        coverage xml"
+    - name: Print diff
+      id: print-diff
+      if: always()
+      run: cat selfdrive/test/process_replay/diff.txt
+    - uses: actions/upload-artifact@v4
+      if: always()
+      continue-on-error: true
+      with:
+        name: process_replay_diff.txt
+        path: selfdrive/test/process_replay/diff.txt
+    - name: Upload reference logs
+      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
+      run: |
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+    - name: Run regen
+      if: false
+      timeout-minutes: 4
+      run: |
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v4
+      with:
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  test_cars:
+    name: cars
+    runs-on:
+      - 'ubuntu-24.04'
+    strategy:
+      fail-fast: false
+      matrix:
+        job: [0, 1, 2, 3]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: routes-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }}
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Test car models
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 6 }}
+      run: |
+        ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
+      env:
+        NUM_JOBS: 4
+        JOB_ID: ${{ matrix.job }}
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v4
+      with:
+        name: ${{ github.job }}-${{ matrix.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  car_docs_diff:
+    name: PR comments
+    runs-on: ubuntu-latest
+    #if: github.event_name == 'pull_request'
+    if: false  # TODO: run this in opendbc?
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.base.ref }}
+      - run: git lfs pull
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Get base car info
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
+          sudo chown -R $USER:$USER ${{ github.workspace }}
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          path: current
+      - run: cd current && git lfs pull
+      - name: Save car docs diff
+        id: save_diff
+        run: |
+          cd current
+          ${{ env.RUN }} "scons -j$(nproc)"
+          output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
+          output="${output//$'\n'/'%0A'}"
+          echo "::set-output name=diff::$output"
+      - name: Find comment
+        if: ${{ env.AZURE_TOKEN != '' }}
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: This PR makes changes to
+      - name: Update comment
+        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: "${{ steps.save_diff.outputs.diff }}"
+          edit-mode: replace
+      - name: Delete comment
+        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.deleteComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: ${{ steps.fc.outputs.comment-id }}
+            })
+
+  simulator_driving:
+    name: simulator driving
+    runs-on:
+      - 'ubuntu-24.04'
+    if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Driving test
+      timeout-minutes: 1
+      run: |
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        source selfdrive/test/setup_vsound.sh && \
+                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
+
+  create_ui_report:
+    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
+    name: Create UI Report
+    runs-on:
+      - 'ubuntu-24.04'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: caching frames
+        id: frames-cache
+        uses: actions/cache@v4
+        with:
+          path: .ci_cache/comma_download_cache
+          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create Test Report
+        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 2 || 4) }}
+        run: >
+            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                            source selfdrive/test/setup_xvfb.sh &&
+                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
+                            chmod -R 777 /tmp/comma_download_cache"
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.REPORT_NAME }}
+          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-with-retry/action.yaml

@@ -10,17 +10,9 @@ inputs:
     required: false
     default: 30
 
-outputs:
-  duration:
-    description: 'Duration of the setup process in seconds'
-    value: ${{ steps.get_duration.outputs.duration }}
-
 runs:
   using: "composite"
   steps:
-    - id: start_time
-      shell: bash
-      run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
     - id: setup1
       uses: ./.github/workflows/setup
       continue-on-error: true
@@ -43,10 +35,3 @@ runs:
       uses: ./.github/workflows/setup
       with:
         is_retried: true
-    - id: get_duration
-      shell: bash
-      run: |
-        END_TIME=$(date +%s)
-        DURATION=$((END_TIME - START_TIME))
-        echo "Total duration: $DURATION seconds"
-        echo "duration=$DURATION" >> $GITHUB_OUTPUT

.github/workflows/stale.yaml

@@ -5,15 +5,14 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 7
-  DAYS_BEFORE_PR_STALE: 24
-  DAYS_BEFORE_PR_STALE_DRAFT: 30
+  DAYS_BEFORE_PR_CLOSE: 2
+  DAYS_BEFORE_PR_STALE: 9
 
 jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 
@@ -25,28 +24,6 @@ jobs:
           exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
           days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
           days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-          exempt-draft-pr: false
-
-          # issue config
-          days-before-issue-stale: -1 # ignore issues for now
-
-  # same as above, but give draft PRs more time
-  stale_drafts:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v10
-        with:
-          exempt-all-milestones: true
-
-          # pull request config
-          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
-          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
-          stale-pr-label: stale
-          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
-          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
-          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }}
-          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-          exempt-draft-pr: true
 
           # issue config
           days-before-issue-stale: -1 # ignore issues for now

.github/workflows/sunnypilot-build-model.yaml

@@ -9,27 +9,6 @@ env:
   MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
 
 on:
-  workflow_call:
-    inputs:
-      upstream_branch:
-        description: 'Upstream branch to build from'
-        required: true
-        default: 'master'
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
-      artifact_suffix:
-        description: 'Suffix for artifact name'
-        required: false
-        type: string
-        default: ''
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -53,53 +32,34 @@ run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from
 jobs:
   get_model:
     runs-on: ubuntu-latest
-    env:
-      REF: ${{ inputs.upstream_branch }}
     outputs:
       model_date: ${{ steps.commit-date.outputs.model_date }}
     steps:
-      # Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
-      # and fallback to sunnypilot if the ref checkout fails.
-      - name: Checkout commaai/openpilot
-        id: checkout_upstream
-        continue-on-error: true
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          repository: commaai/openpilot
-          ref: ${{ inputs.upstream_branch }}
+          repository: ${{ env.UPSTREAM_REPO }}
+          ref: ${{ github.event.inputs.upstream_branch }}
           submodules: recursive
-          path: openpilot
-
-      - name: Fallback to sunnypilot/sunnypilot
-        if: steps.checkout_upstream.outcome == 'failure'
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot
-          ref: ${{ inputs.upstream_branch }}
-          submodules: recursive
-          path: openpilot
       - name: Get commit date
         id: commit-date
         run: |
-          cd ${{ github.workspace }}/openpilot
+          # Get the commit date in YYYY-MM-DD format
           commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
           echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
-      - run: |
-          cd ${{ github.workspace }}/openpilot
-          git lfs pull
+      - run: git lfs pull
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v4
         with:
-          name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
-          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
+          name: models
+          path: ${{ github.workspace }}/selfdrive/modeld/models/*.onnx
 
   build_model:
-    runs-on: [self-hosted, tici]
+    runs-on: self-hosted
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
-      REF: ${{ inputs.upstream_branch }}
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -111,7 +71,7 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
@@ -154,10 +114,8 @@ jobs:
       - name: Download model artifacts
         uses: actions/download-artifact@v4
         with:
-          name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
+          name: models
           path: ${{ github.workspace }}/selfdrive/modeld/models
-      - run: |
-          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |
@@ -199,22 +157,12 @@ jobs:
             --upstream-branch "${{ inputs.upstream_branch }}" \
             ${{ inputs.is_20hz && '--is-20hz' || '' }}
 
-      - name: Write artifact name to file
-        run: echo "model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}" > ${{ env.OUTPUT_DIR }}/artifact_name.txt
-
       - name: Upload Build Artifacts
-        id: upload-artifact
         uses: actions/upload-artifact@v4
         with:
-          name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
+          name: model-${{ env.MODEL_NAME }}-${{ github.run_number }}
           path: ${{ env.OUTPUT_DIR }}
 
-      - name: Upload artifact name file
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
-          path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
-
       - name: Re-enable powersave
         if: always()
         run: |

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -6,121 +6,57 @@ env:
   CI_DIR: ${{ github.workspace }}/release/ci
   SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
-
+  
   # Branch configurations
-  STAGING_SOURCE_BRANCH: 'master'
-
+  STAGING_C3_SOURCE_BRANCH: ${{ vars.STAGING_C3_SOURCE_BRANCH || 'master-new' }} # vars are set on repo settings.
+  DEV_C3_SOURCE_BRANCH: ${{ vars.DEV_C3_SOURCE_BRANCH || 'master-dev-c3-new' }} # vars are set on repo settings.
+  
+  # Target branch configurations
+  STAGING_TARGET_BRANCH: ${{ vars.STAGING_TARGET_BRANCH || 'staging-c3-new' }} # vars are set on repo settings.
+  DEV_TARGET_BRANCH: ${{ vars.DEV_TARGET_BRANCH || 'dev-c3-new' }} # vars are set on repo settings.
+  RELEASE_TARGET_BRANCH: ${{ vars.RELEASE_TARGET_BRANCH || 'release-c3-new' }} # vars are set on repo settings.
+  
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
 on:
   push:
-    branches: [ master, master-dev ]
-    tags: [ 'release/*' ]
+    branches: [ master, master-new, master-dev-c3-new ]
+    tags: [ '*' ]
   pull_request_target:
     types: [ labeled ]
   workflow_dispatch:
     inputs:
       wait_for_tests:
-        description: 'Wait for tests to finish'
+        description: 'Wait for selfdrive_tests to finish'
         required: false
         type: boolean
         default: false
 
 jobs:
-  prepare_strategy:
-    runs-on: ubuntu-24.04
-    if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-    outputs:
-      environment: ${{ steps.strategy.outputs.environment }}
-      new_branch: ${{ steps.strategy.outputs.new_branch }}
-      extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
-      version: ${{ steps.strategy.outputs.version }}
-      cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
-      publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
-      is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
-      build: ${{ steps.strategy.outputs.build }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Extract deploy strategy
-        id: strategy
-        run: |
-          echo '::group::Strategy Extraction'
-          BRANCH="${{ github.head_ref || github.ref_name }}"
-          echo "Current branch: $BRANCH"
-
-          STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
-          CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
-            .configs[] | select(.branch == $branch)
-          ')
-
-          BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
-          if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
-            echo "No exact strategy match found. Falling back to feature/fork logic."
-            IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
-            FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
-            NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
-
-            echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
-            echo "version=$BUILD" >> $GITHUB_OUTPUT
-            echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
-            echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
-            echo "environment=feature-branch" >> $GITHUB_OUTPUT
-            echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
-          else
-          echo "Matched config: $CONFIG"
-            environment=$(echo "$CONFIG" | jq -r '.environment')
-            echo "environment=$environment" >> $GITHUB_OUTPUT
-            echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
-            cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
-            echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
-            echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
-
-            is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
-            echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
-
-            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
-            echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
-            echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
-          fi
-          echo "build=$BUILD" >> $GITHUB_OUTPUT
-          cat $GITHUB_OUTPUT
-
   validate_tests:
     runs-on: ubuntu-24.04
-    needs: [ prepare_strategy ]
-    if: ${{
-      ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
-      (github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
-      contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      }}
+    if: ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) || contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
     steps:
       - uses: actions/checkout@v4
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
         with:
-          workflow: tests.yaml # The workflow file to monitor
+          workflow: selfdrive_tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
 
   build:
-    needs: [ validate_tests, prepare_strategy ]
+    needs: [ validate_tests ]
     concurrency:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
-    runs-on: [self-hosted, tici]
+    runs-on: self-hosted
     outputs:
-      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
-      version: ${{ needs.prepare_strategy.outputs.version }}
-      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
-      commit_sha: ${{ github.sha }}
-    if: ${{
-      (always() && !cancelled() && !failure()) &&
-      needs.prepare_strategy.result == 'success' &&
-      (needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
-      (!contains(github.event_name, 'pull_request') ||
-      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      }}
+      new_branch: ${{ steps.set-env.outputs.new_branch }}
+      version: ${{ steps.set-env.outputs.version }}
+      extra_version_identifier: ${{ steps.set-env.outputs.extra_version_identifier }}
+      commit_sha: ${{ steps.set-env.outputs.commit_sha }}
+    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -134,21 +70,70 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_C3_SOURCE_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}
 
+      - name: Set Feature Branch Prebuilt Configuration
+        id: set_feature_configuration
+        if: (
+              !(env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH) &&
+              !(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH) &&
+              !(startsWith(github.ref, 'refs/tags/'))
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.SOURCE_BRANCH }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-prebuilt" >> $GITHUB_ENV
+          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
+
+      - name: Set dev-c3-new prebuilt Configuration
+        id: set_dev_configuration
+        if: (
+                steps.set_feature_configuration.outcome == 'skipped' && 
+                env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV
+          echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
+          echo "EXTRA_VERSION_IDENTIFIER=${{ github.run_number }}" >> $GITHUB_ENV
+
+      - name: Set staging-c3-new prebuilt Configuration
+        id: set_staging_configuration
+        if: (
+                steps.set_feature_configuration.outcome == 'skipped' &&
+                !contains(github.event_name, 'pull_request') &&
+                steps.set_dev_configuration.outcome == 'skipped' && 
+                (env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH)
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV
+          echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV
+          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-staging" >> $GITHUB_ENV
+
+      - name: Set release-c3-new prebuilt Configuration
+        id: set_tag_configuration
+        if: (
+              steps.set_feature_configuration.outcome == 'skipped' &&
+              !contains(github.event_name, 'pull_request') &&
+              steps.set_staging_configuration.outcome == 'skipped' &&
+              startsWith(github.ref, 'refs/tags/')
+            )
+        run: |
+          echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV
+          echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV
+          echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-release" >> $GITHUB_ENV
+
       - name: Set environment variables
         id: set-env
         run: |
-          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
-          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
-          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
+          # Write to GITHUB_OUTPUT from environment variables
+          echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
+          [[ ! -z "$EXTRA_VERSION_IDENTIFIER" ]] && echo "extra_version_identifier=$EXTRA_VERSION_IDENTIFIER" >> $GITHUB_OUTPUT
+          [[ ! -z "$VERSION" ]] && echo "version=$VERSION" >> $GITHUB_OUTPUT
           echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
-
+          
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
@@ -180,15 +165,6 @@ jobs:
           ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
           cd $BUILD_DIR
           sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
-          echo "Building sunnypilot's modeld..."
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld
-          echo "Building sunnypilot's modeld_v2..."
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
-          echo "Building sunnypilot's locationd..."
-          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
-          echo "Building openpilot's locationd..."
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
-          echo "Building rest of sunnypilot"
           scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
@@ -200,28 +176,37 @@ jobs:
           sudo rm -rf ${OUTPUT_DIR}
           mkdir -p ${OUTPUT_DIR}
           rsync -am${RUNNER_DEBUG:+v} \
+            --include='**/panda/board/' \
+            --include='**/panda/board/obj' \
+            --include='**/panda/board/obj/panda.bin.signed' \
+            --include='**/panda/board/obj/panda_h7.bin.signed' \
+            --include='**/panda/board/obj/bootstub.panda.bin' \
+            --include='**/panda/board/obj/bootstub.panda_h7.bin' \
             --exclude='.sconsign.dblite' \
             --exclude='*.a' \
             --exclude='*.o' \
             --exclude='*.os' \
             --exclude='*.pyc' \
             --exclude='moc_*' \
-            --exclude='__pycache__' \
+            --exclude='*.cc' \
             --exclude='Jenkinsfile' \
+            --exclude='supercombo.onnx' \
+            --exclude='**/panda/board/*' \
+            --exclude='**/panda/board/obj/**' \
+            --exclude='**/panda/certs/' \
+            --exclude='**/panda/crypto/' \
             --exclude='**/release/' \
             --exclude='**/.github/' \
             --exclude='**/selfdrive/ui/replay/' \
             --exclude='**/__pycache__/' \
+            --exclude='**/selfdrive/ui/*.h' \
+            --exclude='**/selfdrive/ui/**/*.h' \
+            --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' \
             --exclude='${{env.SCONS_CACHE_DIR}}' \
             --exclude='**/.git/' \
             --exclude='**/SConstruct' \
             --exclude='**/SConscript' \
             --exclude='**/.venv/' \
-            --exclude='selfdrive/modeld/models/driving_vision.onnx' \
-            --exclude='selfdrive/modeld/models/driving_policy.onnx' \
-            --exclude='sunnypilot/modeld*/models/supercombo.onnx' \
-            --exclude='third_party/*x86*' \
-            --exclude='third_party/*Darwin*' \
             --delete-excluded \
             --chown=comma:comma \
             ${BUILD_DIR}/ ${OUTPUT_DIR}/
@@ -245,15 +230,12 @@ jobs:
 
   publish:
     concurrency:
-      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
-      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
-      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
-      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
-      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
-    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build, prepare_strategy ]
+      group: publish-${{ github.head_ref || github.ref_name }}
+      cancel-in-progress: true
+    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
+    needs: [ build ]
     runs-on: ubuntu-24.04
-    environment: ${{ needs.prepare_strategy.outputs.environment }}
+    environment: ${{ (contains(fromJSON(vars.AUTO_DEPLOY_PREBUILT_BRANCHES), github.head_ref || github.ref_name) || contains(github.event.pull_request.labels.*.name, 'prebuilt')) && 'auto-deploy' || 'feature-branch' }}
     steps:
       - uses: actions/checkout@v4
 
@@ -285,8 +267,8 @@ jobs:
             "${{ needs.build.outputs.new_branch }}" \
             "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.build.outputs.extra_version_identifier }}"
-
+            "-${{ needs.build.outputs.extra_version_identifier }}"
+          
           echo ""
           echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
           echo ""
@@ -294,60 +276,38 @@ jobs:
           echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
           echo "3. Update as needed (JSON array with no spaces)"
 
-      - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
-        if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
-        run: |
-            TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
-            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
-            git push -f origin ${TAG}
-
   notify:
-    needs:
-    - prepare_strategy
-    - build
-    - publish
+    needs: [ build, publish ]
     runs-on: ubuntu-24.04
-    if: ${{ (always() && !cancelled() && !failure())
-      && needs.publish.result == 'success'
-      && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
+    if: ${{ (always() && !failure() && !cancelled()) && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
     steps:
       - uses: actions/checkout@v4
-
-      - name: Prepare notification message
-        id: message
-        run: |
-          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
-          export VERSION="${{ needs.prepare_strategy.outputs.version }}"
-          export branch_name="${{ env.SOURCE_BRANCH }}"
-          export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
-          export commit_sha="${{ github.sha }}"
-          export commit_short_sha="${{ github.sha }}"
-          export commit_short_sha="${commit_short_sha:0:7}"
-          export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
-          export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
-
-          MESSAGE=$(cat << 'EOF' | envsubst
-          ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
-          EOF
-          )
-
-          {
-            echo 'content<<EOFMARKER'
-            echo "$MESSAGE"
-            echo 'EOFMARKER'
-          } >> $GITHUB_OUTPUT
-        shell: bash
-
-      - name: Post to Discourse
-        uses: ./.github/workflows/post-to-discourse
+      - name: Setup Alpine Linux environment
+        uses: jirutka/setup-alpine@v1.2.0
         with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: "system"
-          topic-id: ${{ fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name].topic_id }}
-          message: ${{ steps.message.outputs.content }}
+          packages: 'jq gettext curl'
 
+      - name: Send Discord Notification
+        env:
+          DISCORD_WEBHOOK: ${{ contains(fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES), env.SOURCE_BRANCH) && secrets.DISCORD_DEV_FEEDBACK_CHANNEL_WEBHOOK || secrets.DISCORD_DEV_PRIVATE_CHANNEL_WEBHOOK }}
+        run: |
+          TEMPLATE='${{ vars.DISCORD_GENERAL_UPDATE_NOTICE }}'
+          export EXTRA_VERSION_IDENTIFIER="${{ needs.build.outputs.extra_version_identifier }}"
+          export VERSION="${{ needs.build.outputs.version }}"
+          export branch_name=${{ env.SOURCE_BRANCH }}
+          export new_branch=${{ needs.build.outputs.new_branch }}
+          export extra_version_identifier=${{ needs.build.outputs.extra_version_identifier || github.run_number}}
+          echo ${TEMPLATE} | envsubst | jq -c '.' | tee payload.json
+          curl -X POST -H "Content-Type: application/json" -d @payload.json $DISCORD_WEBHOOK
+          
+          echo ""
+          echo "----   ℹ️ To update the list of branches that notify to dev-feedback  -----"
+          echo ""
+          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/DEV_FEEDBACK_NOTIFICATION_BRANCHES"
+          echo "2. Current value: ${{ vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES }}"
+          echo "3. Update as needed (JSON array with no spaces)"
+        shell: alpine.sh {0}
+  
   manage-pr-labels:
     name: Remove prebuilt label
     runs-on: ubuntu-latest

.github/workflows/sunnypilot-master-dev-c3-prep.yaml

@@ -1,8 +1,9 @@
-name: Build dev
+name: Build dev-c3-new
 
 env:
-  DEFAULT_SOURCE_BRANCH: "master"
-  DEFAULT_TARGET_BRANCH: "master-dev"
+  DEFAULT_SOURCE_BRANCH: "master-new"
+  DEFAULT_TARGET_BRANCH: "master-dev-c3-new"
+  PR_LABEL: "dev-c3"
   LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
   LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
 
@@ -10,21 +11,23 @@ on:
   push:
     branches:
       - master
+      - master-new    
   pull_request_target:
-    types: [ labeled ]
-    branches:
-      - 'master'
+    types: [ labeled ]  
+    branches:    
+      - 'master'        
+      - 'master-new'            
   workflow_dispatch:
     inputs:
       source_branch:
         description: 'Source branch to reset from'
         required: true
-        default: 'master'
+        default: 'master-new'
         type: string
       target_branch:
         description: 'Target branch to reset and squash into'
         required: true
-        default: 'master-dev'
+        default: 'master-dev-c3-new'
         type: string
       cancel_in_progress:
         description: 'Cancel any in-progress runs of this workflow'
@@ -40,25 +43,24 @@ jobs:
   reset-and-squash:
     runs-on: ubuntu-latest
     if: (
-      (github.event_name == 'workflow_dispatch')
-      || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-      || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
-      )
+            (github.event_name == 'workflow_dispatch')
+            || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
+            || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
+        )
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Fetch all history for all branches
           token: ${{ secrets.GITHUB_TOKEN }}
-          persist-credentials: false
 
       - name: Wait for Tests
         uses: ./.github/workflows/wait-for-action # Path to where you place the action
         if: (
-          (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-          || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
-          )
+                (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
+                || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
+            )
         with:
-          workflow: tests.yaml # The workflow file to monitor
+          workflow: selfdrive_tests.yaml # The workflow file to monitor
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Configure Git
@@ -118,8 +120,8 @@ jobs:
         run: |
           # Use GitHub API to get PRs with specific label, ordered by creation date
           PR_LIST=$(gh api graphql -f query='
-            query($search_query:String!) {
-              search(query: $search_query, type:ISSUE, first:40) {
+            query($label:String!) {
+              search(query: $label, type:ISSUE, first:100) {
                 nodes {
                   ... on PullRequest {
                     number
@@ -149,9 +151,8 @@ jobs:
                   }
                 }
               }
-            }' -F search_query="repo:${{ github.repository }} is:pr is:open label:${{ vars.PREBUILT_PR_LABEL }},${{ vars.PREBUILT_PR_LABEL }}-c3 draft:false sort:created-asc")
+            }' -F label="is:pr is:open label:${PR_LABEL} draft:false sort:created-asc")
 
-          PR_LIST=${PR_LIST//\'/}
           echo "PR_LIST=${PR_LIST}" >> $GITHUB_OUTPUT
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -174,38 +175,11 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Restore workflows from source
-        run: |
-          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-          SOURCE_BRANCH="${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-
-          # Ensure we are on the target branch
-          git checkout $TARGET_BRANCH
-
-          echo "Restoring .github/workflows from $SOURCE_BRANCH"
-          git checkout origin/$SOURCE_BRANCH -- .github/workflows
-
-          if ! git diff --cached --quiet; then
-            echo "Workflows differ. Committing restoration."
-            git commit -m "chore: restore .github/workflows from $SOURCE_BRANCH"
-          else
-            echo "Workflows match $SOURCE_BRANCH."
-          fi
-
-      - uses: actions/create-github-app-token@v2
-        id: ci-token
-        with:
-          app-id: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_ID }}
-          private-key: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_PRIVATE_KEY }}
-
       - name: Push changes if there are diffs
-        id: push-changes
+        id: push-changes  # Add an id so we can reference this step
         run: |
           TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
 
-          # Use the App Token to set the remote URL with authentication
-          git remote set-url origin "https://x-access-token:${{ steps.ci-token.outputs.token }}@github.com/${{ github.repository }}.git"
-
           # Fetch the latest from remote
           git fetch origin $TARGET_BRANCH
 
@@ -216,7 +190,7 @@ jobs:
             exit 0
           fi
 
-          # Push with the authenticated origin
+          # If we get here, there are diffs, so push
           if ! git push origin $TARGET_BRANCH --force; then
             echo "Failed to push changes to $TARGET_BRANCH"
             exit 1
@@ -229,13 +203,13 @@ jobs:
         if: steps.push-changes.outputs.has_changes == 'true'
         run: |
           echo "Triggering selfdrive tests..."
-          gh workflow run tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+          gh workflow run selfdrive_tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
 
           echo "Sleeping for 120s to give plenty of time for the action to start and then we wait"
           sleep 120
 
           echo "Getting latest run ID..."
-          RUN_ID=$(gh run list --workflow=tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+          RUN_ID=$(gh run list --workflow=selfdrive_tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
 
           echo "Watching run ID: $RUN_ID"
           gh run watch "$RUN_ID"

.github/workflows/test-discourse.yaml.yml

@@ -1,78 +0,0 @@
-name: Debug Discourse Posting
-
-on:
-  push:
-
-jobs:
-  test-discourse-post:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Post test message to Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
-          topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
-          message: |
-            ## 🧪 Test Post from GitHub Actions
-            
-            **This is a test post to verify Discourse integration**
-            
-            - **Workflow**: ${{ github.workflow }}
-            - **Run Number**: #${{ github.run_number }}
-            - **Branch**: `${{ github.ref_name }}`
-            - **Commit**: ${{ github.sha }}
-            - **Actor**: @${{ github.actor }}
-            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
-            
-            ---
-            
-            ### Fake Build Info (for testing)
-            - **Version**: 0.9.8-test
-            - **Build**: #42
-            - **Branch**: release-test
-            
-            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            
-            *This is an automated test message. Drive safe! 🚗💨*
-      
-      
-      - name: Create topic on Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
-          #topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
-          category-id: 4
-          title: "This is a test of a new topic instead of a reply"
-          message: |
-            ## 🧪 Test Post from GitHub Actions
-            
-            **This is a test post to verify Discourse integration**
-            
-            - **Workflow**: ${{ github.workflow }}
-            - **Run Number**: #${{ github.run_number }}
-            - **Branch**: `${{ github.ref_name }}`
-            - **Commit**: ${{ github.sha }}
-            - **Actor**: @${{ github.actor }}
-            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
-            
-            ---
-            
-            ### Fake Build Info (for testing)
-            - **Version**: 0.9.8-test
-            - **Build**: #42
-            - **Branch**: release-test
-            
-            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            
-            *This is an automated test message. Drive safe! 🚗💨*
-      - name: Display results
-        if: always()
-        run: |
-          echo "::notice::Discourse post test completed"
-          echo "Check your Discourse topic to verify the post appeared correctly"

.github/workflows/tests.yaml

@@ -1,340 +0,0 @@
-name: tests
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  PYTHONWARNINGS: error
-  BASE_IMAGE: sunnypilot-base
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh base
-
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
-  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
-
-jobs:
-  build_release:
-    name: build release
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot and run checks
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "python3 system/manager/build.py"
-    - name: Run tests
-      timeout-minutes: 1
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - name: Check submodules
-      if: github.repository == 'sunnypilot/sunnypilot'
-      timeout-minutes: 3
-      run: |
-        if [ "${{ github.ref }}" != "refs/heads/master" ]; then
-          git fetch  origin master:refs/remotes/origin/master
-
-          SUBMODULE_PATHS=$(git diff origin/master HEAD --name-only | grep -E '^[^/]+$' | while read path; do
-            if git ls-files --stage "$path" | grep -q "^160000"; then
-              echo "$path"
-            fi
-          done | tr '\n' ' ')
-
-          if [ -n "$SUBMODULE_PATHS" ]; then
-            echo "Changed submodule paths: $SUBMODULE_PATHS"
-            export SUBMODULE_PATHS="$SUBMODULE_PATHS"
-            export CHECK_PR_REFS=true
-          fi
-        fi
-        release/check-submodules.sh
-
-  build:
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Setup docker push
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'sunnypilot/sunnypilot'
-      run: |
-        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
-    - uses: ./.github/workflows/setup-with-retry
-    - uses: ./.github/workflows/compile-openpilot
-      timeout-minutes: 30
-
-  build_mac:
-    name: build macOS
-    if: false # tmp disable due to brew install not working
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - name: Homebrew cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        save: false # No need save here if we manually save it later conditionally
-        path: ~/Library/Caches/Homebrew
-        key: brew-macos-${{ hashFiles('tools/Brewfile') }}-${{ github.sha }}
-        restore-keys: |
-          brew-macos-${{ hashFiles('tools/Brewfile') }}
-          brew-macos-
-    - name: Install dependencies
-      run: ./tools/mac_setup.sh
-      env:
-        PYTHONWARNINGS: default  # package install has DeprecationWarnings
-        HOMEBREW_DISPLAY_INSTALL_TIMES: 1
-    - name: Save Homebrew cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: ~/Library/Caches/Homebrew
-        key: brew-macos-${{ hashFiles('tools/Brewfile') }}-${{ github.sha }}
-    - run: git lfs pull
-    - name: Getting scons cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        save: false # No need save here if we manually save it later conditionally
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}-macos
-    - name: Building openpilot
-      run: . .venv/bin/activate && scons -j$(nproc)
-    - name: Save scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-
-  static_analysis:
-    name: static analysis
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      PYTHONWARNINGS: default
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Static analysis
-      timeout-minutes: 1
-      run: ${{ env.RUN }} "scripts/lint/lint.sh"
-
-  unit_tests:
-    name: unit tests
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 999 }}
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        # Pre-compile Python bytecode so each pytest worker doesn't need to
-                        $PYTEST --collect-only -m 'not slow' -qq && \
-                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
-                        chmod -R 777 /tmp/comma_download_cache"
-
-  process_replay:
-    name: process replay
-    if: false  # disable process_replay for forks
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v5
-      with:
-        path: .ci_cache/comma_download_cache
-        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/test_processes.py') }}
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
-      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
-      run: |
-        ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
-                        chmod -R 777 /tmp/comma_download_cache"
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v6
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Checkout ci-artifacts
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v4
-      with:
-        repository: commaai/ci-artifacts
-        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-        path: ${{ github.workspace }}/ci-artifacts
-    - name: Push refs
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      working-directory: ${{ github.workspace }}/ci-artifacts
-      run: |
-        git checkout --orphan process-replay
-        git rm -rf .
-        git config user.name "GitHub Actions Bot"
-        git config user.email "<>"
-        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
-        echo "${{ github.sha }}" > ref_commit
-        git add .
-        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}"
-        git push origin process-replay --force
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      run: |
-        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-                        chmod -R 777 /tmp/comma_download_cache"
-
-  simulator_driving:
-    name: simulator driving
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: false  # FIXME: Started to timeout recently
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-      id: setup-step
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Driving test
-      timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        source selfdrive/test/setup_vsound.sh && \
-                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
-
-  create_raylib_ui_report:
-    name: Create raylib UI Report
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc)"
-      - name: Create raylib UI Report
-        run: >
-          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-                          source selfdrive/test/setup_xvfb.sh &&
-                          python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py"
-      - name: Upload Raylib UI Report
-        uses: actions/upload-artifact@v6
-        with:
-          name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/test_ui/raylib_report/screenshots
-
-  create_mici_raylib_ui_report:
-    name: Create mici raylib UI Report
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc)"
-      - name: Create mici raylib UI Report
-        run: >
-          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-                          source selfdrive/test/setup_xvfb.sh &&
-                          WINDOWED=1 python3 selfdrive/ui/tests/diff/replay.py"
-      - name: Upload Raylib UI Report
-        uses: actions/upload-artifact@v6
-        with:
-          name: mici-raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -1,23 +1,23 @@
-name: "raylib ui preview"
+name: "ui preview"
 on:
   push:
     branches:
       - master
+      - master-new
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
     branches:
       - 'master'
+      - 'master-new'
     paths:
-      - 'selfdrive/assets/**'
       - 'selfdrive/ui/**'
-      - 'system/ui/**'
   workflow_dispatch:
 
 env:
-  UI_JOB_NAME: "Create raylib UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-raylib-ui"
+  UI_JOB_NAME: "Create UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
 
 jobs:
   preview:
@@ -54,31 +54,31 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: raylib-report-1-${{ env.REPORT_NAME }}
+          name: report-1-${{ env.REPORT_NAME }}
           path: ${{ github.workspace }}/pr_ui
 
       - name: Getting master ui
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_ui_raylib
-          ref: openpilot_master_ui_raylib
+          path: ${{ github.workspace }}/master_ui
+          ref: openpilot_master_ui
 
       - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        working-directory: ${{ github.workspace }}/master_ui_raylib
+        if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          git checkout --orphan=new_master_ui_raylib
+          git checkout --orphan=new_master_ui
           git rm -rf *
-          git branch -D openpilot_master_ui_raylib
-          git branch -m openpilot_master_ui_raylib
+          git branch -D openpilot_master_ui
+          git branch -m openpilot_master_ui
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
           mv ${{ github.workspace }}/pr_ui/*.png .
           git add .
-          git commit -m "raylib screenshots for commit ${{ env.SHA }}"
-          git push origin openpilot_master_ui_raylib --force
+          git commit -m "screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui --force
 
       - name: Finding diff
         if: github.event_name == 'pull_request_target'
@@ -96,7 +96,7 @@ jobs:
           for ((i=0; i<${#A[*]}; i=i+1));
           do
             # Check if the master file exists
-            if [ ! -f "${{ github.workspace }}/master_ui_raylib/${A[$i]}.png" ]; then
+            if [ ! -f "${{ github.workspace }}/master_ui/${A[$i]}.png" ]; then
               # This is a new file in PR UI that doesn't exist in master
               DIFF="${DIFF}<details open>"
               DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
@@ -108,12 +108,12 @@ jobs:
 
               DIFF="${DIFF}</table>"
               DIFF="${DIFF}</details>"
-            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
               convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
-              composite mask.png ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png
-              convert -delay 100 ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
 
-              mv ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
 
               DIFF="${DIFF}<details open>"
               DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
@@ -151,7 +151,7 @@ jobs:
 
       - name: Saving proposed ui
         if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_ui_raylib
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
@@ -159,7 +159,7 @@ jobs:
           git rm -rf *
           mv ${{ github.workspace }}/pr_ui/* .
           git add .
-          git commit -m "raylib screenshots for PR #${{ github.event.number }}"
+          git commit -m "screenshots for PR #${{ github.event.number }}"
           git push origin ${{ env.BRANCH_NAME }} --force
 
       - name: Comment Screenshots on PR
@@ -167,9 +167,9 @@ jobs:
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
-            <!-- _(run_id_screenshots_raylib **${{ github.run_id }}**)_ -->
-            ## raylib UI Preview
+            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
+            ## UI Preview
             ${{ steps.find_diff.outputs.DIFF }}
-          comment_tag: run_id_screenshots_raylib
+          comment_tag: run_id_screenshots
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/vendor_third_party.yaml

@@ -1,51 +0,0 @@
-name: vendor third_party
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build:
-    if: github.ref != 'refs/heads/master'
-    strategy:
-      matrix:
-        os: [ubuntu-24.04, macos-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Build
-      run: third_party/build.sh
-    - name: Package artifacts
-      run: |
-        git add -A third_party/
-        git diff --cached --name-only -- third_party/ | tar -cf /tmp/third_party_build.tar -T -
-    - uses: actions/upload-artifact@v4
-      with:
-        name: third-party-${{ runner.os }}
-        path: /tmp/third_party_build.tar
-
-  commit:
-    needs: build
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-    steps:
-    - uses: actions/checkout@v6
-    - uses: actions/download-artifact@v4
-      with:
-        path: /tmp/artifacts
-    - name: Commit vendored libraries
-      run: |
-        for f in /tmp/artifacts/*/third_party_build.tar; do
-          tar xf "$f"
-        done
-        git add third_party/
-        if git diff --cached --quiet; then
-          echo "No changes to commit"
-          exit 0
-        fi
-        git config user.name "github-actions[bot]"
-        git config user.email "github-actions[bot]@users.noreply.github.com"
-        git commit -m "third_party: rebuild vendor libraries"
-        git push

.github/workflows/wait-for-action/action.yaml

@@ -4,7 +4,7 @@ inputs:
   workflow:
     description: 'The workflow file name to monitor'
     required: true
-    default: 'tests.yaml'
+    default: 'selfdrive_tests.yaml'
   branch:
     description: 'The branch to monitor (defaults to current branch)'
     required: false

.gitignore

@@ -10,13 +10,12 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
+model2.png
 a.out
 .hypothesis
-.cache/
 
 /docs_site/
 
-*.mp4
 *.dylib
 *.DSYM
 *.d
@@ -36,23 +35,30 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
-*.mo
+*.qm
 *_pyx.cpp
-*.stats
 config.json
 clcache
 compile_commands.json
 compare_runtime*.html
 
+persist
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
 cereal/messaging/bridge
+selfdrive/mapd/default_speeds_by_region.json
+system/proclogd/proclogd
 selfdrive/ui/translations/tmp
+selfdrive/test/longitudinal_maneuvers/out
 selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 
+notebooks
+hyperthneed
+provisioning
+
 .coverage*
 coverage.xml
 htmlcov
@@ -64,12 +70,13 @@ flycheck_*
 cppcheck_report.txt
 comma*.sh
 
+selfdrive/modeld/thneed/compile
+selfdrive/modeld/models/*.thneed
 selfdrive/modeld/models/*.pkl
 sunnypilot/modeld*/thneed/compile
 sunnypilot/modeld*/models/*.thneed
 sunnypilot/modeld*/models/*.pkl
 
-# openpilot log files
 *.bz2
 *.zst
 
@@ -99,11 +106,6 @@ Pipfile
 .history
 .ionide
 
-.claude/
-.context/
-PLAN.md
-TASK.md
-
 ### JetBrains ###
 !.idea/customTargets.xml
 !.idea/tools/*

.gitmodules

@@ -6,7 +6,7 @@
   url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/commaai/msgq.git
+  url = https://github.com/sunnypilot/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git
@@ -15,7 +15,7 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/sunnypilot/tinygrad.git
+  url = https://github.com/tinygrad/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
   path = sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.idea/tools/External Tools.xml

@@ -2,7 +2,7 @@
   <tool name="uv Scons Build Debug" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
     <exec>
       <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --ccflags=\&quot;-fno-inline\&quot;&quot;" />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db --ccflags=\&quot;-fno-inline\&quot;&quot;" />
       <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
     </exec>
   </tool>
@@ -16,7 +16,7 @@
   <tool name="uv Scons Build Release" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
     <exec>
       <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc)&quot; " />
+      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db&quot; " />
       <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
     </exec>
   </tool>

.run/Build_BIG_UI.run.xml

@@ -1,26 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build_BIG_UI" type="PythonConfigurationType" factoryName="Python">
-    <module name="sunnypilot" />
-    <option name="ENV_FILES" value="" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <envs>
-      <env name="BIG" value="1" />
-    </envs>
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
-    <option name="IS_MODULE_SDK" value="true" />
-    <option name="ADD_CONTENT_ROOTS" value="true" />
-    <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2">
-      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
-    </method>
-  </configuration>
-</component>

.run/Build_SMALL_UI.run.xml

@@ -1,23 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build_SMALL_UI" type="PythonConfigurationType" factoryName="Python">
-    <module name="sunnypilot" />
-    <option name="ENV_FILES" value="" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
-    <option name="IS_MODULE_SDK" value="true" />
-    <option name="ADD_CONTENT_ROOTS" value="true" />
-    <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2">
-      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
-    </method>
-  </configuration>
-</component>

.vscode/launch.json

@@ -23,11 +23,6 @@
       "id": "args",
       "description": "Arguments to pass to the process",
       "type": "promptString"
-    },
-    {
-      "id": "replayArg",
-      "type": "promptString",
-      "description": "Enter route or segment to replay."
     }
   ],
   "configurations": [
@@ -45,41 +40,7 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}"
-    },
-    {
-      "name": "Attach LLDB to Replay drive",
-      "type": "lldb",
-      "request": "attach",
-      "pid": "${command:pickMyProcess}",
-      "initCommands": [
-        "script import time; time.sleep(3)"
-      ]
-    },
-    {
-      "name": "Replay drive",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
-      "args": [
-        "${input:replayArg}"
-      ],
-      "console": "integratedTerminal",
-      "justMyCode": false,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}"
-      },
-      "subProcess": true,
-      "stopOnEntry": false
-    }
-  ],
-  "compounds": [
-    {
-      "name": "Replay drive + Safety LLDB",
-      "configurations": [
-        "Replay drive",
-        "Attach LLDB to Replay drive"
-      ]
+      "cwd": "${workspaceFolder}",
     }
   ]
 }

Dockerfile.openpilot

@@ -9,6 +9,4 @@ WORKDIR ${OPENPILOT_PATH}
 
 COPY . ${OPENPILOT_PATH}/
 
-ENV UV_BIN="/home/batman/.local/bin/"
-ENV PATH="$UV_BIN:$PATH"
-RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc)
+RUN scons --cache-readonly -j$(nproc)

Dockerfile.sunnypilot

@@ -1,14 +1,66 @@
-FROM ghcr.io/sunnypilot/sunnypilot-base:latest
+FROM sunnypilot-base
+
+ARG RUNNER_DEBUG=0
 
 ENV PYTHONUNBUFFERED=1
+ENV OPENPILOT_SRC_PATH=/tmp/openpilot
+ENV BUILD_DIR=/data/openpilot
+ENV OUTPUT_DIR=/output
 
-ENV OPENPILOT_PATH=/home/batman/openpilot
+RUN sudo apt update && sudo apt install -y rsync
 
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
+RUN mkdir -p ${OPENPILOT_SRC_PATH}
+RUN mkdir -p ${BUILD_DIR}
+COPY . ${OPENPILOT_SRC_PATH}
+ENV PYTHONPATH=${BUILD_DIR}
 
-COPY . ${OPENPILOT_PATH}/
+WORKDIR ${OPENPILOT_SRC_PATH}
+RUN ./tools/ubuntu_setup.sh
 
-ENV UV_BIN="/home/batman/.local/bin/"
-ENV PATH="$UV_BIN:$PATH"
-RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc)
+RUN ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
+WORKDIR ${BUILD_DIR}
+RUN sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
+RUN scons --cache-readonly -j$(nproc) --minimal
+RUN touch ${BUILD_DIR}/prebuilt
+RUN sudo rm -rf ${OUTPUT_DIR}
+RUN mkdir -p ${OUTPUT_DIR}
+
+ENTRYPOINT [\
+  "rsync", \
+  "-am", \
+  "--include=**/panda/board/", \
+  "--include=**/panda/board/obj", \
+  "--include=**/panda/board/obj/panda.bin.signed", \
+  "--include=**/panda/board/obj/panda_h7.bin.signed", \
+  "--include=**/panda/board/obj/bootstub.panda.bin", \
+  "--include=**/panda/board/obj/bootstub.panda_h7.bin", \
+  "--exclude=.sconsign.dblite", \
+  "--exclude=*.a", \
+  "--exclude=*.o", \
+  "--exclude=*.os", \
+  "--exclude=*.pyc", \
+  "--exclude=moc_*", \
+  "--exclude=*.cc", \
+  "--exclude=Jenkinsfile", \
+  "--exclude=supercombo.onnx", \
+  "--exclude=**/panda/board/*", \
+  "--exclude=**/panda/board/obj/**", \
+  "--exclude=**/panda/certs/", \
+  "--exclude=**/panda/crypto/", \
+  "--exclude=**/release/", \
+  "--exclude=**/.github/", \
+  "--exclude=**/selfdrive/ui/replay/", \
+  "--exclude=**/__pycache__/", \
+  "--exclude=**/selfdrive/ui/*.h", \
+  "--exclude=**/selfdrive/ui/**/*.h", \
+  "--exclude=**/selfdrive/ui/qt/offroad/sunnypilot/", \
+  #"--exclude=${SCONS_CACHE_DIR:-}", \
+  "--exclude=**/.git/", \
+  "--exclude=**/SConstruct", \
+  "--exclude=**/SConscript", \
+  "--exclude=**/.venv/", \
+  "--delete-excluded", \
+  "--chown=1000:1000", \
+  "/data/openpilot/", \
+  "/output/" \
+]

Dockerfile.sunnypilot_base

@@ -1,83 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/* && \
-    cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
-
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-    rsync \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX=1
-
-RUN dbus-uuidgen > /etc/machine-id
-RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /home/$USER
-COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /home/$USER && \
-    tools/install_python_dependencies.sh && \
-    rm -rf tools/ pyproject.toml uv.lock .cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -22,7 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-#export LOGPRINT=debug # this has gotten too spammy...
+export LOGPRINT=debug
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -167,7 +167,7 @@ node {
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
   def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tici', 'release-tizi', 'release-tizi-staging', 'testing-closet*', 'hotfix-*']
+                         'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
   if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
@@ -178,20 +178,20 @@ node {
 
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh"),
+      deviceStage("build release3-staging", "tici-needs-can", [], [
+        step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
     if (env.BRANCH_NAME == '__nightly') {
       parallel (
         'nightly': {
-          deviceStage("build nightly", "tizi-needs-can", [], [
+          deviceStage("build nightly", "tici-needs-can", [], [
             step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
-          deviceStage("build nightly-dev", "tizi-needs-can", [], [
+          deviceStage("build nightly-dev", "tici-needs-can", [], [
             step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
@@ -200,30 +200,39 @@ node {
 
     if (!env.BRANCH_NAME.matches(excludeRegex)) {
     parallel (
+      // tici tests
       'onroad tests': {
-        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
+        deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
           step("check dirty", "release/check-dirty.sh"),
           step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
-        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
+        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
           step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
+          step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py", [diffPaths: ["system/ubloxd/"]]),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
       'loopback': {
-        deviceStage("loopback", "tizi-loopback", ["UNSAFE=1"], [
+        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
           step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
         ])
       },
+      'camerad AR0231': {
+        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+      },
       'camerad OX03C10': {
-        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
+        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
           step("test exposure", "pytest system/camerad/test/test_exposure.py"),
@@ -237,13 +246,17 @@ node {
         ])
       },
       'sensord': {
-        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
+        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+        ])
+        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
-        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
+        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
           step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
         ])
@@ -253,6 +266,7 @@ node {
           step("build openpilot", "cd system/manager && ./build.py"),
           step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
           // TODO: enable once new AGNOS is available
           // step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),

README.md

@@ -3,23 +3,68 @@
 ## 🌞 What is sunnypilot?
 [sunnypilot](https://github.com/sunnyhaibin/sunnypilot) is a fork of comma.ai's openpilot, an open source driver assistance system. sunnypilot offers the user a unique driving experience for over 300+ supported car makes and models with modified behaviors of driving assist engagements. sunnypilot complies with comma.ai's safety rules as accurately as possible.
 
-## 💭 Join our Community Forum
-Join the official sunnypilot community forum to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
-* https://community.sunnypilot.ai/
+## 💭 Join our Discord
+Join the official sunnypilot Discord server to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
+* https://discord.gg/sunnypilot
+
+  ![](https://dcbadge.vercel.app/api/server/wRW3meAgtx?style=flat) ![Discord Shield](https://discordapp.com/api/guilds/880416502577266699/widget.png?style=shield)
 
 ## Documentation
 https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
 
 ## 🚘 Running on a dedicated device in a car
-First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
+* A supported device to run this software
+    * a [comma three](https://comma.ai/shop/products/three) or a [C3X](https://comma.ai/shop/comma-3x)
+* This software
+* One of [the 300+ supported cars](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
+* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
+
+Detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
 
 ## Installation
-Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
+Please refer to [Recommended Branches](#-recommended-branches) to find your preferred/supported branch. This guide will assume you want to install the latest `release-c3` branch.
+
+* sunnypilot not installed or you installed a version before 0.8.17?
+  1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
+  2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
+  3. Input the installation URL per [Recommended Branches](#-recommended-branches). Example: ```release-c3.sunnypilot.ai```.
+  4. Complete the rest of the installation following the onscreen instructions.
+
+* sunnypilot already installed and you installed a version after 0.8.17?
+  1. On the comma three, go to `Settings` ▶️ `Software`.
+  2. At the `Download` option, press `CHECK`. This will fetch the list of latest branches from sunnypilot.
+  3. At the `Target Branch` option, press `SELECT` to open the Target Branch selector.
+  4. Scroll to select the desired branch per  Recommended Branches (see below). Example: `release-c3`
+
+|    Branch    |         Installation URL         |
+|:------------:|:--------------------------------:|
+| `release-c3` | https://release-c3.sunnypilot.ai |
+| `staging-c3` | https://staging-c3.sunnypilot.ai |
+|   `dev-c3`   | https://dev-c3.sunnypilot.ai     |
+
+### If you want to use our newest branches (our rewrite)
+> [!TIP]
+>You can see the rewrite state on our [rewrite project board](https://github.com/orgs/sunnypilot/projects/2), and to install the new branches, you can use the following links
+
+
+> [!IMPORTANT]
+> It is recommended to [re-flash AGNOS](https://flash.comma.ai/) if you intend to downgrade from the new branches.
+> You can still restore the latest sunnylink backup made on the old branches.
+
+|      Branch      |                 Installation URL              |
+|:----------------:|:---------------------------------------------:|
+| `staging-c3-new` | `https://staging-c3-new.sunnypilot.ai`        |
+|   `dev-c3-new`   | `https://dev-c3-new.sunnypilot.ai`            |
+| `custom-branch`  | `https://install.sunnypilot.ai/{branch_name}` |
+| `release-c3-new` |            **Not yet available**.             |
+
+> [!TIP]
+> Do you require further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
 
 ## 🎆 Pull Requests
 We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.
 
-Pull requests should be against the most current `master` branch.
+Pull requests should be against the most current `master-new` branch.
 
 ## 📊 User Data
 
@@ -28,7 +73,7 @@ By default, sunnypilot uploads the driving data to comma servers. You can also a
 sunnypilot is open source software. The user is free to disable data collection if they wish to do so.
 
 sunnypilot logs the road-facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
+The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
 
 By using this software, you understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
 

RELEASES.md

@@ -1,49 +1,5 @@
-Version 0.10.4 (2026-02-17)
+Version 0.9.10 (2025-06-30)
 ========================
-* Lexus LS 2018 support thanks to Hacheoy!
-
-Version 0.10.3 (2025-12-17)
-========================
-* New driving model #36249
-  * New temporal policy architecture
-  * New on-policy training physics noise model
-* New driver monitoring model #36409
-  * Trained on a new dataset, including comma four data
-* Improved inter-process communication memory efficiency
-
-Version 0.10.2 (2025-11-19)
-========================
-* comma four support
-
-Version 0.10.1 (2025-09-08)
-========================
-* New driving model #36276
-  * World Model: removed global localization inputs
-  * World Model: 2x the number of parameters
-  * World Model: trained on 4x the number of segments
-  * VAE Compression Model: new architecture and training objective
-  * Driving Vision Model: trained on 4x the number of segments
-* New Driver Monitoring model #36198
-* Acura TLX 2021 support thanks to MVL!
-* Honda City 2023 support thanks to vanillagorillaa and drFritz!
-* Honda N-Box 2018 support thanks to miettal!
-* Honda Odyssey 2021-25 support thanks to csouers and MVL!
-* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
-
-Version 0.10.0 (2025-08-05)
-========================
-* New driving model
-  * New training architecture
-     * Described in our CVPR paper: "Learning to Drive from a World Model"
-     * Longitudinal MPC replaced by E2E planning from World Model in Experimental Mode
-     * Action from lateral MPC as training objective replaced by E2E planning from World Model
-  * Low-speed lead car ground-truth fixes
-* Enable live-learned steering actuation delay
-* Opt-in audio recording for dashcam video
-* Acura MDX 2025 support thanks to vanillagorillaa and MVL!
-* Honda Accord 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda CR-V 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda Pilot 2023-25 support thanks to vanillagorillaa and MVL!
 
 Version 0.9.9 (2025-05-23)
 ========================

SConstruct

@@ -3,51 +3,182 @@ import subprocess
 import sys
 import sysconfig
 import platform
-import shlex
 import numpy as np
 
 import SCons.Errors
 
 SCons.Warnings.warningAsException(True)
 
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
+TICI = os.path.isfile('/TICI')
+AGNOS = TICI
+
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+SetOption('num_jobs', int(os.cpu_count()/2))
+
+AddOption('--kaitai',
+          action='store_true',
+          help='Regenerate kaitai struct parsers')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--compile_db',
+          action='store_true',
+          help='build clang compilation database')
+
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
 
-AddOption('--asan', action='store_true', help='turn on ASAN')
-AddOption('--ubsan', action='store_true', help='turn on UBSan')
-AddOption('--mutation', action='store_true', help='generate mutation-ready code')
-AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-# Detect platform
-arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+AddOption('--stock-ui',
+          action='store_true',
+          dest='stock_ui',
+          default=False,
+          help='Build stock openpilot UI instead of sunnypilot UI')
+
+## Architecture name breakdown (arch)
+## - larch64: linux tici aarch64
+## - aarch64: linux pc aarch64
+## - x86_64:  linux pc x64
+## - Darwin:  mac x64 or arm64
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
   brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-elif arch == "aarch64" and os.path.isfile('/TICI'):
+elif arch == "aarch64" and AGNOS:
   arch = "larch64"
-assert arch in [
-  "larch64",  # linux tici arm64
-  "aarch64",  # linux pc arm64
-  "x86_64",   # linux pc x64
-  "Darwin",   # macOS arm64 (x86 not supported)
-]
+assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
+
+lenv = {
+  "PATH": os.environ['PATH'],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+}
+
+rpath = lenv["LD_LIBRARY_PATH"].copy()
+
+if arch == "larch64":
+  cpppath = [
+    "#third_party/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/local/lib",
+    "/system/vendor/lib64",
+    f"#third_party/acados/{arch}/lib",
+  ]
+
+  libpath += [
+    "#third_party/snpe/larch64",
+    "#third_party/libyuv/larch64/lib",
+    "/usr/lib/aarch64-linux-gnu"
+  ]
+  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  rpath += ["/usr/local/lib"]
+else:
+  cflags = []
+  cxxflags = []
+  cpppath = []
+  rpath += []
+
+  # MacOS
+  if arch == "Darwin":
+    libpath = [
+      f"#third_party/libyuv/{arch}/lib",
+      f"#third_party/acados/{arch}/lib",
+      f"{brew_prefix}/lib",
+      f"{brew_prefix}/opt/openssl@3.0/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += [
+      f"{brew_prefix}/include",
+      f"{brew_prefix}/opt/openssl@3.0/include",
+    ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
+  # Linux
+  else:
+    libpath = [
+      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{arch}/lib",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
+
+if GetOption('asan'):
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
+else:
+  ccflags = []
+  ldflags = []
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
+
+if not GetOption('stock_ui'):
+  cflags += ["-DSUNNYPILOT"]
+  cxxflags += ["-DSUNNYPILOT"]
+
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
 
 env = Environment(
-  ENV={
-    "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
-  },
-  CC='clang',
-  CXX='clang++',
+  ENV=lenv,
   CCFLAGS=[
     "-g",
     "-fPIC",
@@ -60,31 +191,37 @@ env = Environment(
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
     "-Wno-vla-cxx-extension",
-  ],
-  CFLAGS=["-std=gnu11"],
-  CXXFLAGS=["-std=c++1z"],
-  CPPPATH=[
+  ] + cflags + ccflags,
+
+  CPPPATH=cpppath + [
     "#",
-    "#msgq",
-    "#third_party",
-    "#third_party/json11",
-    "#third_party/linux/include",
     "#third_party/acados/include",
     "#third_party/acados/include/blasfeo/include",
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
     "#third_party/libyuv/include",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/snpe/include",
+    "#third_party",
+    "#msgq",
   ],
-  LIBPATH=[
-    "#common",
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
+  LIBPATH=libpath + [
     "#msgq_repo",
     "#third_party",
     "#selfdrive/pandad",
+    "#common",
     "#rednose/helpers",
-    f"#third_party/libyuv/{arch}/lib",
-    f"#third_party/acados/{arch}/lib",
   ],
-  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
@@ -92,63 +229,31 @@ env = Environment(
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
 
-# Arch-specific flags and paths
-if arch == "larch64":
-  env.Append(CPPPATH=["#third_party/opencl/include"])
-  env.Append(LIBPATH=[
-    "/usr/local/lib",
-    "/system/vendor/lib64",
-    "/usr/lib/aarch64-linux-gnu",
-  ])
-  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
-  env.Append(CCFLAGS=arch_flags)
-  env.Append(CXXFLAGS=arch_flags)
-elif arch == "Darwin":
-  env.Append(LIBPATH=[
-    f"{brew_prefix}/lib",
-    f"{brew_prefix}/opt/openssl@3.0/lib",
-    f"{brew_prefix}/opt/llvm/lib/c++",
-    "/System/Library/Frameworks/OpenGL.framework/Libraries",
-  ])
-  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
-  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
-  env.Append(CPPPATH=[
-    f"{brew_prefix}/include",
-    f"{brew_prefix}/opt/openssl@3.0/include",
-  ])
-else:
-  env.Append(LIBPATH=[
-    "/usr/lib",
-    "/usr/local/lib",
-  ])
+if arch == "Darwin":
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
 
-# Sanitizers and extra CCFLAGS from CLI
-if GetOption('asan'):
-  env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
-  env.Append(LINKFLAGS=["-fsanitize=address"])
-elif GetOption('ubsan'):
-  env.Append(CCFLAGS=["-fsanitize=undefined"])
-  env.Append(LINKFLAGS=["-fsanitize=undefined"])
+if GetOption('compile_db'):
+  env.CompilationDatabase('compile_commands.json')
 
-_extra_cc = shlex.split(GetOption('ccflags') or '')
-if _extra_cc:
-  env.Append(CCFLAGS=_extra_cc)
+# Setup cache dir
+default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
 
-# no --as-needed on mac linker
-if arch != "Darwin":
-  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
-
-# progress output
 node_interval = 5
 node_count = 0
 def progress_function(node):
   global node_count
   node_count += node_interval
   sys.stderr.write("progress: %d\n" % node_count)
+
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-# ********** Cython build environment **********
+# Cython build environment
 py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
 envCython["CPPPATH"] += [py_include, np.get_include()]
@@ -157,28 +262,84 @@ envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch')
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
 
-# Setup cache dir
-default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
-cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+qt_libs = []
+if arch == "Darwin":
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
+else:
+  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
+  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
 
-# ********** start building stuff **********
+  qt_env['QTDIR'] = qt_install_prefix
+  qt_dirs = [
+    f"{qt_install_headers}",
+  ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
+  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+qt_env['QT3DIR'] = qt_env['QTDIR']
+qt_env.Tool('qt3')
+
+qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_CORE_LIB",
+  "-DQT_MESSAGELOGCONTEXT",
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui', ]
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+
+Export('env', 'qt_env', 'arch', 'real_arch')
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common')
+Import('_common', '_gpucommon')
+
 common = [_common, 'json11', 'zmq']
-Export('common')
+gpucommon = [_gpucommon]
+
+Export('common', 'gpucommon')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
 # Enable swaglog include in submodules
@@ -202,8 +363,14 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
+  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
+if arch != "Darwin":
+  SConscript([
+    'system/logcatd/SConscript',
+    'system/proclogd/SConscript',
+  ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
@@ -220,5 +387,6 @@ if Dir('#tools/cabana/').exists() and GetOption('extras'):
   if arch != "larch64":
     SConscript(['tools/cabana/SConscript'])
 
-
-env.CompilationDatabase('compile_commands.json')
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

cereal/SConscript

@@ -13,7 +13,7 @@ cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
 socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 

cereal/custom.capnp

@@ -25,92 +25,8 @@ struct ModularAssistiveDrivingSystem {
   }
 }
 
-struct IntelligentCruiseButtonManagement {
-  state @0 :IntelligentCruiseButtonManagementState;
-  sendButton @1 :SendButtonState;
-  vTarget @2 :Float32;
-
-  enum IntelligentCruiseButtonManagementState {
-    inactive @0;      # No button press or default state
-    preActive @1;     # Pre-active state before transitioning to increasing or decreasing
-    increasing @2;    # Increasing speed
-    decreasing @3;    # Decreasing speed
-    holding @4;       # Holding steady speed
-  }
-
-  enum SendButtonState {
-    none @0;
-    increase @1;
-    decrease @2;
-  }
-}
-
-# Same struct as Log.RadarState.LeadData
-struct LeadData {
-  dRel @0 :Float32;
-  yRel @1 :Float32;
-  vRel @2 :Float32;
-  aRel @3 :Float32;
-  vLead @4 :Float32;
-  dPath @6 :Float32;
-  vLat @7 :Float32;
-  vLeadK @8 :Float32;
-  aLeadK @9 :Float32;
-  fcw @10 :Bool;
-  status @11 :Bool;
-  aLeadTau @12 :Float32;
-  modelProb @13 :Float32;
-  radar @14 :Bool;
-  radarTrackId @15 :Int32 = -1;
-
-  aLeadDEPRECATED @5 :Float32;
-}
-
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
   mads @0 :ModularAssistiveDrivingSystem;
-  intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
-
-  enum AudibleAlert {
-    none @0;
-
-    engage @1;
-    disengage @2;
-    refuse @3;
-
-    warningSoft @4;
-    warningImmediate @5;
-
-    prompt @6;
-    promptRepeat @7;
-    promptDistracted @8;
-
-    # unused, these are reserved for upstream events so we don't collide
-    reserved9 @9;
-    reserved10 @10;
-    reserved11 @11;
-    reserved12 @12;
-    reserved13 @13;
-    reserved14 @14;
-    reserved15 @15;
-    reserved16 @16;
-    reserved17 @17;
-    reserved18 @18;
-    reserved19 @19;
-    reserved20 @20;
-    reserved21 @21;
-    reserved22 @22;
-    reserved23 @23;
-    reserved24 @24;
-    reserved25 @25;
-    reserved26 @26;
-    reserved27 @27;
-    reserved28 @28;
-    reserved29 @29;
-    reserved30 @30;
-
-    promptSingleLow @31;
-    promptSingleHigh @32;
-  }
 }
 
 struct ModelManagerSP @0xaedffd8f31e7b55d {
@@ -153,7 +69,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
       navigation @1;
       vision @2;
       policy @3;
-      offPolicy @4;
     }
   }
 
@@ -186,13 +101,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
 
 struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
   dec @0 :DynamicExperimentalControl;
-  longitudinalPlanSource @1 :LongitudinalPlanSource;
-  smartCruiseControl @2 :SmartCruiseControl;
-  speedLimit @3 :SpeedLimit;
-  vTarget @4 :Float32;
-  aTarget @5 :Float32;
-  events @6 :List(OnroadEventSP.Event);
-  e2eAlerts @7 :E2eAlerts;
 
   struct DynamicExperimentalControl {
     state @0 :DynamicExperimentalControlState;
@@ -204,97 +112,6 @@ struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
       blended @1;
     }
   }
-
-  struct SmartCruiseControl {
-    vision @0 :Vision;
-    map @1 :Map;
-
-    struct Vision {
-      state @0 :VisionState;
-      vTarget @1 :Float32;
-      aTarget @2 :Float32;
-      currentLateralAccel @3 :Float32;
-      maxPredictedLateralAccel @4 :Float32;
-      enabled @5 :Bool;
-      active @6 :Bool;
-    }
-
-    struct Map {
-      state @0 :MapState;
-      vTarget @1 :Float32;
-      aTarget @2 :Float32;
-      enabled @3 :Bool;
-      active @4 :Bool;
-    }
-
-    enum VisionState {
-      disabled @0; # System disabled or inactive.
-      enabled @1; # No predicted substantial turn on vision range.
-      entering @2; # A substantial turn is predicted ahead, adapting speed to turn comfort levels.
-      turning @3; # Actively turning. Managing acceleration to provide a roll on turn feeling.
-      leaving @4; # Road ahead straightens. Start to allow positive acceleration.
-      overriding @5; # System overriding with manual control.
-    }
-
-    enum MapState {
-      disabled @0; # System disabled or inactive.
-      enabled @1; # No predicted substantial turn on map range.
-      turning @2; # Actively turning. Managing acceleration to provide a roll on turn feeling.
-      overriding @3; # System overriding with manual control.
-    }
-  }
-
-  struct SpeedLimit {
-    resolver @0 :Resolver;
-    assist @1 :Assist;
-
-    struct Resolver {
-      speedLimit @0 :Float32;
-      distToSpeedLimit @1 :Float32;
-      source @2 :Source;
-      speedLimitOffset @3 :Float32;
-      speedLimitLast @4 :Float32;
-      speedLimitFinal @5 :Float32;
-      speedLimitFinalLast @6 :Float32;
-      speedLimitValid @7 :Bool;
-      speedLimitLastValid @8 :Bool;
-    }
-
-    struct Assist {
-      state @0 :AssistState;
-      enabled @1 :Bool;
-      active @2 :Bool;
-      vTarget @3 :Float32;
-      aTarget @4 :Float32;
-    }
-
-    enum Source {
-      none @0;
-      car @1;
-      map @2;
-    }
-
-    enum AssistState {
-      disabled @0;
-      inactive @1; # No speed limit set or not enabled by parameter.
-      preActive @2;
-      pending @3; # Awaiting new speed limit.
-      adapting @4; # Reducing speed to match new speed limit.
-      active @5; # Cruising at speed limit.
-    }
-  }
-
-  enum LongitudinalPlanSource {
-    cruise @0;
-    sccVision @1;
-    sccMap @2;
-    speedLimitAssist @3;
-  }
-
-  struct E2eAlerts {
-    greenLightAlert @0 :Bool;
-    leadDepartAlert @1 :Bool;
-  }
 }
 
 struct OnroadEventSP @0xda96579883444c35 {
@@ -334,22 +151,12 @@ struct OnroadEventSP @0xda96579883444c35 {
     experimentalModeSwitched @14;
     wrongCarModeAlertOnly @15;
     pedalPressedAlertOnly @16;
-    laneTurnLeft @17;
-    laneTurnRight @18;
-    speedLimitPreActive @19;
-    speedLimitActive @20;
-    speedLimitChanged @21;
-    speedLimitPending @22;
-    e2eChime @23;
   }
 }
 
 struct CarParamsSP @0x80ae746ee2596b11 {
   flags @0 :UInt32;        # flags for car specific quirks in sunnypilot
   safetyParam @1 : Int16;  # flags for sunnypilot's custom safety flags
-  pcmCruiseSpeed @3 :Bool;
-  intelligentCruiseButtonManagementAvailable @4 :Bool;
-  enableGasInterceptor @5 :Bool;
 
   neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
 
@@ -367,26 +174,10 @@ struct CarParamsSP @0x80ae746ee2596b11 {
 struct CarControlSP @0xa5cd762cd951a455 {
   mads @0 :ModularAssistiveDrivingSystem;
   params @1 :List(Param);
-  leadOne @2 :LeadData;
-  leadTwo @3 :LeadData;
-  intelligentCruiseButtonManagement @4 :IntelligentCruiseButtonManagement;
 
   struct Param {
     key @0 :Text;
-    type @2 :ParamType;
-    value @3 :Data;
-
-    valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
-  }
-
-  enum ParamType {
-    string @0;
-    bool @1;
-    int @2;
-    float @3;
-    time @4;
-    json @5;
-    bytes @6;
+    value @1 :Text;
   }
 }
 
@@ -433,7 +224,6 @@ struct BackupManagerSP @0xf98d843bfd7004a3 {
 }
 
 struct CarStateSP @0xb86e6369214c01c8 {
-  speedLimit @0 :Float32;
 }
 
 struct LiveMapDataSP @0xf416ec09499d9d19 {
@@ -445,14 +235,7 @@ struct LiveMapDataSP @0xf416ec09499d9d19 {
   roadName @5 :Text;
 }
 
-struct ModelDataV2SP @0xa1680744031fdb2d {
-  laneTurnDirection @0 :TurnDirection;
-
-  enum TurnDirection {
-    none @0;
-    turnLeft @1;
-    turnRight @2;
-  }
+struct CustomReserved9 @0xa1680744031fdb2d {
 }
 
 struct CustomReserved10 @0xcb9fd56c7057593a {

cereal/log.capnp

@@ -87,7 +87,6 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     laneChange @50;
     lowMemory @51;
     stockAeb @52;
-    stockLkas @98;
     ldw @53;
     carUnrecognized @54;
     invalidLkasSetting @55;
@@ -128,9 +127,7 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     espActive @90;
     personalityChanged @91;
     aeb @92;
-    userBookmark @95;
-    excessiveActuation @96;
-    audioFeedback @97;
+    userFlag @95;
 
     soundsUnavailableDEPRECATED @47;
   }
@@ -495,6 +492,7 @@ struct DeviceState @0xa4d8b5af2aa492eb {
   gpuTempC @27 :List(Float32);
   dspTempC @49 :Float32;
   memoryTempC @28 :Float32;
+  nvmeTempC @35 :List(Float32);
   modemTempC @36 :List(Float32);
   pmicTempC @39 :List(Float32);
   intakeTempC @46 :Float32;
@@ -570,7 +568,6 @@ struct DeviceState @0xa4d8b5af2aa492eb {
   chargingDisabledDEPRECATED @18 :Bool;
   usbOnlineDEPRECATED @12 :Bool;
   ambientTempCDEPRECATED @30 :Float32;
-  nvmeTempCDEPRECATED @35 :List(Float32);
 }
 
 struct PandaState @0xa7649e2575e4591e {
@@ -586,8 +583,9 @@ struct PandaState @0xa7649e2575e4591e {
   heartbeatLost @22 :Bool;
   interruptLoad @25 :Float32;
   fanPower @28 :UInt8;
+  fanStallCount @34 :UInt8;
 
-  spiErrorCount @33 :UInt16;
+  spiChecksumErrorCount @33 :UInt16;
 
   harnessStatus @21 :HarnessStatus;
   sbu1Voltage @35 :Float32;
@@ -714,7 +712,6 @@ struct PandaState @0xa7649e2575e4591e {
   usbPowerModeDEPRECATED @12 :PeripheralState.UsbPowerModeDEPRECATED;
   safetyParamDEPRECATED @20 :Int16;
   safetyParam2DEPRECATED @26 :UInt32;
-  fanStallCountDEPRECATED @34 :UInt8;
 }
 
 struct PeripheralState {
@@ -919,8 +916,6 @@ struct ControlsState @0x97ff69c53601abf1 {
     saturated @7 :Bool;
     actualLateralAccel @9 :Float32;
     desiredLateralAccel @10 :Float32;
-    desiredLateralJerk @11 :Float32;
-    version @12 :Int32;
    }
 
   struct LateralLQRState {
@@ -1088,7 +1083,7 @@ struct ModelDataV2 {
   confidence @23: ConfidenceClass;
 
   # Model perceived motion
-  temporalPoseDEPRECATED @21 :Pose;
+  temporalPose @21 :Pose;
 
   # e2e lateral planner
   action @26: Action;
@@ -1478,11 +1473,6 @@ struct ProcLog {
 
     cmdline @15 :List(Text);
     exe @16 :Text;
-
-    # from /proc/<pid>/smaps_rollup (proportional/private memory)
-    memPss @17 :UInt64;        # Pss — shared pages split by mapper count
-    memPssAnon @18 :UInt64;    # Pss_Anon — private anonymous (heap, stack)
-    memPssShmem @19 :UInt64;   # Pss_Shmem — proportional MSGQ/tmpfs share
   }
 
   struct CPUTimes {
@@ -2154,10 +2144,13 @@ struct Joystick {
 struct DriverStateV2 {
   frameId @0 :UInt32;
   modelExecutionTime @1 :Float32;
+  dspExecutionTimeDEPRECATED @2 :Float32;
   gpuExecutionTime @8 :Float32;
   rawPredictions @3 :Data;
 
+  poorVisionProb @4 :Float32;
   wheelOnRightProb @5 :Float32;
+
   leftDriverData @6 :DriverData;
   rightDriverData @7 :DriverData;
 
@@ -2172,14 +2165,10 @@ struct DriverStateV2 {
     leftBlinkProb @7 :Float32;
     rightBlinkProb @8 :Float32;
     sunglassesProb @9 :Float32;
-    phoneProb @13 :Float32;
-    notReadyProbDEPRECATED @12 :List(Float32);
-    occludedProbDEPRECATED @10 :Float32;
-    readyProbDEPRECATED @11 :List(Float32);
+    occludedProb @10 :Float32;
+    readyProb @11 :List(Float32);
+    notReadyProb @12 :List(Float32);
   }
-
-  dspExecutionTimeDEPRECATED @2 :Float32;
-  poorVisionProbDEPRECATED @4 :Float32;
 }
 
 struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
@@ -2231,9 +2220,6 @@ struct DriverMonitoringState @0xb83cda094a1da284 {
   hiStdCount @14 :UInt32;
   isActiveMode @16 :Bool;
   isRHD @4 :Bool;
-  uncertainCount @19 :UInt32;
-  phoneProbOffset @20 :Float32;
-  phoneProbValidCount @21 :UInt32;
 
   isPreviewDEPRECATED @15 :Bool;
   rhdCheckedDEPRECATED @5 :Bool;
@@ -2481,27 +2467,16 @@ struct DebugAlert {
   alertText2 @1 :Text;
 }
 
-struct UserBookmark @0xfe346a9de48d9b50 {
+struct UserFlag {
 }
 
-struct SoundPressure @0xdc24138990726023 {
+struct Microphone {
   soundPressure @0 :Float32;
 
   # uncalibrated, A-weighted
   soundPressureWeighted @3 :Float32;
   soundPressureWeightedDb @1 :Float32;
-
-  filteredSoundPressureWeightedDbDEPRECATED @2 :Float32;
-}
-
-struct AudioData {
-  data @0 :Data;
-  sampleRate @1 :UInt32;
-}
-
-struct AudioFeedback {
-  audio @0 :AudioData;
-  blockNum @1 :UInt16;
+  filteredSoundPressureWeightedDb @2 :Float32;
 }
 
 struct Touch {
@@ -2530,10 +2505,13 @@ struct Event {
     controlsState @7 :ControlsState;
     selfdriveState @130 :SelfdriveState;
     gyroscope @99 :SensorEventData;
+    gyroscope2 @100 :SensorEventData;
     accelerometer @98 :SensorEventData;
+    accelerometer2 @101 :SensorEventData;
     magnetometer @95 :SensorEventData;
     lightSensor @96 :SensorEventData;
     temperatureSensor @97 :SensorEventData;
+    temperatureSensor2 @123 :SensorEventData;
     pandaStates @81 :List(PandaState);
     peripheralState @80 :PeripheralState;
     radarState @13 :RadarState;
@@ -2578,8 +2556,7 @@ struct Event {
     livestreamDriverEncodeIdx @119 :EncodeIndex;
 
     # microphone data
-    soundPressure @103 :SoundPressure;
-    rawAudioData @147 :AudioData;
+    microphone @103 :Microphone;
 
     # systems stuff
     androidLog @20 :AndroidLogEntry;
@@ -2601,13 +2578,9 @@ struct Event {
     mapRenderState @105: MapRenderState;
 
     # UI services
+    userFlag @93 :UserFlag;
     uiDebug @102 :UIDebug;
 
-    # driving feedback
-    userBookmark @93 :UserBookmark;
-    bookmarkButton @148 :UserBookmark;
-    audioFeedback @149 :AudioFeedback;
-
     # *********** debug ***********
     testJoystick @52 :Joystick;
     roadEncodeData @86 :EncodeData;
@@ -2640,7 +2613,7 @@ struct Event {
     backupManagerSP @113 :Custom.BackupManagerSP;
     carStateSP @114 :Custom.CarStateSP;
     liveMapDataSP @115 :Custom.LiveMapDataSP;
-    modelDataV2SP @116 :Custom.ModelDataV2SP;
+    customReserved9 @116 :Custom.CustomReserved9;
     customReserved10 @136 :Custom.CustomReserved10;
     customReserved11 @137 :Custom.CustomReserved11;
     customReserved12 @138 :Custom.CustomReserved12;
@@ -2693,11 +2666,8 @@ struct Event {
     lateralPlanDEPRECATED @64 :LateralPlan;
     navModelDEPRECATED @104 :NavModelData;
     uiPlanDEPRECATED @106 :UiPlan;
-    liveLocationKalman @72 :LiveLocationKalman;
+    liveLocationKalmanDEPRECATED @72 :LiveLocationKalman;
     liveTracksDEPRECATED @16 :List(LiveTracksDEPRECATED);
     onroadEventsDEPRECATED @68: List(Car.OnroadEventDEPRECATED);
-    gyroscope2DEPRECATED @100 :SensorEventData;
-    accelerometer2DEPRECATED @101 :SensorEventData;
-    temperatureSensor2DEPRECATED @123 :SensorEventData;
   }
 }

cereal/messaging/__init__.py

@@ -1,8 +1,10 @@
 # must be built with scons
-from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
-                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
 import msgq
+
 import os
 import capnp
 import time
@@ -11,25 +13,11 @@ from typing import Optional, List, Union, Dict
 
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def pub_sock(endpoint: str) -> PubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.pub_sock(endpoint, segment_size)
-
-
-def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
-                       timeout=timeout, segment_size=segment_size)
-
-
 def reset_context():
   msgq.context = Context()
 

cereal/messaging/bridge.cc

@@ -25,16 +25,15 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<BridgeZmqPoller>();
-  auto pub_context = std::make_unique<Context>();
-  auto sub_context = std::make_unique<BridgeZmqContext>();
-  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
+  auto poller = std::make_unique<ZMQPoller>();
+  auto pub_context = std::make_unique<MSGQContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
+  std::map<SubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new PubSocket();
-    auto sub_sock = new BridgeZmqSubSocket();
-    size_t queue_size = services.at(endpoint).queue_size;
-    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
+    auto pub_sock = new MSGQPubSocket();
+    auto sub_sock = new ZMQSubSocket();
+    pub_sock->connect(pub_context.get(), endpoint);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/bridge_zmq.cc

@@ -1,170 +0,0 @@
-#include "cereal/messaging/bridge_zmq.h"
-
-#include <cassert>
-#include <cstring>
-#include <unistd.h>
-
-static size_t fnv1a_hash(const std::string &str) {
-  const size_t fnv_prime = 0x100000001b3;
-  size_t hash_value = 0xcbf29ce484222325;
-  for (char c : str) {
-    hash_value ^= (unsigned char)c;
-    hash_value *= fnv_prime;
-  }
-  return hash_value;
-}
-
-// FIXME: This is a hack to get the port number from the socket name, might have collisions.
-static int get_port(std::string endpoint) {
-  size_t hash_value = fnv1a_hash(endpoint);
-  int start_port = 8023;
-  int max_port = 65535;
-  return start_port + (hash_value % (max_port - start_port));
-}
-
-BridgeZmqContext::BridgeZmqContext() {
-  context = zmq_ctx_new();
-}
-
-BridgeZmqContext::~BridgeZmqContext() {
-  if (context != nullptr) {
-    zmq_ctx_term(context);
-  }
-}
-
-void BridgeZmqMessage::init(size_t sz) {
-  size = sz;
-  data = new char[size];
-}
-
-void BridgeZmqMessage::init(char *d, size_t sz) {
-  size = sz;
-  data = new char[size];
-  memcpy(data, d, size);
-}
-
-void BridgeZmqMessage::close() {
-  if (size > 0) {
-    delete[] data;
-  }
-  data = nullptr;
-  size = 0;
-}
-
-BridgeZmqMessage::~BridgeZmqMessage() {
-  close();
-}
-
-int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
-
-  if (conflate) {
-    int arg = 1;
-    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
-  }
-
-  int reconnect_ivl = 500;
-  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
-
-  full_endpoint = "tcp://" + address + ":";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  return zmq_connect(sock, full_endpoint.c_str());
-}
-
-void BridgeZmqSubSocket::setTimeout(int timeout) {
-  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
-}
-
-Message *BridgeZmqSubSocket::receive(bool non_blocking) {
-  zmq_msg_t msg;
-  assert(zmq_msg_init(&msg) == 0);
-
-  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
-  int rc = zmq_msg_recv(&msg, sock, flags);
-
-  Message *ret = nullptr;
-  if (rc >= 0) {
-    ret = new BridgeZmqMessage;
-    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
-  }
-
-  zmq_msg_close(&msg);
-  return ret;
-}
-
-BridgeZmqSubSocket::~BridgeZmqSubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  full_endpoint = "tcp://*:";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
-  pid = getpid();
-
-  return zmq_bind(sock, full_endpoint.c_str());
-}
-
-int BridgeZmqPubSocket::sendMessage(Message *message) {
-  assert(pid == getpid());
-  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
-}
-
-int BridgeZmqPubSocket::send(char *data, size_t size) {
-  assert(pid == getpid());
-  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
-}
-
-BridgeZmqPubSocket::~BridgeZmqPubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
-  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
-  polls[num_polls].socket = socket->getRawSocket();
-  polls[num_polls].events = ZMQ_POLLIN;
-
-  sockets.push_back(socket);
-  num_polls++;
-}
-
-std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
-  std::vector<BridgeZmqSubSocket *> ret;
-
-  int rc = zmq_poll(polls, num_polls, timeout);
-  if (rc < 0) {
-    return ret;
-  }
-
-  for (size_t i = 0; i < num_polls; i++) {
-    if (polls[i].revents) {
-      ret.push_back(sockets[i]);
-    }
-  }
-
-  return ret;
-}

cereal/messaging/bridge_zmq.h

@@ -1,72 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <string>
-#include <vector>
-
-#include <zmq.h>
-
-#include "msgq/ipc.h"
-
-class BridgeZmqContext {
-public:
-  BridgeZmqContext();
-  void *getRawContext() { return context; }
-  ~BridgeZmqContext();
-
-private:
-  void *context = nullptr;
-};
-
-class BridgeZmqMessage : public Message {
-public:
-  void init(size_t size);
-  void init(char *data, size_t size);
-  void close();
-  size_t getSize() { return size; }
-  char *getData() { return data; }
-  ~BridgeZmqMessage();
-
-private:
-  char *data = nullptr;
-  size_t size = 0;
-};
-
-class BridgeZmqSubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
-  void setTimeout(int timeout);
-  Message *receive(bool non_blocking = false);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqSubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-};
-
-class BridgeZmqPubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
-  int sendMessage(Message *message);
-  int send(char *data, size_t size);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqPubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-  int pid = -1;
-};
-
-class BridgeZmqPoller {
-public:
-  void registerSocket(BridgeZmqSubSocket *socket);
-  std::vector<BridgeZmqSubSocket *> poll(int timeout);
-
-private:
-  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
-  std::vector<BridgeZmqSubSocket *> sockets;
-  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
-  size_t num_polls = 0;
-};

cereal/messaging/msgq_to_zmq.cc

@@ -2,7 +2,6 @@
 
 #include <cassert>
 
-#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -22,14 +21,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
-  msgq_context = std::make_unique<Context>();
+  zmq_context = std::make_unique<ZMQContext>();
+  msgq_context = std::make_unique<MSGQContext>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -49,7 +48,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -72,7 +71,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -109,8 +108,7 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -130,7 +128,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,8 +7,9 @@
 #include <string>
 #include <vector>
 
+#define private public
 #include "msgq/impl_msgq.h"
-#include "cereal/messaging/bridge_zmq.h"
+#include "msgq/impl_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -21,16 +22,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<Context> msgq_context;
-  std::unique_ptr<BridgeZmqContext> zmq_context;
+  std::unique_ptr<MSGQContext> msgq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -33,7 +33,7 @@ MessageContext message_context;
 struct SubMaster::SubMessage {
   std::string name;
   SubSocket *socket = nullptr;
-  float freq = 0.0f;
+  int freq = 0;
   bool updated = false, alive = false, valid = false, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,8 +187,7 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    service serv = services.at(std::string(name));
-    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
+    PubSocket *socket = PubSocket::create(message_context.context(), name);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/messaging/tests/test_messaging.py

@@ -30,7 +30,7 @@ def zmq_sleep(t=1):
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:
@@ -177,8 +177,8 @@ class TestMessaging:
 
     # wait 5 socket timeouts before sending
     msg = random_carstate()
-    start_time = time.monotonic()
     delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
+    start_time = time.monotonic()
     recvd = messaging.recv_one_retry(sub_sock)
     assert (time.monotonic() - start_time) >= sock_timeout*5
     assert isinstance(recvd, capnp._DynamicStructReader)

cereal/messaging/tests/test_pub_sub_master.py

@@ -86,7 +86,7 @@ class TestSubMaster:
         "cameraOdometry": (20, 10),
         "liveCalibration": (4, 4),
         "carParams": (None, None),
-        "userBookmark": (None, None),
+        "userFlag": (None, None),
       }
 
       for service, (max_freq, min_freq) in checks.items():

cereal/services.py

@@ -1,44 +1,37 @@
 #!/usr/bin/env python3
-from enum import IntEnum
 from typing import Optional
 
 
-# TODO: this should be automatically determined using the capnp schema
-class QueueSize(IntEnum):
-  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
-  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
-  SMALL = 250 * 1024          # 250KB - most services
-
-
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
-               queue_size: QueueSize = QueueSize.SMALL):
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
-    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
+  "accelerometer2": (True, 100., 100),
   "magnetometer": (True, 25.),
   "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
   "touch": (True, 20., 1),
-  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10),
   "selfdriveState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
+  "sendcan": (True, 100., 139),
   "logMessage": (True, 0.),
   "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
@@ -50,7 +43,7 @@ _services: dict[str, tuple] = {
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
   "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15, QueueSize.BIG),
+  "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
@@ -72,26 +65,19 @@ _services: dict[str, tuple] = {
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., None, QueueSize.BIG),
+  "modelV2": (True, 20.),
   "managerState": (True, 2., 1),
   "uploaderState": (True, 0., 1),
   "navInstruction": (True, 1., 10),
   "navRoute": (True, 0.),
   "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
-  "userBookmark": (True, 0., 1),
-  "soundPressure": (True, 10., 10),
-  "rawAudioData": (False, 20.),
-  "bookmarkButton": (True, 0., 1),
-  "audioFeedback": (True, 0., 1),
-  "roadEncodeData": (False, 20., None, QueueSize.BIG),
-  "driverEncodeData": (False, 20., None, QueueSize.BIG),
-  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "userFlag": (True, 0., 1),
+  "microphone": (True, 10., 10),
 
   # sunnypilot
-  "modelManagerSP": (False, 1., 1, QueueSize.BIG),
-  "backupManagerSP": (False, 1., 1, QueueSize.BIG),
+  "modelManagerSP": (False, 1., 1),
+  "backupManagerSP": (False, 1., 1),
   "selfdriveStateSP": (True, 100., 10),
   "longitudinalPlanSP": (True, 20., 10),
   "onroadEventsSP": (True, 1., 1),
@@ -99,19 +85,21 @@ _services: dict[str, tuple] = {
   "carControlSP": (True, 100., 10),
   "carStateSP": (True, 100., 10),
   "liveMapDataSP": (True, 1., 1),
-  "modelDataV2SP": (True, 20., None, QueueSize.BIG),
-  "liveLocationKalman": (True, 20.),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamWideRoadEncodeData": (False, 20.),
+  "livestreamRoadEncodeData": (False, 20.),
+  "livestreamDriverEncodeData": (False, 20.),
   "customReservedRawData0": (True, 0.),
   "customReservedRawData1": (True, 0.),
   "customReservedRawData2": (True, 0.),
@@ -129,13 +117,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
+  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation, v.queue_size)
+    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation)
   h += "};\n"
 
   h += "#endif\n"

codecov.yml

@@ -0,0 +1,13 @@
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch: off
+
+ignore:
+  - "**/test_*.py"
+  - "selfdrive/test/**"
+  - "system/version.py" # codecov changes depending on if we are in a branch or not
+  - "tools"

common/SConscript

@@ -4,12 +4,18 @@ common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
-  'ratekeeper.cc',
-  'clutil.cc',
+  'watchdog.cc',
+  'ratekeeper.cc'
 ]
 
 _common = env.Library('common', common_libs, LIBS="json11")
-Export('_common')
+
+files = [
+  'clutil.cc',
+]
+
+_gpucommon = env.Library('gpucommon', files)
+Export('_common', '_gpucommon')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
@@ -19,6 +25,11 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-common_python = [params_python]
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
 
 Export('common_python')

common/api/__init__.py

@@ -14,13 +14,9 @@ class Api:
   def post(self, *args, **kwargs):
     return self.service.post(*args, **kwargs)
 
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    return self.service.get_token(payload_extra, expiry_hours)
+  def get_token(self, expiry_hours=1):
+    return self.service.get_token(expiry_hours)
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
-  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
-
-
-def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-  return CommaConnectApi(None).get_key_pair()
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, **params)

common/api/base.py

@@ -1,22 +1,18 @@
 import jwt
-import os
 import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
 from openpilot.system.hardware.hw import Paths
 from openpilot.system.version import get_version
 
-# name: jwt signature algorithm
-KEYS = {"id_rsa": "RS256",
-        "id_ecdsa": "ES256"}
-
 
 class BaseApi:
   def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
     self.dongle_id = dongle_id
     self.api_host = api_host
     self.user_agent = user_agent
-    self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
+    with open(f'{Paths.persist_root()}/comma/id_rsa') as f:
+      self.private_key = f.read()
 
   def get(self, *args, **kwargs):
     return self.request('GET', *args, **kwargs)
@@ -27,7 +23,7 @@ class BaseApi:
   def request(self, method, endpoint, timeout=None, access_token=None, **params):
     return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
 
-  def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
+  def _get_token(self, expiry_hours=1, **extra_payload):
     now = datetime.now(UTC).replace(tzinfo=None)
     payload = {
       'identity': self.dongle_id,
@@ -36,22 +32,20 @@ class BaseApi:
       'exp': now + timedelta(hours=expiry_hours),
       **extra_payload
     }
-    if payload_extra is not None:
-      payload.update(payload_extra)
-    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
     if isinstance(token, bytes):
       token = token.decode('utf8')
     return token
 
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    return self._get_token(payload_extra, expiry_hours)
+  def get_token(self, expiry_hours=1):
+    return self._get_token(expiry_hours)
 
   def remove_non_ascii_chars(self, text):
     normalized_text = unicodedata.normalize('NFD', text)
     ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
     return ascii_encoded_text.decode()
 
-  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, json=None, **params):
     headers = {}
     if access_token is not None:
       headers['Authorization'] = "JWT " + access_token
@@ -59,14 +53,4 @@ class BaseApi:
     version = self.remove_non_ascii_chars(get_version())
     headers['User-Agent'] = self.user_agent + version
 
-    # TODO: add session to Api
-    req = requests if session is None else session
-    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
-
-  @staticmethod
-  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-    for key in KEYS:
-      if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
-        with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
-          return KEYS[key], private.read(), public.read()
-    return None, None, None
+    return requests.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)

common/conversions.py

@@ -1,7 +1,6 @@
 import numpy as np
 
-# conversions
-class CV:
+class Conversions:
   # Speed
   MPH_TO_KPH = 1.609344
   KPH_TO_MPH = 1. / MPH_TO_KPH
@@ -18,6 +17,3 @@ class CV:
 
   # Mass
   LB_TO_KG = 0.453592
-
-
-ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/file_helpers.py

@@ -0,0 +1,58 @@
+import io
+import os
+import tempfile
+import contextlib
+import zstandard as zstd
+
+LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+                        overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)
+
+
+def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
+  if not should_compress:
+    file_size = os.path.getsize(filepath)
+    file_stream = open(filepath, "rb")
+    return file_stream, file_size
+
+  # Compress the file on the fly
+  compressed_stream = io.BytesIO()
+  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
+
+  with open(filepath, "rb") as f:
+    compressor.copy_stream(f, compressed_stream)
+    compressed_size = compressed_stream.tell()
+    compressed_stream.seek(0)
+    return compressed_stream, compressed_size

common/filter_simple.py

@@ -15,20 +15,3 @@ class FirstOrderFilter:
       self.initialized = True
       self.x = x
     return self.x
-
-
-class BounceFilter(FirstOrderFilter):
-  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
-    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
-    self.bounce = bounce
-    super().__init__(x0, rc, dt, initialized)
-
-  def update(self, x):
-    super().update(x)
-    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
-    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
-    self.velocity.update(0.0)
-    if abs(self.velocity.x) < 1e-5:
-      self.velocity.x = 0.0
-    self.x += self.velocity.x
-    return self.x

common/git.py

@@ -1,30 +1,30 @@
 from functools import cache
 import subprocess
-from openpilot.common.utils import run_cmd, run_cmd_default
+from openpilot.common.run import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str | None = None) -> str:
+def get_short_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str | None = None) -> str:
+def get_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str | None = None) -> str:
+def get_origin(cwd: str = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
@@ -34,7 +34,7 @@ def get_origin(cwd: str | None = None) -> str:
 
 
 @cache
-def get_normalized_origin(cwd: str | None = None) -> str:
+def get_normalized_origin(cwd: str = None) -> str:
   return get_origin(cwd) \
     .replace("git@", "", 1) \
     .replace(".git", "", 1) \

common/i2c.py

@@ -1,81 +0,0 @@
-import os
-import fcntl
-import ctypes
-
-# I2C constants from /usr/include/linux/i2c-dev.h
-I2C_SLAVE = 0x0703
-I2C_SLAVE_FORCE = 0x0706
-I2C_SMBUS = 0x0720
-
-# SMBus transfer types
-I2C_SMBUS_READ = 1
-I2C_SMBUS_WRITE = 0
-I2C_SMBUS_BYTE_DATA = 2
-I2C_SMBUS_I2C_BLOCK_DATA = 8
-
-I2C_SMBUS_BLOCK_MAX = 32
-
-
-class _I2cSmbusData(ctypes.Union):
-  _fields_ = [
-    ("byte", ctypes.c_uint8),
-    ("word", ctypes.c_uint16),
-    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
-  ]
-
-
-class _I2cSmbusIoctlData(ctypes.Structure):
-  _fields_ = [
-    ("read_write", ctypes.c_uint8),
-    ("command", ctypes.c_uint8),
-    ("size", ctypes.c_uint32),
-    ("data", ctypes.POINTER(_I2cSmbusData)),
-  ]
-
-
-class SMBus:
-  def __init__(self, bus: int):
-    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
-
-  def __enter__(self) -> 'SMBus':
-    return self
-
-  def __exit__(self, *args) -> None:
-    self.close()
-
-  def close(self) -> None:
-    if hasattr(self, '_fd') and self._fd >= 0:
-      os.close(self._fd)
-      self._fd = -1
-
-  def _set_address(self, addr: int, force: bool = False) -> None:
-    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
-    fcntl.ioctl(self._fd, ioctl_arg, addr)
-
-  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
-    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
-    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
-
-  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
-    return int(data.byte)
-
-  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    data.byte = value & 0xFF
-    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
-
-  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
-    self._set_address(addr, force)
-    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
-      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
-
-    data = _I2cSmbusData()
-    data.block[0] = length
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
-    read_len = int(data.block[0]) or length
-    read_len = min(read_len, length)
-    return [int(b) for b in data.block[1 : read_len + 1]]

common/model.h

@@ -1 +1 @@
-#define DEFAULT_MODEL "CD210 (Default)"
+#define DEFAULT_MODEL "Vegetarian Filet o Fish (Default)"

common/params.cc

@@ -103,10 +103,10 @@ Params::~Params() {
   assert(queue.empty());
 }
 
-std::vector<std::string> Params::allKeys(ParamKeyFlag flag) const {
+std::vector<std::string> Params::allKeys(ParamKeyType type) const {
   std::vector<std::string> ret;
   for (auto &p : keys) {
-    if (flag == ALL || (p.second.flags & flag)) {
+    if (type == ALL || (p.second & type)) {
       ret.push_back(p.first);
     }
   }
@@ -117,16 +117,8 @@ bool Params::checkKey(const std::string &key) {
   return keys.find(key) != keys.end();
 }
 
-ParamKeyFlag Params::getKeyFlag(const std::string &key) {
-  return static_cast<ParamKeyFlag>(keys[key].flags);
-}
-
 ParamKeyType Params::getKeyType(const std::string &key) {
-  return keys[key].type;
-}
-
-std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
-  return keys[key].default_value;
+  return static_cast<ParamKeyType>(keys[key]);
 }
 
 int Params::put(const char* key, const char* value, size_t value_size) {
@@ -205,17 +197,17 @@ std::map<std::string, std::string> Params::readAll() {
   return util::read_files_in_dir(getParamPath());
 }
 
-void Params::clearAll(ParamKeyFlag key_flag) {
+void Params::clearAll(ParamKeyType key_type) {
   FileLock file_lock(params_path + "/.lock");
 
-  // 1) delete params of key_flag
+  // 1) delete params of key_type
   // 2) delete files that are not defined in the keys.
   if (DIR *d = opendir(getParamPath().c_str())) {
     struct dirent *de = NULL;
     while ((de = readdir(d))) {
       if (de->d_type != DT_DIR) {
         auto it = keys.find(de->d_name);
-        if (it == keys.end() || (it->second.flags & key_flag)) {
+        if (it == keys.end() || (it->second & key_type)) {
           unlink(getParamPath(de->d_name).c_str());
         }
       }

common/params.h

@@ -2,7 +2,6 @@
 
 #include <future>
 #include <map>
-#include <optional>
 #include <string>
 #include <tuple>
 #include <utility>
@@ -10,34 +9,17 @@
 
 #include "common/queue.h"
 
-enum ParamKeyFlag {
+enum ParamKeyType {
   PERSISTENT = 0x02,
   CLEAR_ON_MANAGER_START = 0x04,
   CLEAR_ON_ONROAD_TRANSITION = 0x08,
   CLEAR_ON_OFFROAD_TRANSITION = 0x10,
   DONT_LOG = 0x20,
   DEVELOPMENT_ONLY = 0x40,
-  CLEAR_ON_IGNITION_ON = 0x80,
-  BACKUP = 0x100,
+  BACKUP = 0x80,
   ALL = 0xFFFFFFFF
 };
 
-enum ParamKeyType {
-  STRING = 0, // must be utf-8 decodable
-  BOOL = 1,
-  INT = 2,
-  FLOAT = 3,
-  TIME = 4, // ISO 8601
-  JSON = 5,
-  BYTES = 6
-};
-
-struct ParamKeyAttributes {
-  uint32_t flags;
-  ParamKeyType type;
-  std::optional<std::string> default_value = std::nullopt;
-};
-
 class Params {
 public:
   explicit Params(const std::string &path = {});
@@ -46,18 +28,16 @@ public:
   Params(const Params&) = delete;
   Params& operator=(const Params&) = delete;
 
-  std::vector<std::string> allKeys(ParamKeyFlag flag = ALL) const;
+  std::vector<std::string> allKeys(ParamKeyType type = ALL) const;
   bool checkKey(const std::string &key);
-  ParamKeyFlag getKeyFlag(const std::string &key);
   ParamKeyType getKeyType(const std::string &key);
-  std::optional<std::string> getKeyDefaultValue(const std::string &key);
   inline std::string getParamPath(const std::string &key = {}) {
     return params_path + params_prefix + (key.empty() ? "" : "/" + key);
   }
 
   // Delete a value
   int remove(const std::string &key);
-  void clearAll(ParamKeyFlag flag);
+  void clearAll(ParamKeyType type);
 
   // helpers for reading values
   std::string get(const std::string &key, bool block = false);

common/params.py

@@ -1,6 +1,5 @@
-from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
 assert Params
-assert ParamKeyFlag
 assert ParamKeyType
 assert UnknownKeyName
 

common/params_keys.h

@@ -3,272 +3,199 @@
 #include <string>
 #include <unordered_map>
 
-#include "cereal/gen/cpp/log.capnp.h"
-
-inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
-    {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
-    {"AdbEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"AlwaysOnDM", {PERSISTENT | BACKUP, BOOL}},
-    {"ApiCache_Device", {PERSISTENT, STRING}},
-    {"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
-    {"AssistNowToken", {PERSISTENT, STRING}},
-    {"AthenadPid", {PERSISTENT, INT}},
-    {"AthenadUploadQueue", {PERSISTENT, JSON}},
-    {"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
-    {"BootCount", {PERSISTENT, INT}},
-    {"CalibrationParams", {PERSISTENT, BYTES}},
-    {"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
-    {"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
-    {"CarBatteryCapacity", {PERSISTENT, INT}},
-    {"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
-    {"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"CarParamsPersistent", {PERSISTENT, BYTES}},
-    {"CarParamsPrevRoute", {PERSISTENT, BYTES}},
-    {"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
-    {"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"CurrentBootlog", {PERSISTENT, STRING}},
-    {"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"DisablePowerDown", {PERSISTENT | BACKUP, BOOL}},
-    {"DisableUpdates", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"DisengageOnAccelerator", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"DongleId", {PERSISTENT, STRING}},
-    {"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
-    {"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
-    {"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
-    {"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"ForcePowerDown", {PERSISTENT, BOOL}},
-    {"GitBranch", {PERSISTENT, STRING}},
-    {"GitCommit", {PERSISTENT, STRING}},
-    {"GitCommitDate", {PERSISTENT, STRING}},
-    {"GitDiff", {PERSISTENT, STRING}},
-    {"GithubSshKeys", {PERSISTENT | BACKUP, STRING}},
-    {"GithubUsername", {PERSISTENT | BACKUP, STRING}},
-    {"GitRemote", {PERSISTENT, STRING}},
-    {"GsmApn", {PERSISTENT | BACKUP, STRING}},
-    {"GsmMetered", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"GsmRoaming", {PERSISTENT | BACKUP, BOOL}},
-    {"HardwareSerial", {PERSISTENT, STRING}},
-    {"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
-    {"InstallDate", {PERSISTENT, TIME}},
-    {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsEngaged", {PERSISTENT, BOOL}},
-    {"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"IsMetric", {PERSISTENT | BACKUP, BOOL}},
-    {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsOnroad", {PERSISTENT, BOOL}},
-    {"IsRhdDetected", {PERSISTENT, BOOL}},
-    {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
-    {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"LastGPSPosition", {PERSISTENT, STRING}},
-    {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
-    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
-    {"LastUpdateTime", {PERSISTENT, TIME}},
-    {"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
-    {"LiveParameters", {PERSISTENT, JSON}},
-    {"LiveParametersV2", {PERSISTENT, BYTES}},
-    {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
-    {"LocationFilterInitialState", {PERSISTENT, BYTES}},
-    {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
-    {"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
-    {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
-    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"PrimeType", {PERSISTENT, INT}},
-    {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
-    {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RecordFront", {PERSISTENT | BACKUP, BOOL}},
-    {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
-    {"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
-    {"ShowDebugInfo", {PERSISTENT, BOOL}},
-    {"RouteCount", {PERSISTENT, INT, "0"}},
-    {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"TermsVersion", {PERSISTENT, STRING}},
-    {"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TrainingVersion", {PERSISTENT, STRING}},
-    {"UbloxAvailable", {PERSISTENT, BOOL}},
-    {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
-    {"UpdaterAvailableBranches", {PERSISTENT, STRING}},
-    {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
-    {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"Version", {PERSISTENT, STRING}},
+inline static std::unordered_map<std::string, uint32_t> keys = {
+    {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
+    {"AdbEnabled", PERSISTENT},
+    {"AlwaysOnDM", PERSISTENT},
+    {"ApiCache_Device", PERSISTENT},
+    {"ApiCache_FirehoseStats", PERSISTENT},
+    {"AssistNowToken", PERSISTENT},
+    {"AthenadPid", PERSISTENT},
+    {"AthenadUploadQueue", PERSISTENT},
+    {"AthenadRecentlyViewedRoutes", PERSISTENT},
+    {"BootCount", PERSISTENT},
+    {"CalibrationParams", PERSISTENT},
+    {"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
+    {"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
+    {"CarBatteryCapacity", PERSISTENT},
+    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsPersistent", PERSISTENT},
+    {"CarParamsPrevRoute", PERSISTENT},
+    {"CompletedTrainingVersion", PERSISTENT},
+    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CurrentBootlog", PERSISTENT},
+    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisablePowerDown", PERSISTENT | BACKUP},
+    {"DisableUpdates", PERSISTENT | BACKUP},
+    {"DisengageOnAccelerator", PERSISTENT | BACKUP},
+    {"DongleId", PERSISTENT},
+    {"DoReboot", CLEAR_ON_MANAGER_START},
+    {"DoShutdown", CLEAR_ON_MANAGER_START},
+    {"DoUninstall", CLEAR_ON_MANAGER_START},
+    {"AlphaLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY | BACKUP},
+    {"ExperimentalMode", PERSISTENT | BACKUP},
+    {"ExperimentalModeConfirmed", PERSISTENT | BACKUP},
+    {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ForcePowerDown", PERSISTENT},
+    {"GitBranch", PERSISTENT},
+    {"GitCommit", PERSISTENT},
+    {"GitCommitDate", PERSISTENT},
+    {"GitDiff", PERSISTENT},
+    {"GithubSshKeys", PERSISTENT | BACKUP},
+    {"GithubUsername", PERSISTENT | BACKUP},
+    {"GitRemote", PERSISTENT},
+    {"GsmApn", PERSISTENT | BACKUP},
+    {"GsmMetered", PERSISTENT | BACKUP},
+    {"GsmRoaming", PERSISTENT | BACKUP},
+    {"HardwareSerial", PERSISTENT},
+    {"HasAcceptedTerms", PERSISTENT},
+    {"InstallDate", PERSISTENT},
+    {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
+    {"IsEngaged", PERSISTENT},
+    {"IsLdwEnabled", PERSISTENT | BACKUP},
+    {"IsMetric", PERSISTENT | BACKUP},
+    {"IsOffroad", CLEAR_ON_MANAGER_START},
+    {"IsOnroad", PERSISTENT},
+    {"IsRhdDetected", PERSISTENT},
+    {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
+    {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
+    {"IsTestedBranch", CLEAR_ON_MANAGER_START},
+    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LanguageSetting", PERSISTENT | BACKUP},
+    {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
+    {"LastGPSPosition", PERSISTENT},
+    {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
+    {"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
+    {"LastUpdateException", CLEAR_ON_MANAGER_START},
+    {"LastUpdateTime", PERSISTENT},
+    {"LiveDelay", PERSISTENT | BACKUP},
+    {"LiveParameters", PERSISTENT},
+    {"LiveParametersV2", PERSISTENT},
+    {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
+    {"LocationFilterInitialState", PERSISTENT},
+    {"LongitudinalManeuverMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LongitudinalPersonality", PERSISTENT | BACKUP},
+    {"NetworkMetered", PERSISTENT},
+    {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
+    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
+    {"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
+    {"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
+    {"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
+    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
+    {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
+    {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
+    {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
+    {"OnroadCycleRequested", CLEAR_ON_MANAGER_START},
+    {"OpenpilotEnabledToggle", PERSISTENT | BACKUP},
+    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSignatures", CLEAR_ON_MANAGER_START},
+    {"PrimeType", PERSISTENT},
+    {"RecordFront", PERSISTENT | BACKUP},
+    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
+    {"SecOCKey", PERSISTENT | DONT_LOG},  // Candidate for | BACKUP
+    {"RouteCount", PERSISTENT},
+    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"SshEnabled", PERSISTENT | BACKUP},
+    {"TermsVersion", PERSISTENT},
+    {"TrainingVersion", PERSISTENT},
+    {"UbloxAvailable", PERSISTENT},
+    {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
+    {"UpdaterAvailableBranches", PERSISTENT},
+    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterState", CLEAR_ON_MANAGER_START},
+    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
+    {"UpdaterLastFetchTime", PERSISTENT},
+    {"Version", PERSISTENT},
 
     // --- sunnypilot params --- //
-    {"ApiCache_DriveStats", {PERSISTENT, JSON}},
-    {"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
-    {"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}},  // seconds
-    {"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}},  // MPH or km/h
-    {"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
-    {"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"CarList", {PERSISTENT, JSON}},
-    {"CarParamsSP", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
-    {"CarParamsSPCache", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"CarParamsSPPersistent", {PERSISTENT, BYTES}},
-    {"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
-    {"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
-    {"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
-    {"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
-    {"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
-    {"DeviceBootMode", {PERSISTENT | BACKUP, INT, "0"}},
-    {"DevUIInfo", {PERSISTENT | BACKUP, INT, "0"}},
-    {"EnableCopyparty", {PERSISTENT | BACKUP, BOOL}},
-    {"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
-    {"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
-    {"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
-    {"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
-    {"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
-    {"IsDevelopmentBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsReleaseSpBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"LastGPSPositionLLK", {PERSISTENT, STRING}},
-    {"LeadDepartAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}},
-    {"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}},
-    {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
-    {"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
-    {"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ApiCache_DriveStats", PERSISTENT},
+    {"AutoLaneChangeBsmDelay", PERSISTENT},
+    {"AutoLaneChangeTimer", PERSISTENT},
+    {"BlinkerMinLateralControlSpeed", PERSISTENT | BACKUP},
+    {"BlinkerPauseLateralControl", PERSISTENT | BACKUP},
+    {"CarParamsSP", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsSPCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsSPPersistent", PERSISTENT},
+    {"CarPlatformBundle", PERSISTENT},
+    {"CustomAccIncrementsEnabled", PERSISTENT | BACKUP},
+    {"CustomAccLongPressIncrement", PERSISTENT | BACKUP},
+    {"CustomAccShortPressIncrement", PERSISTENT | BACKUP},
+    {"DeviceBootMode", PERSISTENT | BACKUP},
+    {"EnableGithubRunner", PERSISTENT | BACKUP},
+    {"MaxTimeOffroad", PERSISTENT | BACKUP},
+    {"Brightness", PERSISTENT | BACKUP},
+    {"ModelRunnerTypeCache", CLEAR_ON_ONROAD_TRANSITION},
+    {"OffroadMode", CLEAR_ON_MANAGER_START},
+    {"QuietMode", PERSISTENT | BACKUP},
 
     // MADS params
-    {"Mads", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"MadsMainCruiseAllowed", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"MadsSteeringMode", {PERSISTENT | BACKUP, INT, "0"}},
-    {"MadsUnifiedEngagementMode", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"Mads", PERSISTENT | BACKUP},
+    {"MadsMainCruiseAllowed", PERSISTENT | BACKUP},
+    {"MadsSteeringMode", PERSISTENT | BACKUP},
+    {"MadsUnifiedEngagementMode", PERSISTENT | BACKUP},
 
     // Model Manager params
-    {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
-    {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT, "0"}},
-    {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
-    {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
-    {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
+    {"ModelManager_ActiveBundle", PERSISTENT},
+    {"ModelManager_DownloadIndex", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ModelManager_LastSyncTime", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"ModelManager_ModelsCache", PERSISTENT | BACKUP},
 
     // Neural Network Lateral Control
-    {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"NeuralNetworkLateralControl", PERSISTENT | BACKUP},
 
     // sunnylink params
-    {"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
-    {"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"SunnylinkCache_Roles", {PERSISTENT, STRING}},
-    {"SunnylinkCache_Users", {PERSISTENT, STRING}},
-    {"SunnylinkDongleId", {PERSISTENT, STRING}},
-    {"SunnylinkdPid", {PERSISTENT, INT}},
-    {"SunnylinkEnabled", {PERSISTENT, BOOL, "1"}},
-    {"SunnylinkTempFault", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL, "0"}},
+    {"EnableSunnylinkUploader", PERSISTENT | BACKUP},
+    {"LastSunnylinkPingTime", CLEAR_ON_MANAGER_START},
+    {"SunnylinkCache_Roles", PERSISTENT},
+    {"SunnylinkCache_Users", PERSISTENT},
+    {"SunnylinkDongleId", PERSISTENT},
+    {"SunnylinkdPid", PERSISTENT},
+    {"SunnylinkEnabled", PERSISTENT},
 
     // Backup Manager params
-    {"BackupManager_CreateBackup", {PERSISTENT, BOOL}},
-    {"BackupManager_RestoreVersion", {PERSISTENT, STRING}},
+    {"BackupManager_CreateBackup", PERSISTENT},
+    {"BackupManager_RestoreVersion", PERSISTENT},
 
     // sunnypilot car specific params
-    {"HyundaiLongitudinalTuning", {PERSISTENT | BACKUP, INT, "0"}},
-    {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"HyundaiLongitudinalTuning", PERSISTENT},
 
-    {"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"DynamicExperimentalControl", PERSISTENT},
+    {"BlindSpot", PERSISTENT | BACKUP},
 
-    // sunnypilot model params
-    {"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
-    {"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
-    {"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
-    {"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
-    {"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
+    // model panel params
+    {"LagdToggle", PERSISTENT | BACKUP},
 
     // mapd
-    {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
-    {"MapdVersion", {PERSISTENT, STRING}},
-    {"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}},
-    {"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_OSMUpdateRequired", {CLEAR_ON_MANAGER_START, JSON}},
-    {"OsmDbUpdatesCheck", {CLEAR_ON_MANAGER_START, BOOL}},  // mapd database update happens with device ON, reset on boot
-    {"OSMDownloadBounds", {PERSISTENT, STRING}},
-    {"OsmDownloadedDate", {PERSISTENT, STRING, "0.0"}},
-    {"OSMDownloadLocations", {PERSISTENT, JSON}},
-    {"OSMDownloadProgress", {CLEAR_ON_MANAGER_START, JSON}},
-    {"OsmLocal", {PERSISTENT, BOOL}},
-    {"OsmLocationName", {PERSISTENT, STRING}},
-    {"OsmLocationTitle", {PERSISTENT, STRING}},
-    {"OsmLocationUrl", {PERSISTENT, STRING}},
-    {"OsmStateName", {PERSISTENT, STRING, "All"}},
-    {"OsmStateTitle", {PERSISTENT, STRING}},
-    {"OsmWayTest", {PERSISTENT, STRING}},
-    {"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
-
-    // Speed Limit
-    {"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
-    {"SpeedLimitOffsetType", {PERSISTENT | BACKUP, INT, "0"}},
-    {"SpeedLimitPolicy", {PERSISTENT | BACKUP, INT, "3"}},
-    {"SpeedLimitValueOffset", {PERSISTENT | BACKUP, INT, "0"}},
-
-    // Smart Cruise Control
-    {"MapTargetVelocities", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"SmartCruiseControlMap", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"SmartCruiseControlVision", {PERSISTENT | BACKUP, BOOL, "0"}},
-
-    // Torque lateral control custom params
-    {"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
-    {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
-    {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
-    {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
-    {"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
-    {"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
+    {"MapAdvisorySpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"MapdVersion", PERSISTENT},
+    {"MapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"NextMapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_OSMUpdateRequired", CLEAR_ON_MANAGER_START},
+    {"OsmDbUpdatesCheck", CLEAR_ON_MANAGER_START},  // mapd database update happens with device ON, reset on boot
+    {"OSMDownloadBounds", PERSISTENT},
+    {"OsmDownloadedDate", PERSISTENT},
+    {"OSMDownloadLocations", PERSISTENT},
+    {"OSMDownloadProgress", CLEAR_ON_MANAGER_START},
+    {"OsmLocal", PERSISTENT},
+    {"OsmLocationName", PERSISTENT},
+    {"OsmLocationTitle", PERSISTENT},
+    {"OsmLocationUrl", PERSISTENT},
+    {"OsmStateName", PERSISTENT},
+    {"OsmStateTitle", PERSISTENT},
+    {"OsmWayTest", PERSISTENT},
+    {"RoadName", CLEAR_ON_ONROAD_TRANSITION},
 };

common/params_pyx.pyx

@@ -1,35 +1,19 @@
 # distutils: language = c++
 # cython: language_level = 3
-import builtins
-import datetime
-import json
 from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
-from libcpp.optional cimport optional
-
-from openpilot.common.swaglog import cloudlog
 
 cdef extern from "common/params.h":
-  cpdef enum ParamKeyFlag:
+  cpdef enum ParamKeyType:
     PERSISTENT
     CLEAR_ON_MANAGER_START
     CLEAR_ON_ONROAD_TRANSITION
     CLEAR_ON_OFFROAD_TRANSITION
     DEVELOPMENT_ONLY
-    CLEAR_ON_IGNITION_ON
     BACKUP
     ALL
 
-  cpdef enum ParamKeyType:
-    STRING
-    BOOL
-    INT
-    FLOAT
-    TIME
-    JSON
-    BYTES
-
   cdef cppclass c_Params "Params":
     c_Params(string) except + nogil
     string get(string, bool) nogil
@@ -40,31 +24,10 @@ cdef extern from "common/params.h":
     void putBoolNonBlocking(string, bool) nogil
     int putBool(string, bool) nogil
     bool checkKey(string) nogil
-    ParamKeyType getKeyType(string) nogil
-    optional[string] getKeyDefaultValue(string) nogil
     string getParamPath(string) nogil
-    void clearAll(ParamKeyFlag)
-    vector[string] allKeys(ParamKeyFlag)
+    void clearAll(ParamKeyType)
+    vector[string] allKeys(ParamKeyType)
 
-PYTHON_2_CPP = {
-  (str, STRING): lambda v: v,
-  (builtins.bool, BOOL): lambda v: "1" if v else "0",
-  (int, INT): str,
-  (float, FLOAT): str,
-  (datetime.datetime, TIME): lambda v: v.isoformat(),
-  (dict, JSON): json.dumps,
-  (list, JSON): json.dumps,
-  (bytes, BYTES): lambda v: v,
-}
-CPP_2_PYTHON = {
-  STRING: lambda v: v.decode("utf-8"),
-  BOOL: lambda v: v == b"1",
-  INT: int,
-  FLOAT: float,
-  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
-  JSON: json.loads,
-  BYTES: lambda v: v,
-}
 
 def ensure_bytes(v):
   return v.encode() if isinstance(v, str) else v
@@ -88,8 +51,8 @@ cdef class Params:
   def __dealloc__(self):
     del self.p
 
-  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
-    self.p.clearAll(tx_flag)
+  def clear_all(self, tx_type=ParamKeyType.ALL):
+    self.p.clearAll(tx_type)
 
   def check_key(self, key):
     key = ensure_bytes(key)
@@ -97,38 +60,21 @@ cdef class Params:
       raise UnknownKeyName(key)
     return key
 
-  def python2cpp(self, proposed_type, expected_type, value, key):
-    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
-    if cast:
-      return cast(value)
-    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
-
-  def _cpp2python(self, t, value, default, key):
-    if value is None:
-      return None
-    try:
-      return CPP_2_PYTHON[t](value)
-    except (KeyError, TypeError, ValueError):
-      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
-      return self._cpp2python(t, default, None, key)
-
-  def get(self, key, bool block=False, bool return_default=False):
+  def get(self, key, bool block=False, encoding=None):
     cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    cdef optional[string] default = self.p.getKeyDefaultValue(k)
     cdef string val
     with nogil:
       val = self.p.get(k, block)
 
-    default_val = (default.value() if default.has_value() else None) if return_default else None
     if val == b"":
       if block:
         # If we got no value while running in blocked mode
         # it means we got an interrupt while waiting
         raise KeyboardInterrupt
       else:
-        return self._cpp2python(t, default_val, None, key)
-    return self._cpp2python(t, val, default_val, key)
+        return None
+
+    return val if encoding is None else val.decode(encoding)
 
   def get_bool(self, key, bool block=False):
     cdef string k = self.check_key(key)
@@ -137,11 +83,6 @@ cdef class Params:
       r = self.p.getBool(k, block)
     return r
 
-  def _put_cast(self, key, dat):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
-
   def put(self, key, dat):
     """
     Warning: This function blocks until the param is written to disk!
@@ -150,7 +91,7 @@ cdef class Params:
     in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
+    cdef string dat_bytes = ensure_bytes(dat)
     with nogil:
       self.p.put(k, dat_bytes)
 
@@ -161,7 +102,7 @@ cdef class Params:
 
   def put_nonblocking(self, key, dat):
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
+    cdef string dat_bytes = ensure_bytes(dat)
     with nogil:
       self.p.putNonBlocking(k, dat_bytes)
 
@@ -179,19 +120,5 @@ cdef class Params:
     cdef string key_bytes = ensure_bytes(key)
     return self.p.getParamPath(key_bytes).decode("utf-8")
 
-  def get_type(self, key):
-    return self.p.getKeyType(self.check_key(key))
-
-  def all_keys(self, flag=ParamKeyFlag.ALL):
-    return self.p.allKeys(flag)
-
-  def get_default_value(self, key):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    cdef optional[string] default = self.p.getKeyDefaultValue(k)
-    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
-
-  def cpp2python(self, key, value):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    return self._cpp2python(t, value, None, key)
+  def all_keys(self, type=ParamKeyType.ALL):
+    return self.p.allKeys(type)

common/pid.py

@@ -2,14 +2,23 @@ import numpy as np
 from numbers import Number
 
 class PIDController:
-  def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
-    self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
-    self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
-    self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
+  def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p = k_p
+    self._k_i = k_i
+    self._k_d = k_d
+    self.k_f = k_f   # feedforward gain
+    if isinstance(self._k_p, Number):
+      self._k_p = [[0], [self._k_p]]
+    if isinstance(self._k_i, Number):
+      self._k_i = [[0], [self._k_i]]
+    if isinstance(self._k_d, Number):
+      self._k_d = [[0], [self._k_d]]
 
-    self.set_limits(pos_limit, neg_limit)
+    self.pos_limit = pos_limit
+    self.neg_limit = neg_limit
 
-    self.i_dt = 1.0 / rate
+    self.i_unwind_rate = 0.3 / rate
+    self.i_rate = 1.0 / rate
     self.speed = 0.0
 
     self.reset()
@@ -26,6 +35,10 @@ class PIDController:
   def k_d(self):
     return np.interp(self.speed, self._k_d[0], self._k_d[1])
 
+  @property
+  def error_integral(self):
+    return self.i/self.k_i
+
   def reset(self):
     self.p = 0.0
     self.i = 0.0
@@ -33,25 +46,25 @@ class PIDController:
     self.f = 0.0
     self.control = 0
 
-  def set_limits(self, pos_limit, neg_limit):
-    self.pos_limit = pos_limit
-    self.neg_limit = neg_limit
-
-  def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
+  def update(self, error, error_rate=0.0, speed=0.0, override=False, feedforward=0., freeze_integrator=False):
     self.speed = speed
-    self.p = self.k_p * float(error)
-    self.d = self.k_d * error_rate
-    self.f = feedforward
 
-    if not freeze_integrator:
-      i = self.i + self.k_i * self.i_dt * error
+    self.p = float(error) * self.k_p
+    self.f = feedforward * self.k_f
+    self.d = error_rate * self.k_d
 
-      # Don't allow windup if already clipping
-      test_control = self.p + i + self.d + self.f
-      i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
-      i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
-      self.i = np.clip(i, i_lowerbound, i_upperbound)
+    if override:
+      self.i -= self.i_unwind_rate * float(np.sign(self.i))
+    else:
+      if not freeze_integrator:
+        self.i = self.i + error * self.k_i * self.i_rate
+
+        # Clip i to prevent exceeding control limits
+        control_no_i = self.p + self.d + self.f
+        control_no_i = np.clip(control_no_i, self.neg_limit, self.pos_limit)
+        self.i = np.clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
 
     control = self.p + self.i + self.d + self.f
+
     self.control = np.clip(control, self.neg_limit, self.pos_limit)
     return self.control

common/prefix.h

@@ -13,11 +13,7 @@ public:
     if (prefix.empty()) {
       prefix = util::random_string(15);
     }
-#ifdef __APPLE__
-    msgq_path = "/tmp/msgq_" + prefix;
-#else
-    msgq_path = "/dev/shm/msgq_" + prefix;
-#endif
+    msgq_path = Path::shm_path() + "/" + prefix;
     bool ret = util::create_directories(msgq_path, 0777);
     assert(ret);
     setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);

common/prefix.py

@@ -1,5 +1,4 @@
 import os
-import platform
 import shutil
 import uuid
 
@@ -10,20 +9,20 @@ from openpilot.system.hardware.hw import Paths
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
-  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
     self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
-    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
-    self.create_dirs_on_enter = create_dirs_on_enter
+    self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
     self.clean_dirs_on_exit = clean_dirs_on_exit
     self.shared_download_cache = shared_download_cache
 
   def __enter__(self):
     self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
     os.environ['OPENPILOT_PREFIX'] = self.prefix
-
-    if self.create_dirs_on_enter:
-      self.create_dirs()
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
 
     if self.shared_download_cache:
       os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
@@ -41,13 +40,6 @@ class OpenpilotPrefix:
       pass
     return False
 
-  def create_dirs(self):
-    try:
-      os.mkdir(self.msgq_path)
-    except FileExistsError:
-      pass
-    os.makedirs(Paths.log_root(), exist_ok=True)
-
   def clean_dirs(self):
     symlink_path = Params().get_param_path()
     if os.path.exists(symlink_path):

common/realtime.py

@@ -6,7 +6,7 @@ import time
 
 from setproctitle import getproctitle
 
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
 from openpilot.system.hardware import PC
 
 

common/retry.py

@@ -0,0 +1,30 @@
+import time
+import functools
+
+from openpilot.common.swaglog import cloudlog
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          cloudlog.exception(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        cloudlog.error(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator
+
+
+if __name__ == "__main__":
+  @retry(attempts=10)
+  def abc():
+    raise ValueError("abc failed :(")
+  abc()

common/run.py

@@ -0,0 +1,13 @@
+import subprocess
+
+
+def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
+  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
+
+
+def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
+  try:
+    return run_cmd(cmd, cwd=cwd, env=env)
+  except subprocess.CalledProcessError:
+    return default
+

common/swaglog.cc

@@ -15,8 +15,6 @@
 #include "common/version.h"
 #include "system/hardware/hw.h"
 
-#include "sunnypilot/common/version.h"
-
 class SwaglogState {
 public:
   SwaglogState() {
@@ -58,7 +56,7 @@ public:
     if (char* daemon_name = getenv("MANAGER_DAEMON")) {
       ctx_j["daemon"] = daemon_name;
     }
-    ctx_j["version"] = SUNNYPILOT_VERSION;
+    ctx_j["version"] = COMMA_VERSION;
     ctx_j["dirty"] = !getenv("CLEAN");
     ctx_j["device"] = Hardware::get_name();
   }

common/tests/test_file_helpers.py

@@ -1,7 +1,7 @@
 import os
 from uuid import uuid4
 
-from openpilot.common.utils import atomic_write
+from openpilot.common.file_helpers import atomic_write_in_dir
 
 
 class TestFileHelpers:
@@ -15,5 +15,5 @@ class TestFileHelpers:
       assert f.read() == "test"
     os.remove(path)
 
-  def test_atomic_write(self):
-    self.run_atomic_write_func(atomic_write)
+  def test_atomic_write_in_dir(self):
+    self.run_atomic_write_func(atomic_write_in_dir)

common/tests/test_markdown.py

@@ -6,7 +6,7 @@ from openpilot.common.markdown import parse_markdown
 
 class TestMarkdown:
   def test_all_release_notes(self):
-    with open(os.path.join(BASEDIR, "CHANGELOG.md")) as f:
+    with open(os.path.join(BASEDIR, "RELEASES.md")) as f:
       release_notes = f.read().split("\n\n")
       assert len(release_notes) > 10
 

common/tests/test_params.py

@@ -1,11 +1,10 @@
 import pytest
-import datetime
 import os
 import threading
 import time
 import uuid
 
-from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
+from openpilot.common.params import Params, ParamKeyType, UnknownKeyName
 
 class TestParams:
   def setup_method(self):
@@ -13,7 +12,7 @@ class TestParams:
 
   def test_params_put_and_get(self):
     self.params.put("DongleId", "cb38263377b873ee")
-    assert self.params.get("DongleId") == "cb38263377b873ee"
+    assert self.params.get("DongleId") == b"cb38263377b873ee"
 
   def test_params_non_ascii(self):
     st = b"\xe1\x90\xff"
@@ -21,7 +20,7 @@ class TestParams:
     assert self.params.get("CarParams") == st
 
   def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", b"test")
+    self.params.put("CarParams", "test")
     self.params.put("DongleId", "cb38263377b873ee")
     assert self.params.get("CarParams") == b"test"
 
@@ -30,24 +29,24 @@ class TestParams:
       f.write("test")
     assert os.path.isfile(undefined_param)
 
-    self.params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
+    self.params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
     assert self.params.get("CarParams") is None
     assert self.params.get("DongleId") is not None
     assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
     self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", 123)
-    assert self.params.get("DongleId") == "bob"
-    assert self.params.get("AthenadPid") == 123
+    self.params.put("AthenadPid", "123")
+    assert self.params.get("DongleId") == b"bob"
+    assert self.params.get("AthenadPid") == b"123"
 
   def test_params_get_block(self):
     def _delayed_writer():
       time.sleep(0.1)
-      self.params.put("CarParams", b"test")
+      self.params.put("CarParams", "test")
     threading.Thread(target=_delayed_writer).start()
     assert self.params.get("CarParams") is None
-    assert self.params.get("CarParams", block=True) == b"test"
+    assert self.params.get("CarParams", True) == b"test"
 
   def test_params_unknown_key_fails(self):
     with pytest.raises(UnknownKeyName):
@@ -77,17 +76,17 @@ class TestParams:
     self.params.put_bool("IsMetric", False)
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", True)
+    self.params.put("IsMetric", "1")
     assert self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", False)
+    self.params.put("IsMetric", "0")
     assert not self.params.get_bool("IsMetric")
 
   def test_put_non_blocking_with_get_block(self):
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_nonblocking("CarParams", b"test")
+      Params().put_nonblocking("CarParams", "test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
@@ -108,34 +107,3 @@ class TestParams:
     assert len(keys) > 20
     assert len(keys) == len(set(keys))
     assert b"CarParams" in keys
-
-  def test_params_default_value(self):
-    self.params.remove("LanguageSetting")
-    self.params.remove("LongitudinalPersonality")
-    self.params.remove("LiveParameters")
-
-    assert self.params.get("LanguageSetting") is None
-    assert self.params.get("LanguageSetting", return_default=False) is None
-    assert isinstance(self.params.get("LanguageSetting", return_default=True), str)
-    assert isinstance(self.params.get("LongitudinalPersonality", return_default=True), int)
-    assert self.params.get("LiveParameters") is None
-    assert self.params.get("LiveParameters", return_default=True) is None
-
-  def test_params_get_type(self):
-    # json
-    self.params.put("ApiCache_FirehoseStats", {"a": 0})
-    assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
-
-    # int
-    self.params.put("BootCount", 1441)
-    assert self.params.get("BootCount") == 1441
-
-    # bool
-    self.params.put("AdbEnabled", True)
-    assert self.params.get("AdbEnabled")
-    assert isinstance(self.params.get("AdbEnabled"), bool)
-
-    # time
-    now = datetime.datetime.now(datetime.UTC)
-    self.params.put("InstallDate", now)
-    assert self.params.get("InstallDate") == now

common/tests/test_swaglog.cc

@@ -9,8 +9,6 @@
 #include "system/hardware/hw.h"
 #include "third_party/json11/json11.hpp"
 
-#include "sunnypilot/common/version.h"
-
 std::string daemon_name = "testy";
 std::string dongle_id = "test_dongle_id";
 int LINE_NO = 0;
@@ -55,7 +53,7 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {
     REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
     REQUIRE(ctx["dirty"].bool_value() == true);
 
-    REQUIRE(ctx["version"].string_value() == SUNNYPILOT_VERSION);
+    REQUIRE(ctx["version"].string_value() == COMMA_VERSION);
 
     std::string device = Hardware::get_name();
     REQUIRE(ctx["device"].string_value() == device);

common/transformations/SConscript

@@ -0,0 +1,5 @@
+Import('env', 'envCython')
+
+transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc'])
+transformations_python = envCython.Program('transformations.so', 'transformations.pyx')
+Export('transformations', 'transformations_python')

common/transformations/coordinates.cc

@@ -1,6 +1,6 @@
 #define _USE_MATH_DEFINES
 
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 #include <iostream>
 #include <cmath>

common/transformations/orientation.cc

@@ -4,8 +4,8 @@
 #include <cmath>
 #include <eigen3/Eigen/Dense>
 
-#include "sunnypilot/common/transformations/orientation.hpp"
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/orientation.hpp"
+#include "common/transformations/coordinates.hpp"
 
 Eigen::Quaterniond ensure_unique(const Eigen::Quaterniond &quat) {
   if (quat.w() > 0){
@@ -141,3 +141,4 @@ Eigen::Vector3d ned_euler_from_ecef(const ECEF &ecef_init, const Eigen::Vector3d
 
   return {phi, theta, psi};
 }
+

common/transformations/orientation.hpp

@@ -1,6 +1,6 @@
 #pragma once
 #include <eigen3/Eigen/Dense>
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 
 Eigen::Quaterniond ensure_unique(const Eigen::Quaterniond &quat);

common/transformations/tests/test_coordinates.py

@@ -102,36 +102,3 @@ class TestNED:
     np.testing.assert_allclose(converter.ned2ecef(ned_offsets_batch),
                                                            ecef_positions_offset_batch,
                                                            rtol=1e-9, atol=1e-7)
-
-  def test_errors(self):
-    # Test wrong shape/type for geodetic2ecef
-    # numpy_wrap raises IndexError for scalar input
-    with np.testing.assert_raises(IndexError):
-      coord.geodetic2ecef(1.0)
-
-    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
-      coord.geodetic2ecef([0, 0])
-
-    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
-      coord.geodetic2ecef([0, 0, 0, 0])
-
-    with np.testing.assert_raises(TypeError):
-      coord.geodetic2ecef(['a', 'b', 'c'])
-
-    # Test LocalCoord constructor errors
-    with np.testing.assert_raises(ValueError):
-      coord.LocalCoord.from_geodetic([0, 0])
-
-    with np.testing.assert_raises(ValueError):
-      coord.LocalCoord.from_geodetic(1)
-
-    with np.testing.assert_raises(TypeError):
-      coord.LocalCoord.from_geodetic(['a', 'b', 'c'])
-
-    # Test wrong shape/type for ecef2geodetic
-    with np.testing.assert_raises(ValueError):
-      coord.ecef2geodetic([1, 2])
-    with np.testing.assert_raises(ValueError):
-      coord.ecef2geodetic([1, 2, 3, 4])
-    with np.testing.assert_raises(IndexError):
-      coord.ecef2geodetic(1.0)

common/transformations/tests/test_orientation.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pytest
 
 from openpilot.common.transformations.orientation import euler2quat, quat2euler, euler2rot, rot2euler, \
                                                rot2quat, quat2rot, \
@@ -60,32 +59,3 @@ class TestOrientation:
       np.testing.assert_allclose(ned_eulers[i], ned_euler_from_ecef(ecef_positions[i], eulers[i]), rtol=1e-7)
       #np.testing.assert_allclose(eulers[i], ecef_euler_from_ned(ecef_positions[i], ned_eulers[i]), rtol=1e-7)
     # np.testing.assert_allclose(ned_eulers, ned_euler_from_ecef(ecef_positions, eulers), rtol=1e-7)
-
-  def test_inputs(self):
-    with pytest.raises(ValueError):
-      euler2quat([1, 2])
-
-    with pytest.raises(ValueError):
-      quat2rot([1, 2, 3])
-
-    with pytest.raises(IndexError):
-      rot2quat(np.zeros((2, 2)))
-
-  def test_euler_rot_consistency(self):
-    rpy = [0.1, 0.2, 0.3]
-    R = euler2rot(rpy)
-
-    # R -> q -> R
-    q = rot2quat(R)
-    R_new = quat2rot(q)
-    np.testing.assert_allclose(R, R_new, atol=1e-15)
-
-    # q -> R -> Euler (quat2euler) -> R
-    rpy_new = quat2euler(q)
-    R_new2 = euler2rot(rpy_new)
-    np.testing.assert_allclose(R, R_new2, atol=1e-15)
-
-    # R -> Euler (rot2euler) -> R
-    rpy_from_rot = rot2euler(R)
-    R_new3 = euler2rot(rpy_from_rot)
-    np.testing.assert_allclose(R, R_new3, atol=1e-15)

common/transformations/transformations.pxd

@@ -0,0 +1,72 @@
+# cython: language_level=3
+from libcpp cimport bool
+
+cdef extern from "orientation.cc":
+  pass
+
+cdef extern from "orientation.hpp":
+  cdef cppclass Quaternion "Eigen::Quaterniond":
+    Quaternion()
+    Quaternion(double, double, double, double)
+    double w()
+    double x()
+    double y()
+    double z()
+
+  cdef cppclass Vector3 "Eigen::Vector3d":
+    Vector3()
+    Vector3(double, double, double)
+    double operator()(int)
+
+  cdef cppclass Matrix3 "Eigen::Matrix3d":
+    Matrix3()
+    Matrix3(double*)
+
+    double operator()(int, int)
+
+  Quaternion euler2quat(const Vector3 &)
+  Vector3 quat2euler(const Quaternion &)
+  Matrix3 quat2rot(const Quaternion &)
+  Quaternion rot2quat(const Matrix3 &)
+  Vector3 rot2euler(const Matrix3 &)
+  Matrix3 euler2rot(const Vector3 &)
+  Matrix3 rot_matrix(double, double, double)
+  Vector3 ecef_euler_from_ned(const ECEF &, const Vector3 &)
+  Vector3 ned_euler_from_ecef(const ECEF &, const Vector3 &)
+
+
+cdef extern from "coordinates.cc":
+  cdef struct ECEF:
+    double x
+    double y
+    double z
+
+  cdef struct NED:
+    double n
+    double e
+    double d
+
+  cdef struct Geodetic:
+    double lat
+    double lon
+    double alt
+    bool radians
+
+  ECEF geodetic2ecef(const Geodetic &)
+  Geodetic ecef2geodetic(const ECEF &)
+
+  cdef cppclass LocalCoord_c "LocalCoord":
+    Matrix3 ned2ecef_matrix
+    Matrix3 ecef2ned_matrix
+
+    LocalCoord_c(const Geodetic &, const ECEF &)
+    LocalCoord_c(const Geodetic &)
+    LocalCoord_c(const ECEF &)
+
+    NED ecef2ned(const ECEF &)
+    ECEF ned2ecef(const NED &)
+    NED geodetic2ned(const Geodetic &)
+    Geodetic ned2geodetic(const NED &)
+
+cdef extern from "coordinates.hpp":
+  pass

common/transformations/transformations.py

@@ -1,342 +0,0 @@
-import numpy as np
-
-
-# Constants
-a = 6378137.0
-b = 6356752.3142
-esq = 6.69437999014e-3
-e1sq = 6.73949674228e-3
-
-
-def geodetic2ecef_single(g):
-  """
-  Convert geodetic coordinates (latitude, longitude, altitude) to ECEF.
-  """
-  try:
-    if len(g) != 3:
-      raise ValueError("Geodetic must be size 3")
-  except TypeError:
-    raise ValueError("Geodetic must be a sequence of length 3") from None
-
-  lat, lon, alt = g
-  lat = np.radians(lat)
-  lon = np.radians(lon)
-  xi = np.sqrt(1.0 - esq * np.sin(lat)**2)
-  x = (a / xi + alt) * np.cos(lat) * np.cos(lon)
-  y = (a / xi + alt) * np.cos(lat) * np.sin(lon)
-  z = (a / xi * (1.0 - esq) + alt) * np.sin(lat)
-  return np.array([x, y, z])
-
-
-def ecef2geodetic_single(e):
-  """
-  Convert ECEF to geodetic coordinates using Ferrari's solution.
-  """
-  x, y, z = e
-  r = np.sqrt(x**2 + y**2)
-  Esq = a**2 - b**2
-  F = 54 * b**2 * z**2
-  G = r**2 + (1 - esq) * z**2 - esq * Esq
-  C = (esq**2 * F * r**2) / (G**3)
-  S = np.cbrt(1 + C + np.sqrt(C**2 + 2 * C))
-  P = F / (3 * (S + 1 / S + 1)**2 * G**2)
-  Q = np.sqrt(1 + 2 * esq**2 * P)
-  r_0 = -(P * esq * r) / (1 + Q) + np.sqrt(0.5 * a**2 * (1 + 1.0 / Q) - P * (1 - esq) * z**2 / (Q * (1 + Q)) - 0.5 * P * r**2)
-  U = np.sqrt((r - esq * r_0)**2 + z**2)
-  V = np.sqrt((r - esq * r_0)**2 + (1 - esq) * z**2)
-  Z_0 = b**2 * z / (a * V)
-  h = U * (1 - b**2 / (a * V))
-  lat = np.arctan((z + e1sq * Z_0) / r)
-  lon = np.arctan2(y, x)
-  return np.array([np.degrees(lat), np.degrees(lon), h])
-
-
-def euler2quat_single(euler):
-  """
-  Convert Euler angles (roll, pitch, yaw) to a quaternion.
-  Rotation order: Z-Y-X (yaw, pitch, roll).
-  """
-  phi, theta, psi = euler
-
-  c_phi, s_phi = np.cos(phi / 2), np.sin(phi / 2)
-  c_theta, s_theta = np.cos(theta / 2), np.sin(theta / 2)
-  c_psi, s_psi = np.cos(psi / 2), np.sin(psi / 2)
-
-  w = c_phi * c_theta * c_psi + s_phi * s_theta * s_psi
-  x = s_phi * c_theta * c_psi - c_phi * s_theta * s_psi
-  y = c_phi * s_theta * c_psi + s_phi * c_theta * s_psi
-  z = c_phi * c_theta * s_psi - s_phi * s_theta * c_psi
-
-  if w < 0:
-    return np.array([-w, -x, -y, -z])
-  return np.array([w, x, y, z])
-
-
-def quat2euler_single(q):
-  """
-  Convert a quaternion to Euler angles (roll, pitch, yaw).
-  """
-  w, x, y, z = q
-  gamma = np.arctan2(2 * (w * x + y * z), 1 - 2 * (x**2 + y**2))
-  sin_arg = 2 * (w * y - z * x)
-  sin_arg = np.clip(sin_arg, -1.0, 1.0)
-  theta = np.arcsin(sin_arg)
-  psi = np.arctan2(2 * (w * z + x * y), 1 - 2 * (y**2 + z**2))
-  return np.array([gamma, theta, psi])
-
-
-def quat2rot_single(q):
-  """
-  Convert a quaternion to a 3x3 rotation matrix.
-  """
-  w, x, y, z = q
-  xx, yy, zz = x * x, y * y, z * z
-  xy, xz, yz = x * y, x * z, y * z
-  wx, wy, wz = w * x, w * y, w * z
-
-  mat = np.array([
-    [1 - 2 * (yy + zz), 2 * (xy - wz), 2 * (xz + wy)],
-    [2 * (xy + wz), 1 - 2 * (xx + zz), 2 * (yz - wx)],
-    [2 * (xz - wy), 2 * (yz + wx), 1 - 2 * (xx + yy)]
-  ])
-  return mat
-
-
-def rot2quat_single(rot):
-  """
-  Convert a 3x3 rotation matrix to a quaternion.
-  """
-  trace = np.trace(rot)
-  if trace > 0:
-    s = 0.5 / np.sqrt(trace + 1.0)
-    w = 0.25 / s
-    x = (rot[2, 1] - rot[1, 2]) * s
-    y = (rot[0, 2] - rot[2, 0]) * s
-    z = (rot[1, 0] - rot[0, 1]) * s
-  else:
-    if rot[0, 0] > rot[1, 1] and rot[0, 0] > rot[2, 2]:
-      s = 2.0 * np.sqrt(1.0 + rot[0, 0] - rot[1, 1] - rot[2, 2])
-      w = (rot[2, 1] - rot[1, 2]) / s
-      x = 0.25 * s
-      y = (rot[0, 1] + rot[1, 0]) / s
-      z = (rot[0, 2] + rot[2, 0]) / s
-    elif rot[1, 1] > rot[2, 2]:
-      s = 2.0 * np.sqrt(1.0 + rot[1, 1] - rot[0, 0] - rot[2, 2])
-      w = (rot[0, 2] - rot[2, 0]) / s
-      x = (rot[0, 1] + rot[1, 0]) / s
-      y = 0.25 * s
-      z = (rot[1, 2] + rot[2, 1]) / s
-    else:
-      s = 2.0 * np.sqrt(1.0 + rot[2, 2] - rot[0, 0] - rot[1, 1])
-      w = (rot[1, 0] - rot[0, 1]) / s
-      x = (rot[0, 2] + rot[2, 0]) / s
-      y = (rot[1, 2] + rot[2, 1]) / s
-      z = 0.25 * s
-
-  if w < 0:
-    return np.array([-w, -x, -y, -z])
-  return np.array([w, x, y, z])
-
-
-def euler2rot_single(euler):
-  """
-  Convert Euler angles (roll, pitch, yaw) to a 3x3 rotation matrix.
-  Rotation order: Z-Y-X (yaw, pitch, roll).
-  """
-  phi, theta, psi = euler
-
-  cx, sx = np.cos(phi), np.sin(phi)
-  cy, sy = np.cos(theta), np.sin(theta)
-  cz, sz = np.cos(psi), np.sin(psi)
-
-  Rx = np.array([[1, 0, 0], [0, cx, -sx], [0, sx, cx]])
-  Ry = np.array([[cy, 0, sy], [0, 1, 0], [-sy, 0, cy]])
-  Rz = np.array([[cz, -sz, 0], [sz, cz, 0], [0, 0, 1]])
-
-  return Rz @ Ry @ Rx
-
-
-def rot2euler_single(rot):
-  """
-  Convert a 3x3 rotation matrix to Euler angles (roll, pitch, yaw).
-  """
-  return quat2euler_single(rot2quat_single(rot))
-
-
-def rot_matrix(roll, pitch, yaw):
-  """
-  Create a 3x3 rotation matrix from roll, pitch, and yaw angles.
-  """
-  return euler2rot_single([roll, pitch, yaw])
-
-
-def axis_angle_to_rot(axis, angle):
-  """
-  Convert an axis-angle representation to a 3x3 rotation matrix.
-  """
-  c = np.cos(angle / 2)
-  s = np.sin(angle / 2)
-  q = np.array([c, s*axis[0], s*axis[1], s*axis[2]])
-  return quat2rot_single(q)
-
-
-class LocalCoord:
-  """
-  A class to handle conversions between ECEF and local NED coordinates.
-  """
-  def __init__(self, geodetic=None, ecef=None):
-    """
-    Initialize LocalCoord with either geodetic or ECEF coordinates.
-    """
-    if geodetic is not None:
-      self.init_ecef = geodetic2ecef_single(geodetic)
-      lat, lon, _ = geodetic
-    elif ecef is not None:
-      self.init_ecef = np.array(ecef)
-      lat, lon, _ = ecef2geodetic_single(ecef)
-    else:
-      raise ValueError("Must provide geodetic or ecef")
-
-    lat = np.radians(lat)
-    lon = np.radians(lon)
-
-    self.ned2ecef_matrix = np.array([
-      [-np.sin(lat) * np.cos(lon), -np.sin(lon), -np.cos(lat) * np.cos(lon)],
-      [-np.sin(lat) * np.sin(lon), np.cos(lon), -np.cos(lat) * np.sin(lon)],
-      [np.cos(lat), 0, -np.sin(lat)]
-    ])
-    self.ecef2ned_matrix = self.ned2ecef_matrix.T
-
-  @classmethod
-  def from_geodetic(cls, geodetic):
-    """
-    Create a LocalCoord instance from geodetic coordinates.
-    """
-    return cls(geodetic=geodetic)
-
-  @classmethod
-  def from_ecef(cls, ecef):
-    """
-    Create a LocalCoord instance from ECEF coordinates.
-    """
-    return cls(ecef=ecef)
-
-  def ecef2ned_single(self, ecef):
-    """
-    Convert a single ECEF point to NED coordinates relative to the origin.
-    """
-    return self.ecef2ned_matrix @ (ecef - self.init_ecef)
-
-  def ned2ecef_single(self, ned):
-    """
-    Convert a single NED point to ECEF coordinates.
-    """
-    return self.ned2ecef_matrix @ ned + self.init_ecef
-
-  def geodetic2ned_single(self, geodetic):
-    """
-    Convert a single geodetic point to NED coordinates.
-    """
-    ecef = geodetic2ecef_single(geodetic)
-    return self.ecef2ned_single(ecef)
-
-  def ned2geodetic_single(self, ned):
-    """
-    Convert a single NED point to geodetic coordinates.
-    """
-    ecef = self.ned2ecef_single(ned)
-    return ecef2geodetic_single(ecef)
-
-  @property
-  def ned_from_ecef_matrix(self):
-    """
-    Returns the rotation matrix from ECEF to NED coordinates.
-    """
-    return self.ecef2ned_matrix
-
-  @property
-  def ecef_from_ned_matrix(self):
-    """
-    Returns the rotation matrix from NED to ECEF coordinates.
-    """
-    return self.ned2ecef_matrix
-
-
-def ecef_euler_from_ned_single(ecef_init, ned_pose):
-  """
-  Convert NED Euler angles (roll, pitch, yaw) at a given ECEF origin
-  to equivalent ECEF Euler angles.
-  """
-  converter = LocalCoord(ecef=ecef_init)
-  zero = np.array(ecef_init)
-
-  x0 = converter.ned2ecef_single([1, 0, 0]) - zero
-  y0 = converter.ned2ecef_single([0, 1, 0]) - zero
-  z0 = converter.ned2ecef_single([0, 0, 1]) - zero
-
-  phi, theta, psi = ned_pose
-
-  x1 = axis_angle_to_rot(z0, psi) @ x0
-  y1 = axis_angle_to_rot(z0, psi) @ y0
-  z1 = axis_angle_to_rot(z0, psi) @ z0
-
-  x2 = axis_angle_to_rot(y1, theta) @ x1
-  y2 = axis_angle_to_rot(y1, theta) @ y1
-  z2 = axis_angle_to_rot(y1, theta) @ z1
-
-  x3 = axis_angle_to_rot(x2, phi) @ x2
-  y3 = axis_angle_to_rot(x2, phi) @ y2
-
-  x0 = np.array([1.0, 0, 0])
-  y0 = np.array([0, 1.0, 0])
-  z0 = np.array([0, 0, 1.0])
-
-  psi_out = np.arctan2(np.dot(x3, y0), np.dot(x3, x0))
-  theta_out = np.arctan2(-np.dot(x3, z0), np.sqrt(np.dot(x3, x0)**2 + np.dot(x3, y0)**2))
-
-  y2 = axis_angle_to_rot(z0, psi_out) @ y0
-  z2 = axis_angle_to_rot(y2, theta_out) @ z0
-
-  phi_out = np.arctan2(np.dot(y3, z2), np.dot(y3, y2))
-
-  return np.array([phi_out, theta_out, psi_out])
-
-
-def ned_euler_from_ecef_single(ecef_init, ecef_pose):
-  """
-  Convert ECEF Euler angles (roll, pitch, yaw) at a given ECEF origin
-  to equivalent NED Euler angles.
-  """
-  converter = LocalCoord(ecef=ecef_init)
-
-  x0 = np.array([1.0, 0, 0])
-  y0 = np.array([0, 1.0, 0])
-  z0 = np.array([0, 0, 1.0])
-
-  phi, theta, psi = ecef_pose
-
-  x1 = axis_angle_to_rot(z0, psi) @ x0
-  y1 = axis_angle_to_rot(z0, psi) @ y0
-  z1 = axis_angle_to_rot(z0, psi) @ z0
-
-  x2 = axis_angle_to_rot(y1, theta) @ x1
-  y2 = axis_angle_to_rot(y1, theta) @ y1
-  z2 = axis_angle_to_rot(y1, theta) @ z1
-
-  x3 = axis_angle_to_rot(x2, phi) @ x2
-  y3 = axis_angle_to_rot(x2, phi) @ y2
-
-  zero = np.array(ecef_init)
-  x0 = converter.ned2ecef_single([1, 0, 0]) - zero
-  y0 = converter.ned2ecef_single([0, 1, 0]) - zero
-  z0 = converter.ned2ecef_single([0, 0, 1]) - zero
-
-  psi_out = np.arctan2(np.dot(x3, y0), np.dot(x3, x0))
-  theta_out = np.arctan2(-np.dot(x3, z0), np.sqrt(np.dot(x3, x0)**2 + np.dot(x3, y0)**2))
-
-  y2 = axis_angle_to_rot(z0, psi_out) @ y0
-  z2 = axis_angle_to_rot(y2, theta_out) @ z0
-
-  phi_out = np.arctan2(np.dot(y3, z2), np.dot(y3, y2))
-
-  return np.array([phi_out, theta_out, psi_out])

common/transformations/transformations.pyx

@@ -0,0 +1,173 @@
+# distutils: language = c++
+# cython: language_level = 3
+from openpilot.common.transformations.transformations cimport Matrix3, Vector3, Quaternion
+from openpilot.common.transformations.transformations cimport ECEF, NED, Geodetic
+
+from openpilot.common.transformations.transformations cimport euler2quat as euler2quat_c
+from openpilot.common.transformations.transformations cimport quat2euler as quat2euler_c
+from openpilot.common.transformations.transformations cimport quat2rot as quat2rot_c
+from openpilot.common.transformations.transformations cimport rot2quat as rot2quat_c
+from openpilot.common.transformations.transformations cimport euler2rot as euler2rot_c
+from openpilot.common.transformations.transformations cimport rot2euler as rot2euler_c
+from openpilot.common.transformations.transformations cimport rot_matrix as rot_matrix_c
+from openpilot.common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from openpilot.common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from openpilot.common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
+from openpilot.common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
+from openpilot.common.transformations.transformations cimport LocalCoord_c
+
+
+import numpy as np
+cimport numpy as np
+
+cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m):
+    return np.array([
+        [m(0, 0), m(0, 1), m(0, 2)],
+        [m(1, 0), m(1, 1), m(1, 2)],
+        [m(2, 0), m(2, 1), m(2, 2)],
+    ])
+
+cdef Matrix3 numpy2matrix(np.ndarray[double, ndim=2, mode="fortran"] m):
+    assert m.shape[0] == 3
+    assert m.shape[1] == 3
+    return Matrix3(<double*>m.data)
+
+cdef ECEF list2ecef(ecef):
+    cdef ECEF e
+    e.x = ecef[0]
+    e.y = ecef[1]
+    e.z = ecef[2]
+    return e
+
+cdef NED list2ned(ned):
+    cdef NED n
+    n.n = ned[0]
+    n.e = ned[1]
+    n.d = ned[2]
+    return n
+
+cdef Geodetic list2geodetic(geodetic):
+    cdef Geodetic g
+    g.lat = geodetic[0]
+    g.lon = geodetic[1]
+    g.alt = geodetic[2]
+    return g
+
+def euler2quat_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Quaternion q = euler2quat_c(e)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def quat2euler_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Vector3 e = quat2euler_c(q)
+    return [e(0), e(1), e(2)]
+
+def quat2rot_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Matrix3 r = quat2rot_c(q)
+    return matrix2numpy(r)
+
+def rot2quat_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Quaternion q = rot2quat_c(r)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def euler2rot_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Matrix3 r = euler2rot_c(e)
+    return matrix2numpy(r)
+
+def rot2euler_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Vector3 e = rot2euler_c(r)
+    return [e(0), e(1), e(2)]
+
+def rot_matrix(roll, pitch, yaw):
+    return matrix2numpy(rot_matrix_c(roll, pitch, yaw))
+
+def ecef_euler_from_ned_single(ecef_init, ned_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ned_pose[0], ned_pose[1], ned_pose[2])
+
+    cdef Vector3 e = ecef_euler_from_ned_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def ned_euler_from_ecef_single(ecef_init, ecef_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ecef_pose[0], ecef_pose[1], ecef_pose[2])
+
+    cdef Vector3 e = ned_euler_from_ecef_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def geodetic2ecef_single(geodetic):
+    cdef Geodetic g = list2geodetic(geodetic)
+    cdef ECEF e = geodetic2ecef_c(g)
+    return [e.x, e.y, e.z]
+
+def ecef2geodetic_single(ecef):
+    cdef ECEF e = list2ecef(ecef)
+    cdef Geodetic g = ecef2geodetic_c(e)
+    return [g.lat, g.lon, g.alt]
+
+
+cdef class LocalCoord:
+    cdef LocalCoord_c * lc
+
+    def __init__(self, geodetic=None, ecef=None):
+        assert (geodetic is not None) or (ecef is not None)
+        if geodetic is not None:
+            self.lc = new LocalCoord_c(list2geodetic(geodetic))
+        elif ecef is not None:
+            self.lc = new LocalCoord_c(list2ecef(ecef))
+
+    @property
+    def ned2ecef_matrix(self):
+        return matrix2numpy(self.lc.ned2ecef_matrix)
+
+    @property
+    def ecef2ned_matrix(self):
+        return matrix2numpy(self.lc.ecef2ned_matrix)
+
+    @property
+    def ned_from_ecef_matrix(self):
+        return self.ecef2ned_matrix
+
+    @property
+    def ecef_from_ned_matrix(self):
+        return self.ned2ecef_matrix
+
+    @classmethod
+    def from_geodetic(cls, geodetic):
+        return cls(geodetic=geodetic)
+
+    @classmethod
+    def from_ecef(cls, ecef):
+        return cls(ecef=ecef)
+
+    def ecef2ned_single(self, ecef):
+        assert self.lc
+        cdef ECEF e = list2ecef(ecef)
+        cdef NED n = self.lc.ecef2ned(e)
+        return [n.n, n.e, n.d]
+
+    def ned2ecef_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef ECEF e = self.lc.ned2ecef(n)
+        return [e.x, e.y, e.z]
+
+    def geodetic2ned_single(self, geodetic):
+        assert self.lc
+        cdef Geodetic g = list2geodetic(geodetic)
+        cdef NED n = self.lc.geodetic2ned(g)
+        return [n.n, n.e, n.d]
+
+    def ned2geodetic_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef Geodetic g = self.lc.ned2geodetic(n)
+        return [g.lat, g.lon, g.alt]
+
+    def __dealloc__(self):
+        del self.lc

common/util.cc

@@ -1,5 +1,4 @@
 #include "common/util.h"
-#include "common/swaglog.h"
 
 #include <sys/ioctl.h>
 #include <sys/stat.h>
@@ -13,7 +12,6 @@
 #include <iomanip>
 #include <random>
 #include <sstream>
-#include <limits>
 
 #ifdef __linux__
 #include <sys/prctl.h>
@@ -80,9 +78,8 @@ std::string read_file(const std::string& fn) {
   std::ifstream f(fn, std::ios::binary | std::ios::in);
   if (f.is_open()) {
     f.seekg(0, std::ios::end);
-    std::streamsize size = f.tellg();
-    // seekg and tellg on a directory doesn't return pos_type(-1) but max(streamsize)
-    if (f.good() && size > 0 && size < std::numeric_limits<std::streamsize>::max()) {
+    int size = f.tellg();
+    if (f.good() && size > 0) {
       std::string result(size, '\0');
       f.seekg(0, std::ios::beg);
       f.read(result.data(), size);
@@ -152,16 +149,11 @@ int safe_fflush(FILE *stream) {
   return ret;
 }
 
-int safe_ioctl(int fd, unsigned long request, void *argp, const char* exception_msg) {
+int safe_ioctl(int fd, unsigned long request, void *argp) {
   int ret;
   do {
     ret = ioctl(fd, request, argp);
   } while ((ret == -1) && (errno == EINTR));
-
-  if (ret == -1 && exception_msg) {
-    LOGE("safe_ioctl error: %s %s(%d) (fd: %d request: %lx argp: %p)", exception_msg, strerror(errno), errno, fd, request, argp);
-    throw std::runtime_error(exception_msg);
-  }
   return ret;
 }
 

common/util.h

@@ -36,7 +36,6 @@ const double MS_TO_KPH = 3.6;
 const double MS_TO_MPH = MS_TO_KPH * KM_TO_MILE;
 const double METER_TO_MILE = KM_TO_MILE / 1000.0;
 const double METER_TO_FOOT = 3.28084;
-const double METER_TO_KM = 1. / 1000.0;
 
 #define ALIGNED_SIZE(x, align) (((x) + (align)-1) & ~((align)-1))
 
@@ -89,7 +88,7 @@ int write_file(const char* path, const void* data, size_t size, int flags = O_WR
 FILE* safe_fopen(const char* filename, const char* mode);
 size_t safe_fwrite(const void * ptr, size_t size, size_t count, FILE * stream);
 int safe_fflush(FILE *stream);
-int safe_ioctl(int fd, unsigned long request, void *argp, const char* exception_msg = nullptr);
+int safe_ioctl(int fd, unsigned long request, void *argp);
 
 std::string readlink(const std::string& path);
 bool file_exists(const std::string& fn);