Merge branch 'master' into ui-neutral

.dockerignore

@@ -0,0 +1,18 @@
+**/.git
+.DS_Store
+*.dylib
+*.DSYM
+*.d
+*.pyc
+*.pyo
+.*.swp
+.*.swo
+.*.un~
+*.tmp
+*.o
+*.o-*
+*.os
+*.os-*
+
+venv/
+.venv/

.gitattributes

@@ -10,5 +10,14 @@
 *.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
-openpilot/selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
+selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-openpilot/common/hardware/comma/updater filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
+third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
+third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,8 @@
+---
+name: Enhancement
+about: For openpilot enhancement suggestions
+title: ''
+labels: 'enhancement'
+assignees: ''
+---
+

.github/labeler.yaml

@@ -8,24 +8,24 @@ chore:
 
 car:
   - changed-files:
-    - any-glob-to-all-files: '{openpilot/selfdrive/car/**,opendbc_repo}'
+    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
 
 simulation:
   - changed-files:
-    - any-glob-to-all-files: 'openpilot/tools/sim/**'
+    - any-glob-to-all-files: 'tools/sim/**'
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{openpilot/selfdrive/assets/**,openpilot/selfdrive/ui/**,openpilot/system/ui/**}'
+    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
 
 tools:
   - changed-files:
-    - any-glob-to-all-files: 'openpilot/tools/**'
+    - any-glob-to-all-files: 'tools/**'
 
 multilanguage:
   - changed-files:
-    - any-glob-to-all-files: 'openpilot/selfdrive/ui/translations/**'
+    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
 
 autonomy:
   - changed-files:
-    - any-glob-to-all-files: "{openpilot/selfdrive/modeld/models/**,openpilot/selfdrive/test/process_replay/model_replay_ref_commit,openpilot/sunnypilot/modeld*/models/**}"
+    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit,sunnypilot/modeld*/models/**}"

.github/pull_request_template.md

@@ -0,0 +1,68 @@
+<!-- Please copy and paste the relevant template -->
+
+<!--- ***** Template: Fingerprint *****
+
+**Car**
+Which car (make, model, year) this fingerprint is for
+
+**Route**
+A route with the fingerprint
+
+-->
+
+<!--- ***** Template: Car Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+**Route**
+
+Route: [a route with the bug fix]
+
+
+-->
+
+<!--- ***** Template: Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+
+-->
+
+<!--- ***** Template: Car Port *****
+
+**Checklist**
+
+- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
+- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
+- [ ] route with openpilot:
+- [ ] route with stock system:
+- [ ] car harness used (if comma doesn't sell it, put N/A):
+
+
+-->
+
+<!--- ***** Template: Refactor *****
+
+**Description**
+
+A description of the refactor, including the goals it accomplishes. 
+
+**Verification**
+
+Explain how you tested the refactor for regressions. 
+
+
+-->
+

.github/workflows/auto-cache/action.yaml

@@ -0,0 +1,58 @@
+name: 'automatically cache based on current runner'
+
+inputs:
+  path:
+    description: 'path to cache'
+    required: true
+  key:
+    description: 'key'
+    required: true
+  restore-keys:
+    description: 'restore-keys'
+    required: true
+  save:
+    description: 'whether to save the cache'
+    default: 'true'
+    required: false
+outputs:
+  cache-hit:
+    description: 'cache hit occurred'
+    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
+               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
+               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: setup namespace cache
+      id: ns-cache
+      if: ${{ contains(runner.name, 'nsc') }}
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        path: ${{ inputs.path }}
+
+    - name: setup github cache
+      id: gha-cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
+      uses: 'actions/cache@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: setup github cache
+      id: gha-cache-ro
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
+      uses: 'actions/cache/restore@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.path }}
+        sudo chmod -R 777 ${{ inputs.path }}
+        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -12,19 +12,19 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v6
+    - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
 
     # Check PR target branch
     - name: check branch
-      uses: Vankka/pr-target-branch-action@5da68a42bcb7b43d39104295a876a6f3f8d7908b
+      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
       if: github.repository == 'sunnypilot/sunnypilot'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -55,13 +55,13 @@ jobs:
               repo: context.repo.repo,
               issue_number: prNumber
             });
-
+            
             const hasDevC3Label = labels.some(label => label.name === process.env.PR_LABEL);
             const hasTrustLabel = labels.some(label => label.name === process.env.TRUST_FORK_PR_LABEL);
-
+            
             console.log(`PR #${prNumber} has ${process.env.PR_LABEL} label: ${hasDevC3Label}`);
             console.log(`PR #${prNumber} has ${process.env.TRUST_FORK_PR_LABEL} label: ${hasTrustLabel}`);
-
+            
             core.setOutput('has-dev', hasDevC3Label ? 'true' : 'false');
             core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
 
@@ -81,7 +81,7 @@ jobs:
             });
 
             console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
-
+            
             // Add a comment to the PR
             await github.rest.issues.createComment({
               owner: context.repo.owner,

.github/workflows/badges.yaml

@@ -0,0 +1,37 @@
+name: badges
+on:
+  schedule:
+    - cron: '0 * * * *'
+  workflow_dispatch:
+
+env:
+  BASE_IMAGE: sunnypilot-base
+  DOCKER_REGISTRY: ghcr.io/sunnypilot
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
+
+jobs:
+  badges:
+    name: create badges
+    runs-on: ubuntu-latest
+    if: github.repository == 'sunnypilot/sunnypilot'
+    permissions:
+      contents: write
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Push badges
+      run: |
+        ${{ env.RUN }} "python3 selfdrive/ui/translations/create_badges.py"
+
+        rm .gitattributes
+
+        git checkout --orphan badges
+        git rm -rf --cached .
+        git config user.email "badge-researcher@sunnypilot.ai"
+        git config user.name "Badge Researcher"
+
+        git add translation_badge.svg
+        git commit -m "Add/Update badges"
+        git push -f origin HEAD

.github/workflows/build-all-tinygrad-models.yaml

@@ -7,19 +7,6 @@ on:
         description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
         required: true
         type: string
-      target_hardware:
-        description: 'Hardware target to compile for (qcom or usbgpu)'
-        required: true
-        type: choice
-        default: 'qcom'
-        options:
-          - qcom
-          - usbgpu
-      hf_repo:
-        description: 'Hugging Face dataset repository'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot_models_v1'
 
 jobs:
   setup:
@@ -43,14 +30,14 @@ jobs:
         run: |
           cd sunnypilot
           export PYTHONPATH=$(pwd)
-          ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
+          ref=$(python3 sunnypilot/models/tinygrad_ref.py)
           echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
           echo "tinygrad_ref is $ref"
 
-      - name: Checkout docs repo (sunnypilot-models, gh-pages)
+      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -59,14 +46,13 @@ jobs:
         id: get-json
         run: |
           cd docs/docs
-          PREFIX="driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_' || '' }}v"
+          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
-          latest=$(ls ${PREFIX}*.json | sed -E "s/${PREFIX}([0-9]+)\.json/\1/" | sort -n | tail -1)
           next=$((latest+1))
-          json_file="${PREFIX}${next}.json"
+          json_file="driving_models_v${next}.json"
-          cp "${PREFIX}${latest}.json" "$json_file"
+          cp "driving_models_v${latest}.json" "$json_file"
           echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
           echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
-          echo "SRC_JSON_FILE=docs/docs/${PREFIX}${latest}.json" >> $GITHUB_ENV
+          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
 
       - name: Extract tinygrad models
         id: set-matrix
@@ -75,23 +61,45 @@ jobs:
           jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
           echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
-      - name: Get next recompiled dir number
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+      - run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      - name: Clone GitLab docs repo and create new recompiled dir
         id: create-recompiled-dir
         env:
-          HF_REPO: ${{ github.event.inputs.hf_repo }}
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
-          pip install huggingface_hub
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          recompiled_dir=$(python3 -c "
+          cd gitlab_docs
-          from huggingface_hub import HfApi
+          git checkout main
-          import re, sys
+          git sparse-checkout set --no-cone models/
-          api = HfApi()
+          cd models
-          files = api.list_repo_files(repo_id=sys.argv[1], repo_type='dataset')
+          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          dirs = [re.search(r'models/recompiled([0-9]+)', f) for f in files]
+          if [[ -z "$latest_dir" ]]; then
-          nums = [int(m.group(1)) for m in dirs if m]
+            next_dir=1
-          print(max(nums) + 1)
+          else
-          " "$HF_REPO")
+            next_dir=$((latest_dir+1))
+          fi
+          recompiled_dir="${next_dir}"
+          mkdir -p "recompiled${recompiled_dir}"
+          touch "recompiled${recompiled_dir}/.gitkeep"
+          cd ../..
           echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
 
+      - name: Push empty recompiled dir to GitLab
+        run: |
+          cd gitlab_docs
+          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
+          git push origin main
+
       - name: Push new JSON to GitHub docs repo
         run: |
           cd docs
@@ -112,33 +120,27 @@ jobs:
     with:
       upstream_branch: ${{ matrix.model.ref }}
       custom_name: ${{ matrix.model.display_name }}
-      is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
-      target_hardware: ${{ github.event.inputs.target_hardware }}
-      hf_repo: ${{ github.event.inputs.hf_repo }}
-      set_min_version: ${{ github.event.inputs.set_min_version }}
-      tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
     secrets: inherit
 
   retry_failed_models:
     needs: [setup, get_and_build]
     runs-on: ubuntu-latest
-    if: ${{ !cancelled() && needs.setup.result == 'success' && (needs.get_and_build.result == 'success' || needs.get_and_build.result == 'failure') }}
+    if: ${{ needs.setup.result != 'failure' && !cancelled() }}
     outputs:
       retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
     steps:
       - uses: actions/download-artifact@v4
         with:
-          pattern: artifact-name-*
+          pattern: model-*
           path: output
-        continue-on-error: true
 
       - id: set-retry-matrix
         run: |
           echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
-          built=(); while IFS= read -r line; do [ -n "$line" ] && built+=("$line"); done < <(
+          built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'artifact-name-*' -printf "%f\n" 2>/dev/null | sed -E 's/^artifact-name-//' | awk '{gsub(/^ +| +$/, ""); print}'
+            ls output | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
           )
           jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
             'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
@@ -146,7 +148,7 @@ jobs:
 
   retry_get_and_build:
     needs: [setup, get_and_build, retry_failed_models]
-    if: ${{ !cancelled() && needs.retry_failed_models.result == 'success' && needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '' }}
+    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
     strategy:
       matrix:
         model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
@@ -155,12 +157,147 @@ jobs:
     with:
       upstream_branch: ${{ matrix.model.ref }}
       custom_name: ${{ matrix.model.display_name }}
-      is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
-      target_hardware: ${{ github.event.inputs.target_hardware }}
       artifact_suffix: -retry
-      hf_repo: ${{ github.event.inputs.hf_repo }}
-      set_min_version: ${{ github.event.inputs.set_min_version }}
-      tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
     secrets: inherit
+
+  publish_models:
+    name: Publish models sequentially
+    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
+    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      matrix:
+        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
+    env:
+      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
+      JSON_FILE: ${{ needs.setup.outputs.json_file }}
+      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
+    steps:
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+
+      - name: Add GitLab.com SSH key to known_hosts
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      - name: Clone GitLab docs repo
+        env:
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+        run: |
+          echo "Cloning GitLab"
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          cd gitlab_docs
+          echo "checkout models/${RECOMPILED_DIR}"
+          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
+          git checkout main
+          cd ..
+
+      - name: Checkout docs repo
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/sunnypilot-docs
+          ref: gh-pages
+          path: docs
+          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
+
+      - name: Validate recompiled dir and JSON version
+        run: |
+          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
+            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
+            exit 1
+          fi
+          if [ ! -f "$JSON_FILE" ]; then
+            echo "JSON file $JSON_FILE does not exist!"
+            exit 1
+          fi
+
+      - name: Download artifact name file
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
+          path: artifact_name
+
+      - name: Read artifact name
+        id: read-artifact-name
+        run: |
+          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+
+      - name: Download model artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
+          path: output
+
+      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
+        run: |
+          find output -type f -name '*.onnx' -delete
+          find output -type f -name 'big_*.pkl' -delete
+          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
+
+      - name: Copy model artifacts to gitlab
+        env:
+          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+        run: |
+          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          mkdir -p "$ARTIFACT_DIR"
+          for path in output/*; do
+            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
+              continue
+            fi
+            name="$(basename "$path")"
+            if [ -d "$path" ]; then
+              mkdir -p "$ARTIFACT_DIR/$name"
+              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
+              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
+            else
+              cp "$path" "$ARTIFACT_DIR/"
+              echo "Copied file $name -> $ARTIFACT_DIR/"
+            fi
+          done
+
+      - name: Push recompiled dir to GitLab
+        env:
+          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+        run: |
+          cd gitlab_docs
+          git checkout main
+          git pull origin main
+          for d in models/"$RECOMPILED_DIR"/*/; do
+            git sparse-checkout add "$d"
+          done
+          git add models/"$RECOMPILED_DIR"
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
+          git push origin main
+      - run: |
+          cd docs
+          git pull origin gh-pages
+
+      - name: update json
+        run: |
+          ARGS=""
+          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
+          ARGS="$ARGS --sort-by-date"
+          ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
+          eval python3 docs/json_parser.py \
+            --json-path "$JSON_FILE" \
+            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
+            $ARGS
+
+      - name: Push updated json to GitHub
+        run: |
+          cd docs
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git checkout gh-pages
+          git add docs/"$(basename $JSON_FILE)"
+          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
+          git push origin gh-pages

.github/workflows/build-single-tinygrad-model.yaml

@@ -12,11 +12,11 @@ on:
         required: false
         type: string
       recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 1 for recompiled1)'
+        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
         required: true
         type: string
       json_version:
-        description: 'driving_models version number to update (e.g. 18 for driving_models_v18.json)'
+        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
         required: true
         type: string
       artifact_suffix:
@@ -24,29 +24,11 @@ on:
         required: false
         type: string
         default: ''
-      is_20hz:
+      bypass_push:
-        description: 'Is this a 20Hz model'
+        description: 'Bypass pushing to GitLab for build-all'
         required: false
-        type: boolean
         default: true
-      target_hardware:
+        type: boolean
-        description: 'Hardware target to compile for (qcom or usbgpu)'
-        required: false
-        type: string
-        default: 'qcom'
-      hf_repo:
-        description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot_models_v1'
-      set_min_version:
-        description: 'Minimum selector version'
-        required: false
-        type: string
-      tinygrad_ref:
-        description: 'Tinygrad reference'
-        required: false
-        type: string
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -57,11 +39,6 @@ on:
         description: 'Custom name for the model (no date, only name)'
         required: false
         type: string
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
       recompiled_dir:
         description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
         required: true
@@ -76,11 +53,12 @@ on:
         default: 'None'
         options:
           - None
-          - Master Models
+          - Simple Plan Models
-          - Release Models
+          - Space Lab Models
-          - 2026 World Models
+          - TR Models
-          - 2026 Deep RL Models
+          - DTR Models
           - Custom Merge Models
+          - FOF series models
           - Other
       custom_model_folder:
         description: 'Custom model folder name (if "Other" selected)'
@@ -94,22 +72,9 @@ on:
         description: 'Minimum selector version'
         required: false
         type: string
-      target_hardware:
-        description: 'Hardware target to compile for'
-        required: false
-        type: choice
-        default: 'qcom'
-        options:
-          - qcom
-          - usbgpu
-      hf_repo:
-        description: 'Hugging Face dataset repository'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot_models_v1'
 env:
   RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
-  JSON_FILE: docs/docs/driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_v' || 'v' }}${{ inputs.json_version }}.json
+  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
 
 jobs:
   build_model:
@@ -117,52 +82,58 @@ jobs:
     with:
       upstream_branch: ${{ inputs.upstream_branch }}
       custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
-      is_20hz: ${{ inputs.is_20hz }}
+      is_20hz: true
       artifact_suffix: ${{ inputs.artifact_suffix }}
-      target_hardware: ${{ inputs.target_hardware }}
     secrets: inherit
 
   publish_model:
-    if: ${{ !cancelled() && needs.build_model.result == 'success' }}
+    if: ${{ !inputs.bypass_push && !cancelled() }}
     concurrency:
-      group: hf-push-${{ inputs.recompiled_dir }}
+      group: gitlab-push-${{ inputs.recompiled_dir }}
       cancel-in-progress: false
     needs: build_model
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: write
     steps:
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+
+      - name: Add GitLab.com SSH key to known_hosts
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      - name: Clone GitLab docs repo
+        env:
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+        run: |
+          echo "Cloning GitLab"
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          cd gitlab_docs
+          echo "checkout models/${RECOMPILED_DIR}"
+          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
+          git checkout main
+          cd ..
+
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
-          repository: sunnypilot/sunnypilot-models
+          repository: sunnypilot/sunnypilot-docs
           ref: gh-pages
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
 
-      - name: Install huggingface_hub
+      - name: Validate recompiled dir and JSON version
-        run: pip install --upgrade "huggingface_hub>=0.22.0"
-
-      - name: Validate hf_repo and JSON version
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
         run: |
+          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
+            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
+            exit 1
+          fi
           if [ ! -f "$JSON_FILE" ]; then
             echo "JSON file $JSON_FILE does not exist!"
             exit 1
           fi
-          python3 -c "
-          import sys
-          from huggingface_hub import HfApi
-          try:
-              api = HfApi()
-              api.repo_info(repo_id=sys.argv[1], repo_type='dataset')
-              print(f'Success: Repo {sys.argv[1]} exists.')
-          except Exception as e:
-              print('HF validation failed:', e)
-              sys.exit(1)
-          " "${{ inputs.hf_repo }}"
 
       - name: Download artifact name file
         uses: actions/download-artifact@v4
@@ -182,26 +153,49 @@ jobs:
           name: ${{ steps.read-artifact-name.outputs.artifact_name }}
           path: output
 
-      - name: Create models folder
+      - name: Remove unwanted files
+        run: |
+          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
+          find output -type f -name 'dmonitoring_model.onnx' -delete
+
+      - name: Copy model artifact(s) to GitLab recompiled dir
         env:
           ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
         run: |
-          mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/"
+          mkdir -p "$ARTIFACT_DIR"
-          rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt"
+          for path in output/*; do
+            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
+              continue
+            fi
+            name="$(basename "$path")"
+            if [ -d "$path" ]; then
+              mkdir -p "$ARTIFACT_DIR/$name"
+              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
+              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
+            else
+              cp "$path" "$ARTIFACT_DIR/"
+              echo "Copied file $name -> $ARTIFACT_DIR/"
+            fi
+          done
 
-      - name: Upload to Hugging Face
+      - name: Push recompiled dir to GitLab
         env:
-          HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
+          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
         run: |
-          hf upload ${{ inputs.hf_repo }} \
+          cd gitlab_docs
-            output/ \
+          git checkout main
-            "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \
+          git pull origin main
-            --repo-type=dataset
+          for d in models/"$RECOMPILED_DIR"/*/; do
+            git sparse-checkout add "$d"
+          done
+          git add models/"$RECOMPILED_DIR"
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
+          git push origin main
 
-      - name: Pull gh-pages
+      - run: |
-        run: |
           cd docs
           git pull origin gh-pages
 
@@ -217,11 +211,9 @@ jobs:
           fi
           [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
           [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
-          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
-          [ -n "${{ inputs.tinygrad_ref }}" ] && ARGS="$ARGS --tinygrad-ref \"${{ inputs.tinygrad_ref }}\""
           eval python3 docs/json_parser.py \
             --json-path "$JSON_FILE" \
-            --recompiled-dir "local_models/$RECOMPILED_DIR" \
+            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
             --sort-by-date \
             $ARGS
 

.github/workflows/cereal_validation.yaml

@@ -6,7 +6,7 @@ on:
       - master
   pull_request:
     paths:
-      - 'openpilot/cereal/**'
+      - 'cereal/**'
   workflow_dispatch:
   workflow_call:
     inputs:
@@ -20,78 +20,57 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CI: 1
+  PYTHONWARNINGS: error
+  BASE_IMAGE: openpilot-base
+  BUILD: selfdrive/test/docker_build.sh base
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
+  generate_cereal_artifact:
+    name: Generate cereal validation artifacts
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
+      - name: Generate the log file
+        run: |
+          ${{ env.RUN }} "cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema_instances.bin" && \
+          ls -la
+          ls -la cereal/messaging/tests
+      - name: 'Prepare artifact'
+        run: |
+          mkdir -p "cereal/messaging/tests/cereal_validations"
+          cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py"
+          cp schema_instances.bin "cereal/messaging/tests/cereal_validations/schema_instances.bin"
+      - name: 'Upload Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: cereal_validations
+          path: cereal/messaging/tests/cereal_validations
+
   validate_cereal_with_upstream:
     name: Validate cereal with Upstream
     runs-on: ubuntu-24.04
+    needs: generate_cereal_artifact
     steps:
-      - name: Checkout sunnypilot cereal
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v6
-        with:
-          sparse-checkout: |
-            openpilot/cereal
-            uv.lock
-          submodules: false
-
-      - name: Init sunnypilot opendbc submodule
-        run: git submodule update --init --depth 1 opendbc_repo
-
-      - name: Locate sunnypilot capnp import paths
-        id: locate-sp-capnp
-        run: |
-          SP_IMPORT_ARGS=""
-          SP_CAR_CAPNP=$(find opendbc_repo -maxdepth 4 -name car.capnp -path '*/opendbc/car/car.capnp' -printf '%h\n' -quit)
-          if [ -n "$SP_CAR_CAPNP" ]; then
-            SP_IMPORT_ARGS="-I $SP_CAR_CAPNP"
-            echo "Found sunnypilot car.capnp at: $SP_CAR_CAPNP"
-          fi
-          echo "import_args=$SP_IMPORT_ARGS" >> "$GITHUB_OUTPUT"
-
-      - name: Checkout upstream openpilot
-        uses: actions/checkout@v6
         with:
           repository: 'commaai/openpilot'
-          path: upstream_openpilot
+          submodules: true
           ref: "refs/heads/master"
-
+      - uses: ./.github/workflows/setup-with-retry
-      - name: Init upstream opendbc submodule
+      - name: Build openpilot
-        working-directory: upstream_openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc) cereal"
-        run: git submodule update --init --depth 1 opendbc_repo
+      - name: Download build artifacts
-
+        uses: actions/download-artifact@v4
-      - name: Locate upstream capnp paths
+        with:
-        id: locate-capnp
+          name: cereal_validations
+          path: cereal/messaging/tests/cereal_validations
+      - name: 'Run the validation'
         run: |
-          CEREAL_DIR=$(find upstream_openpilot -maxdepth 4 -name log.capnp -path '*/cereal/log.capnp' -printf '%h\n' -quit)
+          chmod +x cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py
-          if [ -z "$CEREAL_DIR" ]; then
+          ${{ env.RUN }} "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f cereal/messaging/tests/cereal_validations/schema_instances.bin"
-            echo "::error::Could not locate cereal/log.capnp in upstream openpilot"
-            exit 1
-          fi
-          echo "cereal_dir=$CEREAL_DIR" >> "$GITHUB_OUTPUT"
-          echo "Found upstream cereal at: $CEREAL_DIR"
-
-          IMPORT_ARGS=""
-          CAR_CAPNP=$(find upstream_openpilot -maxdepth 5 -name car.capnp -path '*/opendbc/car/car.capnp' -printf '%h\n' -quit)
-          if [ -n "$CAR_CAPNP" ]; then
-            IMPORT_ARGS="-I $CAR_CAPNP"
-            echo "Found car.capnp at: $CAR_CAPNP"
-          fi
-          echo "import_args=$IMPORT_ARGS" >> "$GITHUB_OUTPUT"
-
-      - name: Install pycapnp
-        run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
-          pip install "pycapnp==${PYCAPNP_VER}"
-
-      - name: Generate sunnypilot schema
-        run: |
-          python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
-            -g -f /tmp/sp_schema.json --cereal-dir openpilot/cereal \
-            ${{ steps.locate-sp-capnp.outputs.import_args }}
-
-      - name: Validate against upstream
-        run: |
-          python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
-            -r -f /tmp/sp_schema.json --cereal-dir ${{ steps.locate-capnp.outputs.cereal_dir }} \
-            ${{ steps.locate-capnp.outputs.import_args }}

.github/workflows/ci_weekly_report.yaml

@@ -0,0 +1,101 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
+jobs:
+  setup:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
+    steps:
+      - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
+        run: |
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.run_number }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always() && github.repository == 'commaai/openpilot'
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
+              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
+                case "failure":
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
+                case "canceled":
+                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
+              }
+            });
+            return JSON.stringify({"jobs": report});
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
+
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -0,0 +1,17 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    uses: sunnypilot/sunnypilot/.github/workflows/tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,21 @@
+name: 'compile openpilot'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/diff_report.yaml

@@ -1,58 +0,0 @@
-name: diff report
-
-on:
-  pull_request_target:
-    types: [opened, synchronize, reopened]
-
-jobs:
-  comment:
-    name: comment
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-    - name: Wait for process replay
-      id: wait
-      continue-on-error: true
-      uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        check-name: process replay
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        allowed-conclusions: success,failure
-        wait-interval: 20
-    - name: Get tests run ID
-      if: steps.wait.outcome == 'success'
-      id: tests-run
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        run_id=$(gh api \
-          "repos/${{ github.repository }}/actions/workflows/tests.yaml/runs?event=pull_request&head_sha=${{ github.event.pull_request.head.sha }}&per_page=1" \
-          --jq '.workflow_runs[0].id // empty')
-        if [ -z "$run_id" ]; then
-          echo "No tests.yaml run found for ${{ github.event.pull_request.head.sha }}" >&2
-          exit 1
-        fi
-        echo "run-id=$run_id" >> "$GITHUB_OUTPUT"
-    - name: Download diff
-      if: steps.wait.outcome == 'success'
-      continue-on-error: true
-      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        repository: ${{ github.repository }}
-        run-id: ${{ steps.tests-run.outputs.run-id }}
-        name: diff_report_${{ github.event.number }}
-        path: .
-    - name: Comment on PR
-      if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != ''
-      uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
-      with:
-        file-path: diff_report.txt
-        comment-tag: diff_report
-        pr-number: ${{ github.event.number }}
-        github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -15,11 +15,6 @@ concurrency:
   group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
-env:
-  GIT_CONFIG_COUNT: 1
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
-
 jobs:
   docs:
     name: build docs
@@ -27,18 +22,19 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
     # Build
     - name: Build docs
       run: |
-        git lfs pull
+        # TODO: can we install just the "docs" dependency group without the normal deps?
-        python docs/serve.py --build
+        pip install mkdocs
+        mkdocs build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs
@@ -49,14 +45,14 @@ jobs:
       run: |
         set -x
 
-        source tools/release/identity.sh
+        source release/identity.sh
 
         cd openpilot-docs
         git checkout --orphan tmp
         git rm -rf .
 
         # copy over docs
-        cp -r ../docs/_site/ docs/
+        cp -r ../docs_site/ docs/
 
         # GitHub pages config
         touch docs/.nojekyll

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,44 +5,7 @@ on:
     types: [created, edited]
 
 jobs:
-  cleanup-branches:
+  # TODO: gc old branches in a separate job in this workflow
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Delete stale Jenkins branches
-        uses: actions/github-script@v9
-        with:
-          script: |
-            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
-            const prefixes = ['tmp-jenkins', '__jenkins'];
-
-            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              per_page: 100,
-            })) {
-              for (const branch of response.data) {
-                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
-
-                const { data: commit } = await github.rest.repos.getCommit({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: branch.commit.sha,
-                });
-
-                const commitDate = new Date(commit.commit.committer.date).getTime();
-                if (commitDate < cutoff) {
-                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
-                  await github.rest.git.deleteRef({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                    ref: `heads/${branch.name}`,
-                  });
-                }
-              }
-            }
-
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
@@ -52,7 +15,7 @@ jobs:
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v9
+      uses: actions/github-script@v7
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +35,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v7
+      uses: actions/checkout@v4
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -86,7 +49,7 @@ jobs:
 
     - name: Delete trigger comment
       if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v9
+      uses: actions/github-script@v7
       with:
         script: |
           await github.rest.issues.deleteComment({

.github/workflows/lfs-maintenance.yaml

@@ -65,21 +65,8 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Configure LFS transfer settings
-        run: |
-          git config lfs.activitytimeout 300
-          git config lfs.transfer.maxretries 5
-          git config lfs.concurrenttransfers 4
-
       - name: Push LFS
         id: sync-and-commit
         run: |
           git lfs ls-files -l
-          for attempt in 1 2 3; do
+          git lfs push --all origin
-            echo "Push attempt $attempt..."
-            git lfs push --all origin && exit 0
-            echo "Attempt $attempt failed, retrying in 30s..."
-            sleep 30
-          done
-          echo "All push attempts failed"
-          exit 1

.github/workflows/mici_raylib_ui_preview.yaml

@@ -0,0 +1,151 @@
+name: "mici raylib ui preview"
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened, edited]
+    branches:
+      - 'master'
+    paths:
+      - 'selfdrive/assets/**'
+      - 'selfdrive/ui/**'
+      - 'system/ui/**'
+  workflow_dispatch:
+
+env:
+  UI_JOB_NAME: "Create mici raylib UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-mici-raylib-ui"
+  MASTER_BRANCH_NAME: "openpilot_master_ui_mici_raylib"
+  # All report files are pushed here
+  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
+
+jobs:
+  preview:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    name: preview
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Waiting for ui generation to end
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ env.SHA }}
+          check-name: ${{ env.UI_JOB_NAME }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allowed-conclusions: success
+          wait-interval: 20
+
+      - name: Getting workflow run ID
+        id: get_run_id
+        run: |
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+
+      - name: Getting proposed ui  # filename: pr_ui/mici_ui_replay.mp4
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          run_id: ${{ steps.get_run_id.outputs.run_id }}
+          search_artifacts: true
+          name: mici-raylib-report-1-${{ env.REPORT_NAME }}
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui  # filename: master_ui_raylib/mici_ui_replay.mp4
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui_raylib
+          ref: ${{ env.MASTER_BRANCH_NAME }}
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git checkout --orphan=new_master_ui_mici_raylib
+          git rm -rf *
+          git branch -D ${{ env.MASTER_BRANCH_NAME }}
+          git branch -m ${{ env.MASTER_BRANCH_NAME }}
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
+          git commit -m "mici raylib video for commit ${{ env.SHA }}"
+          git push origin ${{ env.MASTER_BRANCH_NAME }} --force
+
+      - name: Setup FFmpeg
+        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: |
+          # Find the video file from PR
+          pr_video="${{ github.workspace }}/pr_ui/mici_ui_replay_proposed.mp4"
+          mv "${{ github.workspace }}/pr_ui/mici_ui_replay.mp4" "$pr_video"
+
+          master_video="${{ github.workspace }}/pr_ui/mici_ui_replay_master.mp4"
+          mv "${{ github.workspace }}/master_ui_raylib/mici_ui_replay.mp4" "$master_video"
+
+          # Run report
+          export PYTHONPATH=${{ github.workspace }}
+          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
+          diff_exit_code=0
+          python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py "${{ github.workspace }}/pr_ui/mici_ui_replay_master.mp4" "${{ github.workspace }}/pr_ui/mici_ui_replay_proposed.mp4" "diff.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
+
+          # Copy diff report files
+          cp ${{ github.workspace }}/selfdrive/ui/tests/diff/report/diff.html ${{ github.workspace }}/pr_ui/
+          cp ${{ github.workspace }}/selfdrive/ui/tests/diff/report/diff.mp4 ${{ github.workspace }}/pr_ui/
+
+          REPORT_URL="https://sunnypilot.github.io/ci-artifacts/diff_pr_${{ github.event.number }}.html"
+          if [ $diff_exit_code -eq 0 ]; then
+            DIFF="✅ Videos are identical! [View Diff Report]($REPORT_URL)"
+          else
+            DIFF="❌ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"
+          fi
+          echo "DIFF=$DIFF" >> "$GITHUB_OUTPUT"
+
+      - name: Saving proposed ui
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          # Overwrite PR branch w/ proposed ui, and master ui at this point in time for future reference
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
+          git commit -m "mici raylib video for PR #${{ github.event.number }}"
+          git push origin ${{ env.BRANCH_NAME }} --force
+
+          # Append diff report to report files branch
+          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
+          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
+          cp ${{ github.workspace }}/selfdrive/ui/tests/diff/report/diff.html diff_pr_${{ github.event.number }}.html
+          git add diff_pr_${{ github.event.number }}.html
+          git commit -m "mici raylib ui diff report for PR #${{ github.event.number }}" || echo "No changes to commit"
+          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
+
+      - name: Comment Video on PR
+        if: github.event_name == 'pull_request_target'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_video_mici_raylib **${{ github.run_id }}**)_ -->
+            ## mici raylib UI Preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_video_mici_raylib
+          pr_number: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/model_review.yaml

@@ -0,0 +1,42 @@
+name: "model review"
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - 'selfdrive/modeld/models/*.onnx'
+  workflow_dispatch:
+
+jobs:
+  comment:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Checkout master
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          path: base
+      - run: git lfs pull
+      - run: cd base && git lfs pull
+
+      - run: pip install onnx
+
+      - name: scripts/reporter.py
+        id: report
+        run: |
+          echo "content<<EOF" >> $GITHUB_OUTPUT
+          echo "## Model Review" >> $GITHUB_OUTPUT
+          MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Post model report comment
+        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
+        with:
+          header: model-review
+          message: ${{ steps.report.outputs.content }}

.github/workflows/prebuilt.yaml

@@ -0,0 +1,39 @@
+name: prebuilt
+on:
+  schedule:
+    - cron: '0 * * * *'
+  workflow_dispatch:
+
+env:
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: release/ci/docker_build_sp.sh prebuilt
+
+jobs:
+  build_prebuilt:
+    name: build prebuilt
+    runs-on: ubuntu-latest
+    if: github.repository == 'sunnypilot/sunnypilot'
+    env:
+      PUSH_IMAGE: true
+    permissions:
+      checks: read
+      contents: read
+      packages: write
+    steps:
+    - name: Wait for green check mark
+      if: ${{ github.event_name != 'workflow_dispatch' }}
+      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
+      with:
+        ref: master
+        wait-interval: 30
+        running-workflow-name: 'build prebuilt'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        check-regexp: ^((?!.*(build master-ci).*).)*$
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - run: git lfs pull
+    - name: Build and Push docker image
+      run: |
+        $DOCKER_LOGIN
+        eval "$BUILD"

.github/workflows/raylib_ui_preview.yaml

@@ -0,0 +1,175 @@
+name: "raylib ui preview"
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened, edited]
+    branches:
+      - 'master'
+    paths:
+      - 'selfdrive/assets/**'
+      - 'selfdrive/ui/**'
+      - 'system/ui/**'
+  workflow_dispatch:
+
+env:
+  UI_JOB_NAME: "Create raylib UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-raylib-ui"
+
+jobs:
+  preview:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    name: preview
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Waiting for ui generation to start
+        run: sleep 30
+
+      - name: Waiting for ui generation to end
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ env.SHA }}
+          check-name: ${{ env.UI_JOB_NAME }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allowed-conclusions: success
+          wait-interval: 20
+
+      - name: Getting workflow run ID
+        id: get_run_id
+        run: |
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+
+      - name: Getting proposed ui
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          run_id: ${{ steps.get_run_id.outputs.run_id }}
+          search_artifacts: true
+          name: raylib-report-1-${{ env.REPORT_NAME }}
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui_raylib
+          ref: openpilot_master_ui_raylib
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git checkout --orphan=new_master_ui_raylib
+          git rm -rf *
+          git branch -D openpilot_master_ui_raylib
+          git branch -m openpilot_master_ui_raylib
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "raylib screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui_raylib --force
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: >-
+          sudo apt-get update && sudo apt-get install -y imagemagick
+
+          scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device')
+          A=($scenes)
+
+          DIFF=""
+          TABLE="<details><summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
+
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+            # Check if the master file exists
+            if [ ! -f "${{ github.workspace }}/master_ui_raylib/${A[$i]}.png" ]; then
+              # This is a new file in PR UI that doesn't exist in master
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              mv ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            else
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
+            fi
+          done
+
+          TABLE="${TABLE}</table></details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
+
+      - name: Saving proposed ui
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
+          git commit -m "raylib screenshots for PR #${{ github.event.number }}"
+          git push origin ${{ env.BRANCH_NAME }} --force
+
+      - name: Comment Screenshots on PR
+        if: github.event_name == 'pull_request_target'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_screenshots_raylib **${{ github.run_id }}**)_ -->
+            ## raylib UI Preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_screenshots_raylib
+          pr_number: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -4,33 +4,39 @@ on:
     - cron: '0 9 * * *'
   workflow_dispatch:
 
-env:
-  GIT_CONFIG_COUNT: 1
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
-
 jobs:
   build___nightly:
     name: build __nightly
+    env:
+      ImageOS: ubuntu24
+    container:
+      image: ghcr.io/sunnypilot/sunnypilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     permissions:
       checks: read
       contents: write
     steps:
+    - name: Install wait-on-check-action dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
-      uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
+      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build __nightly'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
+        check-regexp: ^((?!.*(build prebuilt).*).)*$
-    - uses: actions/checkout@v4 # checkout version > v4 breaks nightly release script
+    - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-    - run: ./tools/op.sh setup
+    - name: Pull LFS
+      run: |
+        git config --global --add safe.directory '*'
+        git lfs pull
     - name: Push __nightly
-      run: BRANCH=__nightly tools/release/build_stripped.sh
+      run: BRANCH=__nightly release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -3,64 +3,65 @@ name: repo maintenance
 on:
   schedule:
     - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
-  pull_request:
-    types: [closed]
   workflow_dispatch:
 
 env:
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: sunnypilot-base
-  GIT_CONFIG_COUNT: 1
+  BUILD: release/ci/docker_build_sp.sh base
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
 
 jobs:
+  update_translations:
+    runs-on: ubuntu-latest
+    if: github.repository == 'sunnypilot/sunnypilot'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Update translations
+        run: |
+          ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish"
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
+        with:
+          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
+          commit-message: "Update translations"
+          title: "[bot] Update translations"
+          body: "Automatic PR from repo-maintenance -> update_translations"
+          branch: "update-translations"
+          base: "master"
+          delete-branch: true
+          labels: bot
+
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
-    if: >-
+    container:
-      github.repository == 'sunnypilot/sunnypilot' &&
+      image: ghcr.io/sunnypilot/sunnypilot-base:latest
-      (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+    if: github.repository == 'sunnypilot/sunnypilot'
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
     - name: uv lock
-      run: uv lock --upgrade
+      if: github.repository == 'commaai/openpilot'
-    - name: uv pip tree
-      id: pip_tree
       run: |
-        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
+        python3 -m ensurepip --upgrade
-        uv pip tree >> $GITHUB_OUTPUT
+        pip3 install uv
-        echo 'EOF' >> $GITHUB_OUTPUT
+        uv lock --upgrade
-    - name: venv size
-      id: venv_size
-      run: |
-        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
-        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
-        echo "" >> $GITHUB_OUTPUT
-        echo "Top 10 by size:" >> $GITHUB_OUTPUT
-        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
-          | grep -v '\.dist-info' \
-          | grep -v '__pycache__' \
-          | sort -rh \
-          | head -10 \
-          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
     - name: bump submodules
       run: |
-        git config submodule.msgq.update none
+        git config --global --add safe.directory '*'
-        git config submodule.rednose_repo.update none
-        git config submodule.teleoprtc_repo.update none
         git config submodule.tinygrad.update none
         git submodule update --remote
         git add .
     - name: update car docs
       run: |
-        python openpilot/selfdrive/car/docs.py
+        export PYTHONPATH="$PWD"
+        scons -j$(nproc) --minimal opendbc_repo
+        python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
+      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:
         author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
         token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
@@ -69,64 +70,5 @@ jobs:
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: |
+        body: 'Automatic PR from repo-maintenance -> package_updates'
-          Automatic PR from repo-maintenance -> package_updates
-
-          ```
-          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
-          ${{ steps.venv_size.outputs.VENV_SIZE }}
-          ```
-
-          ```
-          $ uv pip tree
-          ${{ steps.pip_tree.outputs.PIP_TREE }}
-          ```
         labels: bot
-
-  cleanup_closed_branches:
-    if: github.repository == 'commaai/openpilot' && github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: read
-    steps:
-      - uses: actions/github-script@v9
-        with:
-          script: |
-            const { owner, repo } = context.repo;
-            const upstream = `${owner}/${repo}`;
-
-            const closed = context.payload.pull_request;
-            if (closed) {
-              if (closed.head.repo?.full_name === upstream) {
-                await github.rest.git.deleteRef({ owner, repo, ref: `heads/${closed.head.ref}` }).catch(console.log);
-              }
-              return;
-            }
-
-            for await (const response of github.paginate.iterator(github.rest.pulls.list, {
-              owner,
-              repo,
-              state: 'closed',
-              per_page: 100,
-            })) {
-              for (const pr of response.data) {
-                if (pr.head.repo?.full_name !== upstream) continue;
-
-                const branch = pr.head.ref;
-                try {
-                  await github.rest.git.deleteRef({
-                    owner,
-                    repo,
-                    ref: `heads/${branch}`,
-                  });
-                  console.log(`Deleted branch ${branch} (PR #${pr.number})`);
-                } catch (error) {
-                  if (error.status === 422 || error.status === 403) {
-                    console.log(`Skipping branch ${branch} (PR #${pr.number}): ${error.message}`);
-                    continue;
-                  }
-                  throw error;
-                }
-              }
-            }

.github/workflows/setup-with-retry/action.yaml

@@ -0,0 +1,52 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: false
+    default: ''
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
+
+outputs:
+  duration:
+    description: 'Duration of the setup process in seconds'
+    value: ${{ steps.get_duration.outputs.duration }}
+
+runs:
+  using: "composite"
+  steps:
+    - id: start_time
+      shell: bash
+      run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        is_retried: true
+    - id: get_duration
+      shell: bash
+      run: |
+        END_TIME=$(date +%s)
+        DURATION=$((END_TIME - START_TIME))
+        echo "Total duration: $DURATION seconds"
+        echo "duration=$DURATION" >> $GITHUB_OUTPUT

.github/workflows/setup/action.yaml

@@ -0,0 +1,56 @@
+name: 'openpilot env setup'
+
+inputs:
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    required: false
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
+    - shell: bash
+      name: No retries!
+      run: |
+        if [ "${{ github.run_attempt }}" -gt ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association == 'NONE' && 2 || 1}} ]; then
+          echo -e "\033[0;31m##################################################"
+          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
+          echo -e "\033[0;31m##################################################\033[0m"
+          exit 1
+        fi
+
+    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
+    - shell: bash
+      run: git lfs pull
+
+    # build cache
+    - id: date
+      shell: bash
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - shell: bash
+      run: echo "$CACHE_COMMIT_DATE"
+    - id: scons-cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    # build our docker image
+    - shell: bash
+      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -13,7 +13,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 
@@ -34,7 +34,7 @@ jobs:
   stale_drafts:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 

.github/workflows/sunnypilot-build-model.yaml

@@ -6,7 +6,7 @@ env:
   SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   UPSTREAM_REPO: "commaai/openpilot"
   TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
-  MODELS_DIR: ${{ github.workspace }}/openpilot/selfdrive/modeld/models
+  MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
 
 on:
   workflow_call:
@@ -30,11 +30,6 @@ on:
         required: false
         type: string
         default: ''
-      target_hardware:
-        description: 'Hardware target to compile for (qcom or usbgpu)'
-        required: false
-        type: string
-        default: 'qcom'
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -51,14 +46,6 @@ on:
         required: false
         type: boolean
         default: true
-      target_hardware:
-        description: 'Hardware target to compile for'
-        required: true
-        type: choice
-        options:
-          - qcom
-          - usbgpu
-        default: 'qcom'
 
 
 run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
@@ -80,7 +67,6 @@ jobs:
         with:
           repository: commaai/openpilot
           ref: ${{ inputs.upstream_branch }}
-          fetch-depth: 1
           submodules: recursive
           path: openpilot
 
@@ -90,30 +76,23 @@ jobs:
         with:
           repository: sunnypilot/sunnypilot
           ref: ${{ inputs.upstream_branch }}
-          fetch-depth: 1
           submodules: recursive
           path: openpilot
       - name: Get commit date
         id: commit-date
         run: |
-          cd ${{ github.workspace }}/openpilot/openpilot
+          cd ${{ github.workspace }}/openpilot
           commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
           echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
       - run: |
-          cd ${{ github.workspace }}/openpilot/openpilot
+          cd ${{ github.workspace }}/openpilot
-          if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
+          git lfs pull
-            git lfs pull -X "selfdrive/modeld/models/big_*.onnx" -X "selfdrive/modeld/models/dmonitoring_*.onnx"
-            rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
-          else
-            git lfs pull -I "selfdrive/modeld/models/big_*.onnx"
-            find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
-          fi
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v4
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
-          path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
+          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
 
   build_model:
     runs-on: [self-hosted, tici]
@@ -124,10 +103,24 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 1
           submodules: recursive
 
       - run: git lfs pull
+      - name: Cache SCons
+        uses: actions/cache@v4
+        with:
+          path: ${{env.SCONS_CACHE_DIR}}
+          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
+          restore-keys: |
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}
 
       - name: Set environment variables
         id: set-env
@@ -135,10 +128,7 @@ jobs:
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export UV_PYTHON_PREFERENCE=managed
-          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          uv sync --frozen
           printenv >> $GITHUB_ENV
           if [[ "${{ runner.debug }}" == "1" ]]; then
             cat $GITHUB_OUTPUT
@@ -158,7 +148,6 @@ jobs:
           if [[ "${{ runner.debug }}" == "1" ]]; then
             printenv
           fi
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
           rm -rf ${{ env.MODELS_DIR }}/*.onnx
 
@@ -166,78 +155,27 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
-          path: ${{ env.MODELS_DIR }}
+          path: ${{ github.workspace }}/selfdrive/modeld/models
+      - run: |
+          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |
           source /etc/profile
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
+          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}"
-          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
 
-          COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
+          # Loop through all .onnx files
-          MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
-          CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
-
-          if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
-            echo "USBGPU build"
-            export USBGPU=1
-            TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
-            OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
-          else
-            echo "QCOM build"
-            TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
-            OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
-          fi
-
-          # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
-            echo "Generating metadata: $onnx_file"
+            base_name=$(basename "$onnx_file" .onnx)
-            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            output_file="${{ env.MODELS_DIR }}/${base_name}_tinygrad.pkl"
+
+            echo "Compiling: $onnx_file -> $output_file"
+            QCOM=1 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file"
+            QCOM=1 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
-          # Detect model type and build compile args
-          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
-          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
-          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
-          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
-          SUPERCOMBO_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
-            if [ -f "$f" ]; then
-              SUPERCOMBO_ONNX="$f"
-              break
-            fi
-          done
-
-          MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
-          if [ -f "$VISION_ONNX" ]; then
-            ONNX_ARGS="--vision-onnx $VISION_ONNX"
-            if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_multi_policy
-              ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
-            elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_multi_policy
-              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
-            elif [ -f "$POLICY_ONNX" ]; then
-              MODEL_TYPE=vision_policy
-              ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
-            fi
-          elif [ -f "$SUPERCOMBO_ONNX" ]; then
-            MODEL_TYPE=supercombo
-            ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
-          fi
-
-          if [ -n "$MODEL_TYPE" ]; then
-            echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
-            env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
-              --model-type $MODEL_TYPE \
-              --model-size $MODEL_SIZE \
-              --camera-resolutions $CAMERA_RES \
-              $ONNX_ARGS \
-              --output "$OUTPUT_PKL"
-          fi
-
       - name: Prepare Output
         run: |
           sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -246,8 +184,9 @@ jobs:
           # Copy the model files
           rsync -avm \
             --include='*.dlc' \
-            --include='*.chunk*' \
+            --include='*.thneed' \
-            --include='*.chunkmanifest' \
+            --include='*.pkl' \
+            --include='*.onnx' \
             --exclude='*' \
             --delete-excluded \
             --chown=comma:comma \
@@ -279,5 +218,4 @@ jobs:
       - name: Re-enable powersave
         if: always()
         run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -4,8 +4,12 @@ env:
   BUILD_DIR: "/data/openpilot"
   OUTPUT_DIR: ${{ github.workspace }}/output
   CI_DIR: ${{ github.workspace }}/release/ci
+  SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
 
+  # Branch configurations
+  STAGING_SOURCE_BRANCH: 'master'
+
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
@@ -75,7 +79,7 @@ jobs:
             is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
             echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
 
-            stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
+            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
             echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
             echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
           fi
@@ -105,6 +109,11 @@ jobs:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
     runs-on: [self-hosted, tici]
+    outputs:
+      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
+      version: ${{ needs.prepare_strategy.outputs.version }}
+      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
+      commit_sha: ${{ github.sha }}
     if: ${{
       (always() && !cancelled() && !failure()) &&
       needs.prepare_strategy.result == 'success' &&
@@ -120,16 +129,34 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
       - run: git lfs pull
 
+      - name: Cache SCons
+        uses: actions/cache@v4
+        with:
+          path: ${{env.SCONS_CACHE_DIR}}
+          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
+          restore-keys: |
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}
+
       - name: Set environment variables
+        id: set-env
         run: |
+          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
+          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
+          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
+          echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
+
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export UV_PYTHON_PREFERENCE=managed
-          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          uv sync
           printenv >> $GITHUB_ENV
+          if [[ "${{ runner.debug }}" == "1" ]]; then
+            cat $GITHUB_OUTPUT
+          fi
 
       - name: Setup build environment
         run: |
@@ -138,56 +165,70 @@ jobs:
           echo "Starting build stage..."
           echo "BUILD_DIR: ${BUILD_DIR}"
           echo "CI_DIR: ${CI_DIR}"
-          echo "VERSION: ${{ needs.prepare_strategy.outputs.version }}"
+          echo "VERSION: ${{ steps.set-env.outputs.version }}"
           echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
           echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
           echo "-------"
           if [[ "${{ runner.debug }}" == "1" ]]; then
             printenv
           fi
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
 
       - name: Build Main Project
         run: |
-          export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
+          export PYTHONPATH="$BUILD_DIR"
-          ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" --
+          ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
-          # outside the checkout, which is wiped each run. /data/scons_cache is the device's, not ours.
-          SCONS_CACHE="$RUNNER_WORKSPACE/scons_cache"
-          mkdir -p "$SCONS_CACHE"
           cd $BUILD_DIR
-          echo "Building locationd..."
+          sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
-          # -j1: parallel rednose generators OOM the device
+          echo "Building sunnypilot's modeld..."
-          scons -j1 cache_dir="$SCONS_CACHE" --minimal \
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld
-            openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
+          echo "Building sunnypilot's modeld_v2..."
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
+          echo "Building sunnypilot's locationd..."
+          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
+          echo "Building openpilot's locationd..."
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
           echo "Building rest of sunnypilot"
-          /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
             ls -la ${BUILD_DIR}
           fi
 
-      - name: Strip release tree
+      - name: Prepare Output
         run: |
-          cd $BUILD_DIR
+          sudo rm -rf ${OUTPUT_DIR}
-          find . -name '*.a' -delete
+          mkdir -p ${OUTPUT_DIR}
-          find . -name '*.o' -delete
+          rsync -am${RUNNER_DEBUG:+v} \
-          find . -name '*.os' -delete
+            --exclude='.sconsign.dblite' \
-          find . -name '*.pyc' -delete
+            --exclude='*.a' \
-          find . -name 'moc_*' -delete
+            --exclude='*.o' \
-          find . -name '__pycache__' -type d -exec rm -rf {} +
+            --exclude='*.os' \
-          find . -name 'SConstruct' -delete
+            --exclude='*.pyc' \
-          find . -name 'SConscript' -delete
+            --exclude='moc_*' \
-          rm -rf .sconsign.dblite Jenkinsfile tools/release/ release/
+            --exclude='__pycache__' \
-          rm -f openpilot/selfdrive/modeld/models/*.onnx*
+            --exclude='Jenkinsfile' \
-          rm -f openpilot/sunnypilot/modeld*/models/*.onnx*
+            --exclude='**/release/' \
-          find openpilot/third_party/ -name '*x86*' -exec rm -r {} +
+            --exclude='**/.github/' \
-          find openpilot/third_party/ -name '*Darwin*' -exec rm -r {} +
+            --exclude='**/selfdrive/ui/replay/' \
-          cd -
+            --exclude='**/__pycache__/' \
+            --exclude='${{env.SCONS_CACHE_DIR}}' \
+            --exclude='**/.git/' \
+            --exclude='**/SConstruct' \
+            --exclude='**/SConscript' \
+            --exclude='**/.venv/' \
+            --exclude='selfdrive/modeld/models/driving_vision.onnx' \
+            --exclude='selfdrive/modeld/models/driving_policy.onnx' \
+            --exclude='sunnypilot/modeld*/models/supercombo.onnx' \
+            --exclude='third_party/*x86*' \
+            --exclude='third_party/*Darwin*' \
+            --delete-excluded \
+            --chown=comma:comma \
+            ${BUILD_DIR}/ ${OUTPUT_DIR}/
 
       - name: 'Tar.gz files'
         run: |
-          tar czf prebuilt.tar.gz -C ${{ env.BUILD_DIR }} .
+          tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} .
           ls -la prebuilt.tar.gz
 
       - name: 'Upload Artifact'
@@ -195,12 +236,10 @@ jobs:
         with:
           name: prebuilt
           path: prebuilt.tar.gz
-          compression-level: 0
 
       - name: Re-enable powersave
         if: always()
         run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
 
 
@@ -237,22 +276,29 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          echo '${{ toJSON(needs.prepare_strategy.outputs) }}'
+          echo '${{ toJSON(needs.build.outputs) }}'
           ls -la ${{ env.OUTPUT_DIR }}
 
           ${{ env.CI_DIR }}/publish.sh \
             "${{ github.workspace }}" \
             "${{ env.OUTPUT_DIR }}" \
-            "${{ needs.prepare_strategy.outputs.new_branch }}" \
+            "${{ needs.build.outputs.new_branch }}" \
-            "${{ needs.prepare_strategy.outputs.version }}" \
+            "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
+            "${{ needs.build.outputs.extra_version_identifier }}"
+
+          echo ""
+          echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
+          echo ""
+          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
+          echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
+          echo "3. Update as needed (JSON array with no spaces)"
 
       - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
         if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
         run: |
             TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
-            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
+            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
             git push -f origin ${TAG}
 
   notify:
@@ -271,6 +317,7 @@ jobs:
       - name: Prepare notification message
         id: message
         run: |
+          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
           export VERSION="${{ needs.prepare_strategy.outputs.version }}"
           export branch_name="${{ env.SOURCE_BRANCH }}"
           export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"

.github/workflows/sunnypilot-master-dev-prep.yaml

@@ -174,24 +174,6 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Restore workflows from source
-        run: |
-          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-          SOURCE_BRANCH="${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-
-          # Ensure we are on the target branch
-          git checkout $TARGET_BRANCH
-
-          echo "Restoring .github/workflows from $SOURCE_BRANCH"
-          git checkout origin/$SOURCE_BRANCH -- .github/workflows
-
-          if ! git diff --cached --quiet; then
-            echo "Workflows differ. Committing restoration."
-            git commit -m "chore: restore .github/workflows from $SOURCE_BRANCH"
-          else
-            echo "Workflows match $SOURCE_BRANCH."
-          fi
-
       - uses: actions/create-github-app-token@v2
         id: ci-token
         with:
@@ -241,3 +223,10 @@ jobs:
           gh run watch "$RUN_ID"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Trigger prebuilt workflow
+        if: success() && steps.push-changes.outputs.has_changes == 'true'
+        run: |
+          gh workflow run sunnypilot-build-prebuilt.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yaml

@@ -18,11 +18,16 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CI: 1
+  PYTHONWARNINGS: error
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: sunnypilot-base
-  GIT_CONFIG_COUNT: 1
+  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
+
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: release/ci/docker_build_sp.sh base
+
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
+  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
 
 jobs:
   build_release:
@@ -31,33 +36,34 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     env:
       STRIPPED_DIR: /tmp/releasepilot
-      PYTHONPATH: /tmp/releasepilot
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Getting LFS files
-      uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
       with:
         timeout_minutes: 2
         max_attempts: 3
         command: git lfs pull
     - name: Build devel
-      timeout-minutes: 3
+      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
+      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot and run checks
-      timeout-minutes: 30
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      working-directory: ${{ env.STRIPPED_DIR }}
+      run: |
-      run: python3 openpilot/system/manager/build.py
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "python3 system/manager/build.py"
     - name: Run tests
       timeout-minutes: 1
-      working-directory: ${{ env.STRIPPED_DIR }}
+      run: |
-      run: tools/release/check-dirty.sh
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "release/check-dirty.sh"
     - name: Check submodules
       if: github.repository == 'sunnypilot/sunnypilot'
       timeout-minutes: 3
@@ -77,20 +83,75 @@ jobs:
             export CHECK_PR_REFS=true
           fi
         fi
-        tools/release/check-submodules.sh
+        release/check-submodules.sh
+
+  build:
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup docker push
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'sunnypilot/sunnypilot'
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
+    - uses: ./.github/workflows/setup-with-retry
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: 30
 
   build_mac:
     name: build macOS
     runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
+    if: false  # There'll be one day that this works. That day is not today.
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
-    - name: Remove Homebrew from environment
+      with:
-      run: |
+        submodules: true
-        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
+    - name: Homebrew cache
-    - run: ./tools/op.sh setup
+      uses: ./.github/workflows/auto-cache
+      with:
+        save: false # No need save here if we manually save it later conditionally
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ hashFiles('tools/Brewfile') }}-${{ github.sha }}
+        restore-keys: |
+          brew-macos-${{ hashFiles('tools/Brewfile') }}
+          brew-macos-
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+      env:
+        PYTHONWARNINGS: default  # package install has DeprecationWarnings
+        HOMEBREW_DISPLAY_INSTALL_TIMES: 1
+    - name: Save Homebrew cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: brew-macos-${{ hashFiles('tools/Brewfile') }}-${{ github.sha }}
+    - run: git lfs pull
+    - name: Getting scons cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        save: false # No need save here if we manually save it later conditionally
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}-macos
     - name: Building openpilot
-      run: scons
+      run: . .venv/bin/activate && scons -j$(nproc)
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   static_analysis:
     name: static analysis
@@ -98,14 +159,18 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
+    env:
+      PYTHONWARNINGS: default
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
-    - run: ./tools/op.sh setup
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
     - name: Static analysis
       timeout-minutes: 1
-      run: scripts/lint/lint.sh
+      run: ${{ env.RUN }} "scripts/lint/lint.sh"
 
   unit_tests:
     name: unit tests
@@ -113,85 +178,74 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
-    - run: ./tools/op.sh setup
+      with:
-    - name: Install raylib headless dependencies
+        submodules: true
-      if: ${{ !contains(runner.name, 'nsc') }}
+    - uses: ./.github/workflows/setup-with-retry
-      run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
+      id: setup-step
     - name: Build openpilot
-      run: scons
+      run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 999 }}
-      env:
+      run: |
-        RAYLIB_BACKEND: headless
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-      run: tools/op.sh test
+                        # Pre-compile Python bytecode so each pytest worker doesn't need to
+                        $PYTEST --collect-only -m 'not slow' -qq && \
+                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
+                        chmod -R 777 /tmp/comma_download_cache"
 
   process_replay:
     name: process replay
+    if: false  # disable process_replay for forks
     runs-on: ${{
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
-    - run: ./tools/op.sh setup
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+      id: setup-step
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
     - name: Build openpilot
-      run: scons
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
-      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
+      run: |
-      run: openpilot/selfdrive/test/process_replay/test_processes.py -j$(nproc)
+        ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache"
     - name: Print diff
       id: print-diff
       if: always()
-      run: cat openpilot/selfdrive/test/process_replay/diff.txt
+      run: cat selfdrive/test/process_replay/diff.txt
-    - name: Print diff report
+    - uses: actions/upload-artifact@v4
-      if: always()
-      run: cat openpilot/selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v7
       if: always()
       continue-on-error: true
       with:
         name: process_replay_diff.txt
-        path: openpilot/selfdrive/test/process_replay/diff.txt
+        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload diff report
+    - name: Upload reference logs
-      uses: actions/upload-artifact@v7
+      if: false # TODO: move this to github instead of azure
-      if: always() && github.event_name == 'pull_request'
-      continue-on-error: true
-      with:
-        name: diff_report_${{ github.event.number }}
-        path: openpilot/selfdrive/test/process_replay/diff_report.txt
-    - name: Checkout ci-artifacts
-      if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v7
-      with:
-        repository: sunnypilot/ci-artifacts
-        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-        path: ${{ github.workspace }}/ci-artifacts
-    - name: Prepare refs
-      if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
-      working-directory: ${{ github.workspace }}/ci-artifacts
       run: |
-        git config user.name "GitHub Actions Bot"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-        git config user.email "<>"
+    - name: Run regen
-        git fetch origin process-replay || true
+      if: false
-        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
+      timeout-minutes: 4
-        cp ${{ github.workspace }}/openpilot/selfdrive/test/process_replay/fakedata/*.zst .
+      run: |
-        echo "${{ github.sha }}" > ref_commit
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-        git add .
+                        chmod -R 777 /tmp/comma_download_cache"
-        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
-    - name: Push refs
-      if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
-      uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force
 
   simulator_driving:
     name: simulator driving
@@ -199,45 +253,73 @@ jobs:
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     if: false  # FIXME: Started to timeout recently
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v4
-    - run: ./tools/op.sh setup
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+      id: setup-step
     - name: Build openpilot
-      run: scons
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
     - name: Driving test
-      timeout-minutes: 2
+      timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
-      env:
+      run: |
-        # MetaDrive renders offscreen through panda3d's EGL pipe on llvmpipe
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-        EGL_PLATFORM: surfaceless
+                        source selfdrive/test/setup_vsound.sh && \
-      run: python openpilot/tools/sim/tests/test_metadrive_bridge.py
+                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
 
-  create_ui_report:
+  create_raylib_ui_report:
-    name: Create UI Report
+    name: Create raylib UI Report
     runs-on: ${{
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
       (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v4
-      - run: ./tools/op.sh setup
-      - name: Install raylib headless dependencies
-        if: ${{ !contains(runner.name, 'nsc') }}
-        run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
-      - name: Build openpilot
-        run: scons
-      - name: Create UI Report
-        env:
-          RAYLIB_BACKEND: headless
-        run: |
-          python3 openpilot/selfdrive/ui/tests/diff/replay.py
-          python3 openpilot/selfdrive/ui/tests/diff/replay.py --big
-      - name: Upload UI Report
-        uses: actions/upload-artifact@v7
         with:
-          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          submodules: true
-          path: openpilot/selfdrive/ui/tests/diff/report
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create raylib UI Report
+        run: >
+          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                          source selfdrive/test/setup_xvfb.sh &&
+                          python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py"
+      - name: Upload Raylib UI Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          path: selfdrive/ui/tests/test_ui/raylib_report/screenshots
+
+  create_mici_raylib_ui_report:
+    name: Create mici raylib UI Report
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create mici raylib UI Report
+        run: >
+          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                          source selfdrive/test/setup_xvfb.sh &&
+                          WINDOWED=1 python3 selfdrive/ui/tests/diff/replay.py"
+      - name: Upload Raylib UI Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: mici-raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -1,176 +0,0 @@
-name: "ui preview"
-on:
-  push:
-    branches:
-      - master
-  pull_request_target:
-    types: [assigned, opened, synchronize, reopened, edited]
-    branches:
-      - 'master'
-    paths:
-      - 'openpilot/selfdrive/assets/**'
-      - 'openpilot/selfdrive/ui/**'
-      - 'openpilot/system/ui/**'
-  workflow_dispatch:
-
-env:
-  UI_JOB_NAME: "Create UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
-  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
-
-  # variant:video_prefix:master_branch
-  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
-
-jobs:
-  preview:
-    if: false # tmp disable due to GH API rate limiting flakiness
-    #if: github.repository == 'sunnypilot/sunnypilot'
-    name: preview
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          submodules: true
-
-      - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
-        with:
-          ref: ${{ env.SHA }}
-          check-name: ${{ env.UI_JOB_NAME }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allowed-conclusions: success
-          wait-interval: 20
-
-      - name: Getting workflow run ID
-        id: get_run_id
-        run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
-
-      - name: Getting proposed ui
-        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ steps.get_run_id.outputs.run_id }}
-          search_artifacts: true
-          name: ui-report-1-${{ env.REPORT_NAME }}
-          path: ${{ github.workspace }}/pr_ui
-
-      - name: Getting mici master ui
-        uses: actions/checkout@v7
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_mici
-          ref: openpilot_master_ui_mici_raylib
-
-      - name: Getting big master ui
-        uses: actions/checkout@v7
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_big
-          ref: openpilot_master_ui_big_raylib
-
-      - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video branch <<< "$variant"
-            master_dir="${{ github.workspace }}/master_${name}"
-            cd "$master_dir"
-            git checkout --orphan=new_branch
-            git rm -rf *
-            git branch -D "$branch"
-            git branch -m "$branch"
-            git config user.name "GitHub Actions Bot"
-            git config user.email "<>"
-            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
-            git add .
-            git commit -m "${name} video for commit ${{ env.SHA }}"
-            git push origin "$branch" --force
-          done
-
-      - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@178e0b4a408f06ce40d896c3cd79f50fa6f8b0c3
-
-      - name: Finding diffs
-        if: github.event_name == 'pull_request_target'
-        id: find_diff
-        run: |
-          export PYTHONPATH=${{ github.workspace }}
-          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
-
-          COMMENT=""
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video _ <<< "$variant"
-            diff_name="${name}_diff"
-
-            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
-            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
-
-            diff_exit_code=0
-            python3 ${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/diff.py \
-              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
-              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
-              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
-
-            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
-
-            REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
-            if [ $diff_exit_code -eq 0 ]; then
-              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
-            else
-              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
-            fi
-          done
-
-          {
-            echo "COMMENT<<EOF"
-            echo "$COMMENT"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Saving proposed ui
-        if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_mici
-        run: |
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          git checkout --orphan=${{ env.BRANCH_NAME }}
-          git rm -rf *
-          mv ${{ github.workspace }}/pr_ui/* .
-          git add .
-          git commit -m "ui videos for PR #${{ github.event.number }}"
-          git push origin ${{ env.BRANCH_NAME }} --force
-
-          # Append diff reports to report files branch
-          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
-          for variant in $VARIANTS; do
-            IFS=':' read -r name _ _ <<< "$variant"
-            diff_name="${name}_diff"
-            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
-            git add "${diff_name}_pr_${{ github.event.number }}.html"
-          done
-          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
-          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-
-      - name: Comment on PR
-        if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
-        with:
-          message: |
-            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
-            ## UI Preview
-            ${{ steps.find_diff.outputs.COMMENT }}
-          comment-tag: run_id_ui_preview
-          pr-number: ${{ github.event.number }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -13,20 +13,13 @@ venv/
 a.out
 .hypothesis
 .cache/
-bin/
 
-# created at launch for comma hardware PYTHONPATH (PC uses editable installs via pyproject.toml)
+/docs_site/
-/msgq
-/opendbc
-/rednose
-/teleoprtc
-/tinygrad
 
 *.mp4
 *.dylib
 *.DSYM
 *.d
-*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -39,8 +32,6 @@ bin/
 *.os-*
 *.so
 *.a
-st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
-*.unchunked
 *.clb
 *.class
 *.pyxbldc
@@ -48,57 +39,65 @@ st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
 *.mo
 *_pyx.cpp
 *.stats
-*.pkl
-*.pkl*
 config.json
+clcache
 compile_commands.json
 compare_runtime*.html
-openpilot/selfdrive/modeld/models/tg_input_devices.json
 
-# build artifacts
+selfdrive/pandad/pandad
-docs/_site/
+cereal/services.h
-openpilot/selfdrive/pandad/pandad
+cereal/gen
-openpilot/cereal/services.h
+cereal/messaging/bridge
-openpilot/cereal/gen
+selfdrive/ui/translations/tmp
-openpilot/cereal/messaging/bridge
+selfdrive/car/tests/cars_dump
-openpilot/selfdrive/ui/translations/tmp
+system/camerad/camerad
-openpilot/selfdrive/car/tests/cars_dump
+system/camerad/test/ae_gray_test
-openpilot/system/camerad/camerad
-openpilot/system/camerad/test/ae_gray_test
 
 .coverage*
 coverage.xml
 htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/models/*.pkl
+sunnypilot/modeld*/thneed/compile
+sunnypilot/modeld*/models/*.thneed
+sunnypilot/modeld*/models/*.pkl
 
 # openpilot log files
 *.bz2
 *.zst
-*.rlog
 
 build/
 
 !**/.gitkeep
 
+poetry.toml
+Pipfile
 
 ### VisualStudioCode ###
-*.vsix
-.history
-.ionide
 .vscode/*
-.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# agents
+# Local History for Visual Studio Code
-.claude/
+.history/
-.context/
+
-PLAN.md
+# Built Visual Studio Code Extensions
-TASK.md
+*.vsix
-CLAUDE.md
+
-SKILL.md
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
 
 ### JetBrains ###
 !.idea/customTargets.xml

.gitmodules

@@ -6,7 +6,7 @@
   url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/sunnypilot/msgq.git
+  url = https://github.com/commaai/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git
@@ -15,7 +15,7 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/sunnypilot/tinygrad.git
+  url = https://github.com/commaai/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
-	path = openpilot/sunnypilot/neural_network_data
+  path = sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.python-version

@@ -1 +0,0 @@
-3.12.13

.vscode/launch.json

@@ -6,9 +6,9 @@
       "type": "pickString",
       "description": "Select the process to debug",
       "options": [
-        "openpilot/selfdrive/controls/controlsd.py",
+        "selfdrive/controls/controlsd.py",
-        "openpilot/system/timed/timed.py",
+        "system/timed/timed.py",
-        "openpilot/tools/sim/run_bridge.py"
+        "tools/sim/run_bridge.py"
       ]
     },
     {
@@ -16,7 +16,7 @@
       "type": "pickString",
       "description": "Select the process to debug",
       "options": [
-        "openpilot/selfdrive/ui/ui"
+        "selfdrive/ui/ui"
       ]
     },
     {
@@ -52,9 +52,6 @@
       "type": "lldb",
       "request": "attach",
       "pid": "${command:pickMyProcess}",
-      "sourceMap": {
-        ".": "${workspaceFolder}/opendbc/safety"
-      },
       "initCommands": [
         "script import time; time.sleep(3)"
       ]

.vscode/settings.json

@@ -17,5 +17,11 @@
     "**/.git",
     "**/.venv",
     "**/__pycache__",
+    // exclude directories that should be using the symlinked version
+    "common/**",
+    "selfdrive/**",
+    "system/**",
+    "third_party/**",
+    "tools/**",
   ]
 }

CHANGELOG.md

@@ -1,221 +1,5 @@
-sunnypilot Version 2026.002.000 (2026-06-28)
+sunnypilot Version 2025.003.000 (20xx-xx-xx)
 ========================
-* What's Changed (sunnypilot/sunnypilot)
-  * ui: update gates for certain toggles by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1830
-  * release: ignore upstream IsReleaseBranch by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1831
-  * manager: disable DEVELOPMENT_ONLY reset by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1833
-  * sunnylink: fix max time offroad values by @nayan8teen in https://github.com/sunnypilot/sunnypilot/pull/1835
-  * ui: show default model name by @nayan8teen in https://github.com/sunnypilot/sunnypilot/pull/1837
-  * sunnylink: add CarParams fallback for brand-specific capabilities by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1839
-  * sunnylink SDUI: tweak DisableUpdate param for clarity by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1842
-  * Revert "DM: Lancia Delta HF Integrale model" by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1849
-  * modeld_v2: safe model validation by @Discountchubbs in https://github.com/sunnypilot/sunnypilot/pull/1855
-  * Revert "deprecate `carState.brake`" for Honda Gas Interceptor by @mvl-boston in https://github.com/sunnypilot/sunnypilot/pull/1860
-  * sunnylink: deprecate legacy params metadata by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1862
-  * ui: reset Enforce Torque Control and NNLC if both are enabled by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1863
-* What's Changed (sunnypilot/opendbc)
-  * Rivian: suppress ACM hold-the-wheel warning during MADS-only lateral by @lukasloetkolben in https://github.com/sunnypilot/opendbc/pull/465
-  * Sync: `commaai/opendbc:master` → `sunnypilot/opendbc:master` by @sunnyhaibin in https://github.com/sunnypilot/opendbc/pull/479
-  * safety: add option to ignore frequency check for RX checks by @sunnyhaibin in https://github.com/sunnypilot/opendbc/pull/480
-  * Revert "deprecate carState.brake" for Honda Gas Interceptor by @mvl-boston in https://github.com/sunnypilot/opendbc/pull/481
-* New Contributors (sunnypilot/sunnypilot)
-  * @mvl-boston made their first contribution in https://github.com/sunnypilot/sunnypilot/pull/1860
-* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2026.001.007...v2026.002.000
-************************
-* Synced with commaai's openpilot (v0.11.1)
-  * master commit 69e2c321e49760e52f7983eaa0a5f77cb95de637 (June 02, 2026)
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Improved thermal policy for comma four
-* Acura MDX 2022-24 support thanks to mvl-boston!
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-sunnypilot Version 2026.001.000 (2026-05-06)
-========================
-* What's Changed (sunnypilot/sunnypilot)
-  * Complete rewrite of the user interface from Qt C++ to Raylib Python
-  * comma four support
-  * ui: sunnypilot toggle style by @nayan8teen
-  * ui: fix scroll panel mouse wheel behavior by @nayan8teen
-  * ui: sunnypilot panels by @nayan8teen
-  * sunnylink: centralize key pair handling in sunnylink registration by @devtekve
-  * ui: reimplement sunnypilot branding with Raylib by @sunnyhaibin
-  * ui: Platform Selector by @Discountchubbs
-  * ui: vehicle brand settings by @Discountchubbs
-  * ui: sunnylink client-side implementation by @nayan8teen
-  * ui: `NetworkUISP` by @Discountchubbs
-  * ui: add sunnypilot font by @nayan8teen
-  * ui: sunnypilot sponsor tier color mapping by @sunnyhaibin
-  * ui: sunnylink panel by @nayan8teen
-  * ui: Models panel by @Discountchubbs
-  * ui: software panel by @Discountchubbs
-  * modeld_v2: support planplus outputs by @Discountchubbs
-  * ui: OSM panel by @Discountchubbs
-  * ui: Developer panel extension by @Discountchubbs
-  * sunnylink: Vehicle Selector support by @sunnyhaibin
-  * [TIZI/TICI] ui: Developer Metrics by @rav4kumar
-  * [comma 4] ui: sunnylink panel by @nayan8teen
-  * ui: lateral-only and longitudinal-only UI statuses support by @royjr
-  * sunnylink: elliptic curve keys support and improve key path handling by @nayan8teen
-  * sunnylink: block remote modification of SSH key parameters by @zikeji
-  * [TIZI/TICI] ui: rainbow path by @rav4kumar
-  * [TIZI/TICI] ui: chevron metrics by @rav4kumar
-  * ui: include MADS enabled state to `engaged` check by @sunnyhaibin
-  * Toyota: Enforce Factory Longitudinal Control by @sunnyhaibin
-  * ui: fix malformed dongle ID display on the PC if dongleID is not set by @dzid26
-  * SL: Re enable and validate ingestion of swaglogs by @devtekve
-  * modeld_v2: planplus model tuning by @Discountchubbs
-  * ui: fix Always Offroad button visibility by @nayan8teen
-  * Reimplement sunnypilot Terms of Service & sunnylink Consent Screens by @sunnyhaibin
-  * [TIZI/TICI] ui: update dmoji position and Developer UI adjustments by @rav4kumar
-  * modeld: configurable camera offset by @Discountchubbs
-  * [TIZI/TICI] ui: sunnylink status on sidebar by @Copilot
-  * ui: Global Brightness Override by @nayan8teen
-  * ui: Customizable Interactive Timeout by @sunnyhaibin
-  * sunnylink: add units to param metadata by @nayan8teen
-  * ui: Customizable Onroad Brightness by @sunnyhaibin
-  * [TIZI/TICI] ui: Steering panel by @nayan8teen
-  * [TIZI/TICI] ui: Rocket Fuel by @rav4kumar
-  * [TIZI/TICI] ui: MICI style turn signals by @rav4kumar
-  * [TIZI/TICI] ui: MICI style blindspot indicators by @sunnyhaibin
-  * [MICI] ui: display blindspot indicators when available by @rav4kumar
-  * [TIZI/TICI] ui: Road Name by @rav4kumar
-  * [TIZI/TICI] ui: Blue "Exit Always Offroad" button by @dzid26
-  * [TIZI/TICI] ui: Speed Limit by @rav4kumar
-  * Reapply "latcontrol_torque: lower kp and lower friction threshold (commaai/openpilot#36619)" by @sunnyhaibin
-  * [TIZI/TICI] ui: steering arc by @royjr
-  * [TIZI/TICI] ui: Smart Cruise Control elements by @sunnyhaibin
-  * [TIZI/TICI] ui: Green Light and Lead Departure elements by @sunnyhaibin
-  * [TIZI/TICI] ui: standstill timer by @sunnyhaibin
-  * [MICI] ui: driving models selector by @Discountchubbs
-  * [TIZI/TICI] ui: Hide vEgo and True vEgo by @sunnyhaibin
-  * [TIZI/TICI] ui: Visuals panel by @nayan8teen
-  * Device: Retain QuickBoot state after op switch by @nayan8teen
-  * [TIZI/TICI] ui: Trips panel by @sunnyhaibin
-  * [TIZI/TICI] ui: dynamic ICBM status by @sunnyhaibin
-  * [TIZI/TICI] ui: Cruise panel by @sunnyhaibin
-  * ui: better wake mode support by @nayan8teen
-  * Pause Lateral Control with Blinker: Post-Blinker Delay by @CHaucke89
-  * SCC-V: Use p97 for predicted lateral accel by @yasu-oh
-  * Controls: Support for Torque Lateral Control v0 Tune by @sunnyhaibin
-  * [TIZI/TICI] ui: ensure null checks for `CarParams` and `CarParamsSP` by @sunnyhaibin
-  * [TIZI/TICI] ui: use `vCruiseCluster` and `vEgoCluster` for SLA `preActive` by @sunnyhaibin
-  * Fix display of values when using use_float_scaling by @CHaucke89
-  * models: fix default & index "0" by @nayan8teen
-  * [TIZI/TICI] visuals: Improved speed limit by @angaz
-  * ICBM: ensure button timers update on disable to clear stale presses by @jamesmikesell
-  * [TIZI/TICI] ui: simplify Smart Cruise Control text rendering by @sunnyhaibin
-  * controlsd: fix steer_limited_by_safety not updating under MADS by @zephleggett
-  * soundd: trigger timeout warning during MADS lateral-only by @zephleggett
-  * pandad: flasher for Rivian long upgrade module by @lukasloetkolben
-  * modeld_v2: tinygrad transformation warp by @Discountchubbs
-  * tools: block `manage_sunnylinkd` in sim startup script by @sunnyhaibin
-  * [MICI] ui: need superclass `_render` in `HudRendererSP` by @sunnyhaibin
-  * [TIZI/TICI] ui: Speed Limit Assist active status by @sunnyhaibin
-  * ui: reimplement "Screen Off" option to Onroad Brightness by @sunnyhaibin
-  * ui: don't hide steering wheel when blindspot disabled by @royjr
-  * ui: Speed Limit Assist `preActive` improvements  by @sunnyhaibin
-  * ui: consolidate Speed Limit Assist `preActive` status rendering by @sunnyhaibin
-  * [MICI] ui: Speed Limit Assist `preActive` status by @sunnyhaibin
-  * sunnypilot modeld: remove thneed modeld by @Discountchubbs
-  * modeld_v2: decouple planplus scaling from accel by @Discountchubbs
-  * sunnylink: Handle exceptions in `getParamsAllKeysV1` to log crashes by @devtekve
-  * [TIZI/TICI] ui: Developer UI cleanup by @sunnyhaibin
-  * [TIZI/TICI] ui: dynamic alert size by @nayan8teen
-  * i18n(fr): Add French translations by @didlawowo
-  * Toyota: Stop and Go Hack (Alpha) by @sunnyhaibin
-  * ui: `AlertFadeAnimator` for longitudinal-related statuses by @sunnyhaibin
-  * pandad: gate unsupported pandas before flashing by @sunnyhaibin
-  * Rivian: Flash xnor's Longitudinal Upgrade Kit prior supported panda check by @lukasloetkolben
-  * [TIZI/TICI] ui: add back gate steering arc behind toggle by @sunnyhaibin
-  * ui: gate Onroad Brightness Delay on readiness by @sunnyhaibin
-  * ui: add new timer options for Onroad Brightness Delay by @sunnyhaibin
-  * [TIZI/TICI] ui: branch switcher is always available by @sunnyhaibin
-  * pandad: always prioritize internal panda by @sunnyhaibin
-  * sunnylinkd: fetch compressed params schema by @sunnyhaibin
-  * sunnypilot locationd: remove unused car_ekf filter by @sunnyhaibin
-  * modeld_v2: update deprecated temporalPose ref by @sunnyhaibin
-  * NNLC: restore pre-v1 PID gains in torque extension by @mmmorks
-  * MADS safety: enable heartbeat and lateral controls mismatch checks by @sunnyhaibin
-  * [MICI] ui: models panel enhancements by @nayan8teen
-  * [TIZI/TICI] ui: fix unintended selection while scrolling in TreeOptionDialog by @TheSecurityDev
-  * tools: script for video concatenation by @Discountchubbs
-  * tools: profile memory usage by @Discountchubbs
-  * [TIZI/TICI] ui: remove per-frame param sync by @sunnyhaibin
-  * [MICI] ui: always offroad by @nayan8teen
-  * controls: always default Torque Lateral Control to v0 Tune by @sunnyhaibin
-  * Revert "controls: always default Torque Lateral Control to v0 Tune" by @sunnyhaibin
-  * Reapply "controls: always default Torque Lateral Control to v0 Tune" (#1806) by @sunnyhaibin
-  * [MICI] ui: add sunnylink info & connectivity check by @nayan8teen
-  * sunnylink: Remove unused API endpoint by @devtekve
-  * DM: wheel touch enforcement in MADS by @sunnyhaibin
-  * torque: show static override values in Dev UI & gate `useParams` on custom torque tune by @sunnyhaibin
-  * MADS: suppress espActive event when long is not engaged by @sunnyhaibin
-  * sunnylink: SDUI by @sunnyhaibin
-  * [MICI] ui: align upstream changes with sunnypilot settings buttons by @nayan8teen
-  * ui: fix cellular toggles by @AmyJeanes
-  * sunnylink: switch athena domain by @devtekve
-  * Platform List: dynamically migrate CarPlatformBundle by @sunnyhaibin
-* What's Changed (sunnypilot/opendbc)
-  * Honda: DBC for Accord 9th Generation by @mvl-boston
-  * FCA: update tire stiffness values for `RAM_HD` by @dparring
-  * Honda: Nidec hybrid baseline brake support by @mvl-boston
-  * Subaru Global Gen2: bump steering limits and update tuning by @sunnyhaibin
-  * Toyota: Enforce Stock Longitudinal Control by @rav4kumar
-  * Nissan: use MADS enabled status for LKAS HUD logic by @downquark7
-  * Reapply "Lateral: lower friction threshold (#2915)" (#378) by @sunnyhaibin
-  * HKG: add KIA_FORTE_2019_NON_SCC fingerprint by @royjr
-  * Nissan: Parse cruise control buttons by @downquark7
-  * Rivian: Add stalk down ACC behavior to match stock Rivian by @lukasloetkolben
-  * Tesla: remove `TESLA_MODEL_X` from `dashcamOnly` by @ssysm
-  * Hyundai Longitudinal: refactor tuning by @Discountchubbs
-  * Tesla: add fingerprint for Model 3 Performance HW4 by @sunnyhaibin
-  * Toyota: do not disable radar when smartDSU or CAN Filter detected by @sunnyhaibin
-  * Honda: add missing `GasInterceptor` messages to Taiwan Odyssey DBC by @mvl-boston
-  * GM: remove `CHEVROLET_EQUINOX_NON_ACC_3RD_GEN` from `dashcamOnly` by @sunnyhaibin
-  * GM: remove `CHEVROLET_BOLT_NON_ACC_2ND_GEN` from `dashcamOnly` by @sunnyhaibin
-  * Hyundai Longitudinal: deprecate ramp update for dynamic tune by @Discountchubbs
-  * Rivian: long upgrade messages on bus 1 by @lukasloetkolben
-  * Toyota: Stop and Go Hack (Alpha) by @sunnyhaibin
-  * Toyota: gate Smart DSU behind Alpha Longitudinal by @sunnyhaibin
-  * Toyota: Gas Interceptor always set `standstill_req` by @sunnyhaibin
-  * MADS safety: dedicated `controls_allowed_lateral` by @sunnyhaibin
-  * Platform List: include community supported platforms by @sunnyhaibin
-* New Contributors (sunnypilot/sunnypilot)
-  * @TheSecurityDev made their first contribution in "ui: fix sidebar scroll in UI screenshots"
-  * @zikeji made their first contribution in "sunnylink: block remote modification of SSH key parameters"
-  * @Candy0707 made their first contribution in "[TIZI/TICI] ui: Fix misaligned turn signals and blindspot indicators with sidebar"
-  * @CHaucke89 made their first contribution in "Pause Lateral Control with Blinker: Post-Blinker Delay"
-  * @yasu-oh made their first contribution in "SCC-V: Use p97 for predicted lateral accel"
-  * @angaz made their first contribution in "[TIZI/TICI] visuals: Improved speed limit"
-  * @jamesmikesell made their first contribution in "ICBM: ensure button timers update on disable to clear stale presses"
-  * @zephleggett made their first contribution in "controlsd: fix steer_limited_by_safety not updating under MADS"
-  * @lukasloetkolben made their first contribution in "pandad: flasher for Rivian long upgrade module"
-  * @didlawowo made their first contribution in "i18n(fr): Add French translations"
-  * @mmmorks made their first contribution in "NNLC: restore pre-v1 PID gains in torque extension"
-* New Contributors (sunnypilot/opendbc)
-  * @AmyJeanes made their first contribution in "Tesla: Fix stock LKAS being blocked when MADS is enabled"
-  * @mvl-boston made their first contribution in "Honda: Update Clarity brake to renamed DBC message name"
-  * @dzid26 made their first contribution in "Tesla: Parse speed limit from CAN"
-  * @firestar5683 made their first contribution in "GM: Non-ACC platforms with steering only support"
-  * @downquark7 made their first contribution in "Nissan: use MADS enabled status for LKAS HUD logic"
-  * @royjr made their first contribution in "HKG: add KIA_FORTE_2019_NON_SCC fingerprint"
-  * @ssysm made their first contribution in "Tesla: remove `TESLA_MODEL_X` from `dashcamOnly`"
-* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2025.002.000...v2026.001.000
-************************
-* Synced with commaai's openpilot (v0.11.1)
-  * master commit c001f3c9b490a80e69539f0af6022f6e07ceb721 (April 16, 2026)
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-* Improved inter-process communication memory efficiency
-* comma four support
 
 sunnypilot Version 2025.002.000 (2025-11-06)
 ========================

Dockerfile.openpilot

@@ -0,0 +1,14 @@
+FROM ghcr.io/commaai/openpilot-base:latest
+
+ENV PYTHONUNBUFFERED=1
+
+ENV OPENPILOT_PATH=/home/batman/openpilot
+
+RUN mkdir -p ${OPENPILOT_PATH}
+WORKDIR ${OPENPILOT_PATH}
+
+COPY . ${OPENPILOT_PATH}/
+
+ENV UV_BIN="/home/batman/.local/bin/"
+ENV PATH="$UV_BIN:$PATH"
+RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc)

Dockerfile.openpilot_base

@@ -0,0 +1,82 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED=1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX=1
+
+RUN dbus-uuidgen > /etc/machine-id
+RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /home/$USER
+COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /home/$USER && \
+    tools/install_python_dependencies.sh && \
+    rm -rf tools/ pyproject.toml uv.lock .cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Dockerfile.sunnypilot

@@ -0,0 +1,12 @@
+FROM ghcr.io/sunnypilot/sunnypilot-base:latest
+
+ENV PYTHONUNBUFFERED=1
+
+ENV OPENPILOT_PATH=/home/batman/openpilot
+
+RUN mkdir -p ${OPENPILOT_PATH}
+WORKDIR ${OPENPILOT_PATH}
+
+COPY . ${OPENPILOT_PATH}/
+
+RUN scons --cache-readonly -j$(nproc)

Dockerfile.sunnypilot_base

@@ -0,0 +1,83 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED=1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+    rsync \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX=1
+
+RUN dbus-uuidgen > /etc/machine-id
+RUN apt-get update && apt-get install -y fonts-noto-cjk fonts-noto-color-emoji
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /home/$USER
+COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /home/$USER && \
+    tools/install_python_dependencies.sh && \
+    rm -rf tools/ pyproject.toml uv.lock .cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -12,7 +12,7 @@ def retryWithDelay(int maxRetries, int delay, Closure body) {
 def device(String ip, String step_label, String cmd) {
   withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
     def ssh_cmd = """
-ssh -o ControlMaster=auto -o ControlPath=/tmp/ssh_control_%C -o ControlPersist=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
+ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
 
 set -e
 
@@ -22,8 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-export COMMA_CACHE=/data/tmp/comma_download_cache
+export LOGPRINT=debug
-#export LOGPRINT=debug # this has gotten too spammy...
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -31,14 +30,14 @@ export GIT_COMMIT=${env.GIT_COMMIT}
 export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
 export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
-# only use 1 thread since most require real hardware that can't be shared
+# only use 1 thread for tici tests since most require HIL
 export PYTEST_ADDOPTS="-n0 -s"
 
 
 export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
 source ~/.bash_profile
-if [ -f /AGNOS ]; then
+if [ -f /TICI ]; then
   source /etc/profile
 
   rm -rf /tmp/tmp*
@@ -64,8 +63,6 @@ if [ -f /data/openpilot/launch_env.sh ]; then
   source /data/openpilot/launch_env.sh
 fi
 
-export LD_LIBRARY_PATH="\$(python -c 'import ffmpeg; print(ffmpeg.LIB_DIR)'):/usr/local/lib:\${LD_LIBRARY_PATH:-}"
-
 ln -snf ${env.TEST_DIR} /data/pythonpath
 
 cd ${env.TEST_DIR} || true
@@ -96,7 +93,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
           retry (3) {
             def date = sh(script: 'date', returnStdout: true).trim();
             device(device_ip, "set time", "date -s '" + date + "'")
-            device(device_ip, "git checkout", extra + "\n" + readFile("openpilot/selfdrive/test/setup_device_ci.sh"))
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }
           steps.each { item ->
             def name = item[0]
@@ -169,8 +166,8 @@ node {
   env.GIT_BRANCH = checkout(scm).GIT_BRANCH
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
-  def excludeBranches = ['__nightly', 'devel', 'devel-staging',
+  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tizi', 'release-tizi-staging', 'release-mici', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
+                         'release-tici', 'release-tizi', 'release-tizi-staging', 'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
   if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
@@ -182,7 +179,7 @@ node {
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/tools/release/build_release.sh"),
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
@@ -190,12 +187,12 @@ node {
       parallel (
         'nightly': {
           deviceStage("build nightly", "tizi-needs-can", [], [
-            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/tools/release/build_release.sh"),
+            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
           deviceStage("build nightly-dev", "tizi-needs-can", [], [
-            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/tools/release/build_release.sh"),
+            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
       )
@@ -205,51 +202,61 @@ node {
     parallel (
       'onroad tests': {
         deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
-          step("build openpilot", "cd openpilot/system/manager && ./build.py"),
+          step("build openpilot", "cd system/manager && ./build.py"),
-          step("check dirty", "tools/release/check-dirty.sh"),
+          step("check dirty", "release/check-dirty.sh"),
-          step("onroad tests", "./openpilot/selfdrive/test/test_onroad.py", [timeout: 60]),
+          step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test power draw", "./openpilot/selfdrive/test/test_power_draw.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
-          step("test encoder", "./openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
+          step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
-          step("test manager", "./openpilot/system/manager/test/test_manager.py"),
+          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
+          step("test manager", "pytest system/manager/test/test_manager.py"),
+        ])
+      },
+      'loopback': {
+        deviceStage("loopback", "tizi-loopback", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
         ])
       },
       'camerad OX03C10': {
         deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'sensord': {
         deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test sensord", "./openpilot/system/sensord/tests/test_sensord.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
         deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
+          step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
-          step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
+          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
         ])
       },
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
-          step("build openpilot", "cd openpilot/system/manager && ./build.py"),
+          step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "./openpilot/selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
-          step("test pandad spi", "./openpilot/selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test amp", "./openpilot/common/hardware/comma/tests/test_amplifier.py"),
+          step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          // TODO: enable once new AGNOS is available
+          // step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),
+          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
         ])
       },
 

README.md

@@ -11,10 +11,66 @@ Join the official sunnypilot community forum to stay up to date with all the lat
 https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
 
 ## 🚘 Running on a dedicated device in a car
-First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
+* A supported device to run this software
+    * a [comma three](https://comma.ai/shop/products/three) or a [C3X](https://comma.ai/shop/comma-3x)
+* This software
+* One of [the 325+ supported cars](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford, and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
+* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
+
+Detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
 
 ## Installation
-Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
+Please refer to [Recommended Branches](#recommended-branches) to find your preferred/supported branch. This guide will assume you want to install the latest `staging` branch.
+
+### If you want to use our newest branches (our rewrite)
+> [!TIP]
+>You can see the rewrite state on our [rewrite project board](https://github.com/orgs/sunnypilot/projects/2), and to install the new branches, you can use the following links
+
+* sunnypilot not installed or you installed a version before 0.8.17?
+  1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
+  2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
+  3. Input the installation URL per [Recommended Branches](#recommended-branches). Example: ```https://staging.sunnypilot.ai```.
+  4. Complete the rest of the installation following the onscreen instructions.
+
+* sunnypilot already installed and you installed a version after 0.8.17?
+  1. On the comma three/3X, go to `Settings` ▶️ `Software`.
+  2. At the `Download` option, press `CHECK`. This will fetch the list of latest branches from sunnypilot.
+  3. At the `Target Branch` option, press `SELECT` to open the Target Branch selector.
+  4. Scroll to select the desired branch per  Recommended Branches (see below). Example: `staging`
+
+### Recommended Branches
+|      Branch     |                 Installation URL              |
+|:---------------:|:---------------------------------------------:|
+| `release`       | `https://release.sunnypilot.ai`               |
+| `staging`       | `https://staging.sunnypilot.ai`               |
+|   `dev`         | `https://dev.sunnypilot.ai`                   |
+| `custom-branch` | `https://install.sunnypilot.ai/{branch_name}` |
+
+> [!TIP]
+> You can use sunnypilot/targetbranch as an install URL. Example: 'sunnypilot/staging'.
+
+> [!NOTE]
+> Do you require further assistance with software installation? Join the [sunnypilot community forum](https://community.sunnypilot.ai/new-topic?category=general/qa) and create a topic in the General/Q&A Category channel.
+
+
+<details>
+
+<summary>Older legacy branches</summary>
+
+### If you want to use our older legacy branches (*not recommended*)
+
+> [**IMPORTANT**]
+> It is recommended to [re-flash AGNOS](https://flash.comma.ai/) if you intend to downgrade from the new branches.
+> You can still restore the latest sunnylink backup made on the old branches.
+
+|    Branch    |         Installation URL         |
+|:------------:|:--------------------------------:|
+| `release-c3` | https://release-c3.sunnypilot.ai |
+| `staging-c3` | https://staging-c3.sunnypilot.ai |
+|   `dev-c3`   | https://dev-c3.sunnypilot.ai     |
+
+</details>
+
 
 ## 🎆 Pull Requests
 We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.

RELEASES.md

@@ -1,32 +1,3 @@
-Version 0.11.2 (2026-08-12)
-=======================
-* New driving model
-  * Big model with 880M parameters
-* Support for big models running on an external GPU
-* Live stream cameras from comma connect
-* Generate dashcam clips from comma connect
-* Remote comma body control from comma connect
-* New alert sounds
-* CUPRA Born 2021-2023 support thanks to DaHansi!
-* Volkswagen ID.4 2021-2025 support thanks to DaHansi!
-
-Version 0.11.1 (2026-05-18)
-========================
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Improved thermal policy for comma four
-* Acura MDX 2022-24 support thanks to mvl-boston!
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-Version 0.11.0 (2026-03-17)
-========================
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-
 Version 0.10.3 (2025-12-17)
 ========================
 * New driving model #36249
@@ -1112,7 +1083,7 @@ Version 0.2.1  (2016-12-14)
 
 Version 0.2  (2016-12-12)
 =========================
- * Car/Radar abstraction layers have shipped, see openpilot/cereal/car.capnp
+ * Car/Radar abstraction layers have shipped, see cereal/car.capnp
  * controlsd has been refactored
  * Shipped plant model and testing maneuvers
  * visiond exits more gracefully now

SConstruct

@@ -4,136 +4,58 @@ import sys
 import sysconfig
 import platform
 import shlex
-import importlib
 import numpy as np
 
 import SCons.Errors
-from SCons.Defaults import _stripixes
-
-COMMA_HARDWARE = os.path.isfile('/AGNOS')
 
 SCons.Warnings.warningAsException(True)
 
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2))))
+SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
 
+AddOption('--kaitai', action='store_true', help='Regenerate kaitai struct parsers')
+AddOption('--asan', action='store_true', help='turn on ASAN')
+AddOption('--ubsan', action='store_true', help='turn on UBSan')
+AddOption('--mutation', action='store_true', help='generate mutation-ready code')
 AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
-AddOption('--verbose', action='store_true', default=False, help='show full build commands')
-release = not os.path.exists(File('#.gitattributes').abspath) # file absent on release branch, see release_files.py
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=(not COMMA_HARDWARE and not release),
+          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-submodule_python_paths = [
-  Dir("#").abspath,
-  Dir("#msgq_repo").abspath,
-  Dir("#opendbc_repo").abspath,
-  Dir("#rednose_repo").abspath,
-  Dir("#teleoprtc_repo").abspath,
-  Dir("#tinygrad_repo").abspath,
-]
-for p in reversed(submodule_python_paths):
-  if p not in sys.path:
-    sys.path.insert(0, p)
-
-if external_pythonpath := os.environ.get("PYTHONPATH"):
-  submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths]
-
 # Detect platform
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and COMMA_HARDWARE:
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-  arch = "comma_arm64"
+elif arch == "aarch64" and os.path.isfile('/TICI'):
+  arch = "larch64"
 assert arch in [
-  "comma_arm64",  # linux comma hardware (AGNOS) arm64
+  "larch64",  # linux tici arm64
-  "aarch64",      # linux pc arm64
+  "aarch64",  # linux pc arm64
-  "x86_64",       # linux pc x64
+  "x86_64",   # linux pc x64
-  "Darwin",       # macOS arm64 (x86 not supported)
+  "Darwin",   # macOS arm64 (x86 not supported)
 ]
 
-pkg_names = ['acados', 'capnproto', 'eigen', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zstd']
-pkgs = [importlib.import_module(name) for name in pkg_names]
-acados = pkgs[pkg_names.index('acados')]
-ffmpeg = pkgs[pkg_names.index('ffmpeg')]
-# Shared package ships .so/.dylib; older device venvs still have static .a only.
-# Keep static link deps (x264/z/va/drm) when the installed package is static so
-# COMMA_HARDWARE CI works without upgrading the device venv yet.
-# TODO: drop the static fallback once device venvs have comma-deps-ffmpeg>=7.1.0.post94
-_ffmpeg_lib_names = os.listdir(ffmpeg.LIB_DIR) if os.path.isdir(ffmpeg.LIB_DIR) else []
-ffmpeg_shared = any(
-  n.startswith('libavcodec.so') or (n.startswith('libavcodec') and n.endswith('.dylib'))
-  for n in _ffmpeg_lib_names
-)
-ffmpeg_libs = ['avformat', 'avcodec', 'swresample', 'avutil']
-if not ffmpeg_shared:
-  ffmpeg_libs += ['x264', 'z']
-  if arch != "Darwin":
-    ffmpeg_libs += ['va', 'va-drm', 'drm']
-acados_include_dirs = [
-  acados.INCLUDE_DIR,
-  os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
-  os.path.join(acados.INCLUDE_DIR, "hpipm", "include"),
-]
-
-
-# ***** enforce a whitelist of system libraries *****
-# this prevents silently relying on a 3rd party package,
-# e.g. apt-installed libusb. all libraries should either
-# be distributed with all Linux distros and macOS, or
-# vendored in commaai/dependencies.
-allowed_system_libs = {
-  "EGL", "GLESv2", "GL",
-  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
-  "dl", "drm", "gbm", "m", "pthread",
-}
-
-def _resolve_lib(env, name):
-  for d in env.Flatten(env.get('LIBPATH', [])):
-    p = Dir(str(d)).abspath
-    for ext in ('.a', '.so', '.dylib'):
-      f = File(os.path.join(p, f'lib{name}{ext}'))
-      if f.exists() or f.has_builder():
-        return name
-  if name in allowed_system_libs:
-    return name
-  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
-
-def _libflags(target, source, env, for_signature):
-  libs = []
-  lp = env.subst('$LIBLITERALPREFIX')
-  for lib in env.Flatten(env.get('LIBS', [])):
-    if isinstance(lib, str):
-      if os.sep in lib or lib.startswith('#'):
-        libs.append(File(lib))
-      elif lib.startswith('-') or (lp and lib.startswith(lp)):
-        libs.append(lib)
-      else:
-        libs.append(_resolve_lib(env, lib))
-    else:
-      libs.append(lib)
-  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
-                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
-
 env = Environment(
   ENV={
     "PATH": os.environ['PATH'],
-    "PYTHONPATH": os.pathsep.join(submodule_python_paths),
+    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": acados.DIR,
+    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
+    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": acados.TERA_PATH
+    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
   },
+  CC='clang',
+  CXX='clang++',
   CCFLAGS=[
     "-g",
     "-fPIC",
-    "-pipe",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow" if arch in ("Darwin", "comma_arm64") else "-Wshadow=local",
+    "-Wshadow",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
@@ -143,54 +65,71 @@ env = Environment(
   CFLAGS=["-std=gnu11"],
   CXXFLAGS=["-std=c++1z"],
   CPPPATH=[
-    "#openpilot",
-    "#msgq_repo",            # #include "msgq/..."
-    "#opendbc_repo",         # #include "opendbc/..."
-    "#rednose_repo",         # #include "rednose/..."
-    "#rednose_repo/rednose", # #include "logger/..." (rednose package root)
-    "#openpilot/cereal/gen/cpp",
-    acados_include_dirs,
-    [x.INCLUDE_DIR for x in pkgs],
     "#",
+    "#msgq",
+    "#third_party",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/acados/include",
+    "#third_party/acados/include/blasfeo/include",
+    "#third_party/acados/include/hpipm/include",
+    "#third_party/catch2/include",
+    "#third_party/libyuv/include",
   ],
   LIBPATH=[
-    "#openpilot/common",
+    "#common",
     "#msgq_repo",
-    "#openpilot/selfdrive/pandad",
+    "#third_party",
-    "#rednose_repo/rednose/helpers",
+    "#selfdrive/pandad",
-    [x.LIB_DIR for x in pkgs],
+    "#rednose/helpers",
+    f"#third_party/libyuv/{arch}/lib",
+    f"#third_party/acados/{arch}/lib",
   ],
-  RPATH=[ffmpeg.LIB_DIR] if ffmpeg_shared else [],
+  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
-  REDNOSE_ROOT="#rednose_repo",
+  REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
-  toolpath=["#msgq_repo/site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
+  toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
-# SCons' Darwin linker tool doesn't define the variables used to expand RPATH.
-if arch == "Darwin":
-  env["RPATHPREFIX"] = "-Wl,-rpath,"
-  env["RPATHSUFFIX"] = ""
-  env["_RPATH"] = "${_concat(RPATHPREFIX, RPATH, RPATHSUFFIX, __env__)}"
-if arch != "comma_arm64":
-  env['_LIBFLAGS'] = _libflags
 
 # Arch-specific flags and paths
-if arch == "comma_arm64":
+if arch == "larch64":
-  env["CC"] = "clang"
+  env.Append(CPPPATH=["#third_party/opencl/include"])
-  env["CXX"] = "clang++"
   env.Append(LIBPATH=[
+    "/usr/local/lib",
+    "/system/vendor/lib64",
     "/usr/lib/aarch64-linux-gnu",
   ])
-  arch_flags = ["-D__COMMA_HARDWARE__", "-mcpu=cortex-a57"]
+  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
   env.Append(CCFLAGS=arch_flags)
   env.Append(CXXFLAGS=arch_flags)
 elif arch == "Darwin":
   env.Append(LIBPATH=[
+    f"{brew_prefix}/lib",
+    f"{brew_prefix}/opt/openssl@3.0/lib",
+    f"{brew_prefix}/opt/llvm/lib/c++",
     "/System/Library/Frameworks/OpenGL.framework/Libraries",
   ])
   env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
   env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CPPPATH=[
+    f"{brew_prefix}/include",
+    f"{brew_prefix}/opt/openssl@3.0/include",
+  ])
+else:
+  env.Append(LIBPATH=[
+    "/usr/lib",
+    "/usr/local/lib",
+  ])
+
+# Sanitizers and extra CCFLAGS from CLI
+if GetOption('asan'):
+  env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
+  env.Append(LINKFLAGS=["-fsanitize=address"])
+elif GetOption('ubsan'):
+  env.Append(CCFLAGS=["-fsanitize=undefined"])
+  env.Append(LINKFLAGS=["-fsanitize=undefined"])
 
 _extra_cc = shlex.split(GetOption('ccflags') or '')
 if _extra_cc:
@@ -200,26 +139,21 @@ if _extra_cc:
 if arch != "Darwin":
   env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
 
-# Shorter build output: show brief descriptions instead of full commands.
+# progress output
-# Full command lines are still printed on failure by scons.
+node_interval = 5
-if not GetOption('verbose'):
+node_count = 0
-  for action, short in (
+def progress_function(node):
-    ("CC",     "CC"),
+  global node_count
-    ("CXX",    "CXX"),
+  node_count += node_interval
-    ("LINK",   "LINK"),
+  sys.stderr.write("progress: %d\n" % node_count)
-    ("SHCC",   "CC"),
+if os.environ.get('SCONS_PROGRESS'):
-    ("SHCXX",  "CXX"),
+  Progress(progress_function, interval=node_interval)
-    ("SHLINK", "LINK"),
-    ("AR",     "AR"),
-    ("RANLIB", "RANLIB"),
-    ("AS",     "AS"),
-  ):
-    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
 
 # ********** Cython build environment **********
+py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
-envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
+envCython["CPPPATH"] += [py_include, np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
@@ -231,28 +165,18 @@ else:
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch', 'acados', 'release', 'ffmpeg_libs')
+Export('env', 'arch')
 
 # Setup cache dir
-default_cache_dir = '/data/scons_cache' if arch == "comma_arm64" else '/tmp/scons_cache'
+default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
 cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
-cache_size_limit = 4e9 if "CI" in os.environ else 2e9
 CacheDir(cache_dir)
 Clean(["."], cache_dir)
 
-def prune_cache_dir(target=None, source=None, env=None):
-  cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime)
-  cache_size = sum(os.path.getsize(f) for f in cache_files)
-  for f in cache_files:
-    if cache_size < cache_size_limit:
-      break
-    cache_size -= os.path.getsize(f)
-    os.unlink(f)
-
 # ********** start building stuff **********
 
 # Build common module
-SConscript(['openpilot/common/SConscript'])
+SConscript(['common/SConscript'])
 Import('_common')
 common = [_common, 'json11', 'zmq']
 Export('common')
@@ -262,8 +186,9 @@ Export('common')
 env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
 
-SConscript(['openpilot/cereal/SConscript'])
+SConscript(['cereal/SConscript'])
 
 Import('socketmaster', 'msgq')
 messaging = [socketmaster, msgq, 'capnp', 'kj',]
@@ -274,79 +199,28 @@ Export('messaging')
 SConscript(['panda/SConscript'])
 
 # Build rednose library
-SConscript(['rednose_repo/rednose/SConscript'])
+SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
-  'openpilot/system/loggerd/SConscript',
+  'system/ubloxd/SConscript',
+  'system/loggerd/SConscript',
 ])
 
-if arch == "comma_arm64":
+if arch == "larch64":
-  SConscript(['openpilot/system/camerad/SConscript'])
+  SConscript(['system/camerad/SConscript'])
 
-# Build selfdrive
+# Build openpilot
-SConscript([
+SConscript(['third_party/SConscript'])
-  'openpilot/selfdrive/pandad/SConscript',
-  'openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'openpilot/selfdrive/locationd/SConscript',
-  'openpilot/selfdrive/modeld/SConscript',
-  'openpilot/selfdrive/ui/SConscript',
-])
 
-SConscript(['openpilot/sunnypilot/SConscript'])
+SConscript(['selfdrive/SConscript'])
 
-# Build desktop-only tools
+SConscript(['sunnypilot/SConscript'])
-if GetOption('extras') and arch != "comma_arm64":
+
-  SConscript([
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
-    'openpilot/tools/replay/SConscript',
+  SConscript(['tools/replay/SConscript'])
-    'openpilot/tools/cabana/SConscript',
+  if arch != "larch64":
-    'openpilot/tools/jotpluggler/SConscript',
+    SConscript(['tools/cabana/SConscript'])
-  ])
 
 
 env.CompilationDatabase('compile_commands.json')
-
-# progress output
-def count_scons_nodes(nodes):
-  seen = set()
-  stack = list(nodes)
-
-  while stack:
-    node = stack.pop().disambiguate()
-    if node in seen:
-      continue
-    seen.add(node)
-    if hasattr(node, 'has_builder') and node.has_builder():
-      build_product_nodes.add(node)
-    executor = node.get_executor()
-    if executor is not None:
-      stack += executor.get_all_prerequisites() + executor.get_all_children()
-
-  return len(seen)
-
-progress_interval = 5
-progress_count = 0
-build_product_nodes = set()
-progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
-
-def progress_function(node):
-  global progress_count
-  if progress_count >= progress_total:
-    return
-  progress_count = min(progress_count + progress_interval, progress_total)
-  progress = round(100. * progress_count / progress_total, 1)
-  sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
-  if progress == 100. and sys.stderr.isatty():
-    sys.stderr.write("\n")
-  sys.stderr.flush()
-
-Progress(progress_function, interval=progress_interval)
-AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)
-
-def check_build_product_size(target, source, env):
-  limit = 50 * 1024 * 1024  # GitHub max size
-  for t in target:
-    if hasattr(t, 'isfile') and t.isfile() and (size := os.path.getsize(t.abspath)) > limit:
-      raise SCons.Errors.UserError(f"{t} is {size / (1024 * 1024):.1f} MiB, exceeding the {limit / (1024 * 1024):.1f} MiB limit")
-if not GetOption('extras'):
-  AddPostAction(list(build_product_nodes), Action(check_build_product_size, None))

cereal/README.md

@@ -24,17 +24,17 @@ things are not. Read more details [here](https://capnproto.org/language.html).
 ### Custom forks
 
 Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
-spec, however this could conflict with future changes made in mainline openpilot's cereal spec. Rebasing against mainline openpilot
+spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
 then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
-[custom.capnp](custom.capnp) that we will leave empty in mainline openpilot's cereal spec. **If you only modify those, you can ensure your
+[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
 An example of compatible changes:
 ```diff
-diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp
+diff --git a/cereal/custom.capnp b/cereal/custom.capnp
 index 3348e859e..3365c7b98 100644
---- a/openpilot/cereal/custom.capnp
+--- a/cereal/custom.capnp
-+++ b/openpilot/cereal/custom.capnp
++++ b/cereal/custom.capnp
 @@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
  # DO rename the structs
  # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
@@ -48,10 +48,10 @@ index 3348e859e..3365c7b98 100644
  }
 
  struct CustomReserved1 @0xaedffd8f31e7b55d {
-diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp
+diff --git a/cereal/log.capnp b/cereal/log.capnp
 index 1209f3fd9..b189f58b6 100644
---- a/openpilot/cereal/log.capnp
+--- a/cereal/log.capnp
-+++ b/openpilot/cereal/log.capnp
++++ b/cereal/log.capnp
 @@ -2558,14 +2558,14 @@ struct Event {
 
      # DO change the name of the field
@@ -76,7 +76,7 @@ index 1209f3fd9..b189f58b6 100644
 Example
 ---
 ```python
-import openpilot.cereal.messaging as messaging
+import cereal.messaging as messaging
 
 # in subscriber
 sm = messaging.SubMaster(['sensorEvents'])

cereal/SConscript

@@ -0,0 +1,20 @@
+Import('env', 'common', 'msgq')
+
+cereal_dir = Dir('.')
+gen_dir = Dir('gen')
+
+# Build cereal
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
+env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
+            schema_files,
+            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
+
+cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
+
+# Build messaging
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+
+socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
+
+Export('cereal', 'socketmaster')

cereal/__init__.py

@@ -0,0 +1,11 @@
+import os
+import capnp
+from importlib.resources import as_file, files
+
+capnp.remove_import_hook()
+
+with as_file(files("cereal")) as fspath:
+  CEREAL_PATH = fspath.as_posix()
+  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -0,0 +1 @@
+../opendbc_repo/opendbc/car/car.capnp

cereal/custom.capnp

@@ -1,4 +1,4 @@
-using Cxx = import "/include/c++.capnp";
+using Cxx = import "./include/c++.capnp";
 $Cxx.namespace("cereal");
 
 @0xb526ba661d550a59;
@@ -69,8 +69,6 @@ struct LeadData {
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
   mads @0 :ModularAssistiveDrivingSystem;
   intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
-  buttonsPressed @2 :UInt16;
-  buttonsReleaseToggle @3 :UInt16;
 
   enum AudibleAlert {
     none @0;
@@ -139,16 +137,10 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
     eta @2 :UInt32;
   }
 
-  struct Chunk {
-    fileName @0 :Text;
-    sha256 @1 :Text;
-  }
-
   struct Artifact {
     fileName @0 :Text;
     downloadUri @1 :DownloadUri;
     downloadProgress @2 :DownloadProgress;
-    chunks @3 :List(Chunk);
   }
 
   struct Model {
@@ -161,9 +153,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
       navigation @1;
       vision @2;
       policy @3;
-      offPolicy @4;
-      onPolicy @5;
-      chunked @6;
     }
   }
 
@@ -351,7 +340,6 @@ struct OnroadEventSP @0xda96579883444c35 {
     speedLimitChanged @21;
     speedLimitPending @22;
     e2eChime @23;
-    laneChangeRoadEdge @24;
   }
 }
 
@@ -458,8 +446,6 @@ struct LiveMapDataSP @0xf416ec09499d9d19 {
 
 struct ModelDataV2SP @0xa1680744031fdb2d {
   laneTurnDirection @0 :TurnDirection;
-  leftLaneChangeEdgeBlock @1 :Bool;
-  rightLaneChangeEdgeBlock @2 :Bool;
 
   enum TurnDirection {
     none @0;

cereal/legacy.capnp

@@ -1,9 +1,9 @@
-using Cxx = import "/include/c++.capnp";
+using Cxx = import "./include/c++.capnp";
 $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
 
-# deprecated.capnp: a home for deprecated structs
+# legacy.capnp: a home for deprecated structs
 
 struct LogRotate @0x9811e1f38f62f2d1 {
   segmentNum @0 :Int32;
@@ -571,217 +571,4 @@ struct LidarPts @0xe3d6685d4e9d8f7a {
   pkt @4 :Data;
 }
 
-struct LiveTracksDEPRECATED @0xb16f60103159415a {
-  trackId @0 :Int32;
-  dRel @1 :Float32;
-  yRel @2 :Float32;
-  vRel @3 :Float32;
-  aRel @4 :Float32;
-  timeStamp @5 :Float32;
-  status @6 :Float32;
-  currentTime @7 :Float32;
-  stationary @8 :Bool;
-  oncoming @9 :Bool;
-}
 
-struct LiveMpcData @0x92a5e332a85f32a0 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  psi @2 :List(Float32);
-  curvature @3 :List(Float32);
-  qpIterations @4 :UInt32;
-  calculationTime @5 :UInt64;
-  cost @6 :Float64;
-}
-
-struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
-  xEgo @0 :List(Float32);
-  vEgo @1 :List(Float32);
-  aEgo @2 :List(Float32);
-  xLead @3 :List(Float32);
-  vLead @4 :List(Float32);
-  aLead @5 :List(Float32);
-  aLeadTau @6 :Float32;    # lead accel time constant
-  qpIterations @7 :UInt32;
-  mpcId @8 :UInt32;
-  calculationTime @9 :UInt64;
-  cost @10 :Float64;
-}
-
-struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
-  frameId @0 :UInt32;
-  modelExecutionTime @14 :Float32;
-  dspExecutionTime @16 :Float32;
-  rawPredictions @15 :Data;
-
-  faceOrientation @3 :List(Float32);
-  facePosition @4 :List(Float32);
-  faceProb @5 :Float32;
-  leftEyeProb @6 :Float32;
-  rightEyeProb @7 :Float32;
-  leftBlinkProb @8 :Float32;
-  rightBlinkProb @9 :Float32;
-  faceOrientationStd @11 :List(Float32);
-  facePositionStd @12 :List(Float32);
-  sunglassesProb @13 :Float32;
-  poorVision @17 :Float32;
-  partialFace @18 :Float32;
-  distractedPose @19 :Float32;
-  distractedEyes @20 :Float32;
-  eyesOnRoad @21 :Float32;
-  phoneUse @22 :Float32;
-  occludedProb @23 :Float32;
-
-  readyProb @24 :List(Float32);
-  notReadyProb @25 :List(Float32);
-
-  irPwrDEPRECATED @10 :Float32;
-  descriptorDEPRECATED @1 :List(Float32);
-  stdDEPRECATED @2 :Float32;
-}
-
-struct NavModelData @0xac3de5c437be057a {
-  frameId @0 :UInt32;
-  locationMonoTime @6 :UInt64;
-  modelExecutionTime @1 :Float32;
-  dspExecutionTime @2 :Float32;
-  features @3 :List(Float32);
-  # predicted future position
-  position @4 :XYData;
-  desirePrediction @5 :List(Float32);
-
-  # All SI units and in device frame
-  struct XYData @0xbe09e615b2507e26 {
-    x @0 :List(Float32);
-    y @1 :List(Float32);
-    xStd @2 :List(Float32);
-    yStd @3 :List(Float32);
-  }
-}
-
-struct AndroidBuildInfo @0xfe2919d5c21f426c {
-  board @0 :Text;
-  bootloader @1 :Text;
-  brand @2 :Text;
-  device @3 :Text;
-  display @4 :Text;
-  fingerprint @5 :Text;
-  hardware @6 :Text;
-  host @7 :Text;
-  id @8 :Text;
-  manufacturer @9 :Text;
-  model @10 :Text;
-  product @11 :Text;
-  radioVersion @12 :Text;
-  serial @13 :Text;
-  supportedAbis @14 :List(Text);
-  tags @15 :Text;
-  time @16 :Int64;
-  type @17 :Text;
-  user @18 :Text;
-
-  versionCodename @19 :Text;
-  versionRelease @20 :Text;
-  versionSdk @21 :Int32;
-  versionSecurityPatch @22 :Text;
-}
-
-struct AndroidSensor @0x9b513b93a887dbcd {
-  id @0 :Int32;
-  name @1 :Text;
-  vendor @2 :Text;
-  version @3 :Int32;
-  handle @4 :Int32;
-  type @5 :Int32;
-  maxRange @6 :Float32;
-  resolution @7 :Float32;
-  power @8 :Float32;
-  minDelay @9 :Int32;
-  fifoReservedEventCount @10 :UInt32;
-  fifoMaxEventCount @11 :UInt32;
-  stringType @12 :Text;
-  maxDelay @13 :Int32;
-}
-
-struct IosBuildInfo @0xd97e3b28239f5580 {
-  appVersion @0 :Text;
-  appBuild @1 :UInt32;
-  osVersion @2 :Text;
-  deviceModel @3 :Text;
-}
-
-enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
-  unknown @0;
-  neo @1;
-  chffrAndroid @2;
-  front @3;
-}
-
-struct AndroidCaptureResult @0xbcc3efbac41d2048 {
-  sensitivity @0 :Int32;
-  frameDuration @1 :Int64;
-  exposureTime @2 :Int64;
-  rollingShutterSkew @3 :UInt64;
-  colorCorrectionTransform @4 :List(Int32);
-  colorCorrectionGains @5 :List(Float32);
-  displayRotation @6 :Int8;
-}
-
-enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
-  none @0;
-  client @1;
-  cdp @2;
-  dcp @3;
-}
-
-struct LateralINDIState @0x939463348632375e {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  steeringRateDeg @2 :Float32;
-  steeringAccelDeg @3 :Float32;
-  rateSetPoint @4 :Float32;
-  accelSetPoint @5 :Float32;
-  accelError @6 :Float32;
-  delayedOutput @7 :Float32;
-  delta @8 :Float32;
-  output @9 :Float32;
-  saturated @10 :Bool;
-  steeringAngleDesiredDeg @11 :Float32;
-  steeringRateDesiredDeg @12 :Float32;
-}
-
-struct LateralLQRState @0x9024e2d790c82ade {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  i @2 :Float32;
-  output @3 :Float32;
-  lqrOutput @4 :Float32;
-  saturated @5 :Bool;
-  steeringAngleDesiredDeg @6 :Float32;
-}
-
-struct LateralPlannerSolution @0x84caeca5a6b4acfe {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  yaw @2 :List(Float32);
-  yawRate @3 :List(Float32);
-  xStd @4 :List(Float32);
-  yStd @5 :List(Float32);
-  yawStd @6 :List(Float32);
-  yawRateStd @7 :List(Float32);
-}
-
-struct GpsTrajectory @0x8cfeb072f5301000 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-}
-
-struct AudioFeedbackDEPRECATED @0xed47e3c075be372a {
-  audio @0 :AudioData;
-  blockNum @1 :UInt16;
-
-  struct AudioData {
-    data @0 :Data;
-    sampleRate @1 :UInt32;
-  }
-}

cereal/messaging/__init__.py

@@ -1,48 +1,19 @@
 # must be built with scons
-from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, drain_sock_raw
 import msgq
+
 import os
 import capnp
 import time
 
-from typing import Union
+from typing import Optional, List, Union, Dict
 
-from openpilot.cereal import log
+from cereal import log
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
-
-__all__ = (
-  "NO_TRAVERSAL_LIMIT",
-  "Context",
-  "FrequencyTracker",
-  "IpcError",
-  "MultiplePublishersError",
-  "Poller",
-  "PubMaster",
-  "PubSocket",
-  "SocketEventHandle",
-  "SubMaster",
-  "SubSocket",
-  "delete_fake_prefix",
-  "drain_sock",
-  "drain_sock_raw",
-  "fake_event_handle",
-  "get_fake_prefix",
-  "log_from_bytes",
-  "new_message",
-  "pub_sock",
-  "recv_one",
-  "recv_one_or_none",
-  "recv_one_retry",
-  "recv_sock",
-  "reset_context",
-  "set_fake_prefix",
-  "sub_sock",
-  "toggle_fake_events",
-  "wait_for_one_event",
-)
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
@@ -53,8 +24,8 @@ def pub_sock(endpoint: str) -> PubSocket:
   return msgq.pub_sock(endpoint, segment_size)
 
 
-def sub_sock(endpoint: str, poller: Poller | None = None, addr: str = "127.0.0.1",
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: int | None = None) -> SubSocket:
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
   service = SERVICE_LIST.get(endpoint)
   segment_size = service.queue_size if service else 0
   return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
@@ -70,7 +41,7 @@ def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event
     return msg
 
 
-def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
+def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
   args = {
     'valid': False,
     'logMonoTime': int(time.monotonic() * 1e9),
@@ -85,14 +56,14 @@ def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp
   return dat
 
 
-def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> list[capnp.lib.capnp._DynamicStructReader]:
+def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
   """Receive all message currently available on the queue"""
   msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
   return [log_from_bytes(m) for m in msgs]
 
 
 # TODO: print when we drop packets?
-def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicStructReader | None:
+def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
   """Same as drain sock, but only returns latest message. Consider using conflate instead."""
   dat = None
 
@@ -113,14 +84,14 @@ def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicSt
   return dat
 
 
-def recv_one(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
+def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive()
   if dat is not None:
     dat = log_from_bytes(dat)
   return dat
 
 
-def recv_one_or_none(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
+def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive(non_blocking=True)
   if dat is not None:
     dat = log_from_bytes(dat)
@@ -179,27 +150,27 @@ class FrequencyTracker:
 
 
 class SubMaster:
-  def __init__(self, services: list[str], poll: str | None = None,
+  def __init__(self, services: List[str], poll: Optional[str] = None,
-               ignore_alive: list[str] | None = None, ignore_avg_freq: list[str] | None = None,
+               ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
-               ignore_valid: list[str] | None = None, addr: str = "127.0.0.1", frequency: float | None = None):
+               ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
     self.frame = -1
     self.services = services
-    self.seen = dict.fromkeys(services, False)
+    self.seen = {s: False for s in services}
-    self.updated = dict.fromkeys(services, False)
+    self.updated = {s: False for s in services}
-    self.recv_time = dict.fromkeys(services, 0.0)
+    self.recv_time = {s: 0. for s in services}
-    self.recv_frame = dict.fromkeys(services, 0)
+    self.recv_frame = {s: 0 for s in services}
     self.sock = {}
     self.data = {}
-    self.logMonoTime = dict.fromkeys(services, 0)
+    self.logMonoTime = {s: 0 for s in services}
 
     # zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
     on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
-    self.static_freq_services = {s for s in services if not on_demand[s]}
+    self.static_freq_services = set(s for s in services if not on_demand[s])
     self.alive = {s: on_demand[s] for s in services}
     self.freq_ok = {s: on_demand[s] for s in services}
     self.valid = {s: on_demand[s] for s in services}
 
-    self.freq_tracker: dict[str, FrequencyTracker] = {}
+    self.freq_tracker: Dict[str, FrequencyTracker] = {}
     self.poller = Poller()
     polled_services = set([poll, ] if poll is not None else services)
     self.non_polled_services = set(services) - polled_services
@@ -242,7 +213,7 @@ class SubMaster:
       msgs.append(recv_one_or_none(self.sock[s]))
     self.update_msgs(time.monotonic(), msgs)
 
-  def update_msgs(self, cur_time: float, msgs: list[capnp.lib.capnp._DynamicStructReader]) -> None:
+  def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1
     self.updated = dict.fromkeys(self.services, False)
     for msg in msgs:
@@ -265,21 +236,21 @@ class SubMaster:
       self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
       self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
 
-  def all_alive(self, service_list: list[str] | None = None) -> bool:
+  def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
 
-  def all_freq_ok(self, service_list: list[str] | None = None) -> bool:
+  def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
 
-  def all_valid(self, service_list: list[str] | None = None) -> bool:
+  def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
 
-  def all_checks(self, service_list: list[str] | None = None) -> bool:
+  def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
     return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
 
 
 class PubMaster:
-  def __init__(self, services: list[str]):
+  def __init__(self, services: List[str]):
     self.sock = {}
     for s in services:
       self.sock[s] = pub_sock(s)
@@ -290,11 +261,11 @@ class PubMaster:
     self.sock[s].send(dat)
 
   def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
-    try:
+    for _ in range(int(timeout*(1./dt))):
-      self.sock[s].wait_for_readers(timeout=timeout, interval=dt)
+      if self.sock[s].all_readers_updated():
-      return True
+        return True
-    except TimeoutError:
+      time.sleep(dt)
-      return False
+    return False
 
   def all_readers_updated(self, s: str) -> bool:
-    return self.sock[s].all_readers_updated()
+    return self.sock[s].all_readers_updated()  # type: ignore

cereal/messaging/bridge.cc

@@ -1,7 +1,7 @@
 #include <cassert>
 
-#include "openpilot/cereal/messaging/msgq_to_zmq.h"
+#include "cereal/messaging/msgq_to_zmq.h"
-#include "openpilot/cereal/services.h"
+#include "cereal/services.h"
 #include "common/util.h"
 
 ExitHandler do_exit;
@@ -25,16 +25,15 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<BridgeZmqPoller>();
+  auto poller = std::make_unique<ZMQPoller>();
-  auto pub_context = std::make_unique<Context>();
+  auto pub_context = std::make_unique<MSGQContext>();
-  auto sub_context = std::make_unique<BridgeZmqContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
-  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
+  std::map<SubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new PubSocket();
+    auto pub_sock = new MSGQPubSocket();
-    auto sub_sock = new BridgeZmqSubSocket();
+    auto sub_sock = new ZMQSubSocket();
-    size_t queue_size = services.at(endpoint).queue_size;
+    pub_sock->connect(pub_context.get(), endpoint);
-    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/messaging.h

@@ -8,7 +8,7 @@
 
 #include <capnp/serialize.h>
 
-#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "cereal/gen/cpp/log.capnp.h"
 #include "common/timing.h"
 #include "msgq/ipc.h"
 

cereal/messaging/msgq_to_zmq.cc

@@ -1,8 +1,7 @@
-#include "openpilot/cereal/messaging/msgq_to_zmq.h"
+#include "cereal/messaging/msgq_to_zmq.h"
 
 #include <cassert>
 
-#include "openpilot/cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -22,14 +21,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
+  zmq_context = std::make_unique<ZMQContext>();
-  msgq_context = std::make_unique<Context>();
+  msgq_context = std::make_unique<MSGQContext>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -49,7 +48,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -72,7 +71,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -109,8 +108,7 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -130,7 +128,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,8 +7,9 @@
 #include <string>
 #include <vector>
 
+#define private public
 #include "msgq/impl_msgq.h"
-#include "openpilot/cereal/messaging/bridge_zmq.h"
+#include "msgq/impl_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -21,16 +22,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<Context> msgq_context;
+  std::unique_ptr<MSGQContext> msgq_context;
-  std::unique_ptr<BridgeZmqContext> zmq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -3,8 +3,8 @@
 #include <string>
 #include <mutex>
 
-#include "openpilot/cereal/services.h"
+#include "cereal/services.h"
-#include "openpilot/cereal/messaging/messaging.h"
+#include "cereal/messaging/messaging.h"
 
 const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
 

cereal/messaging/tests/test_messaging.py

@@ -1,16 +1,16 @@
+import os
 import capnp
 import multiprocessing
 import numbers
 import random
 import threading
 import time
-from openpilot.common.test import OpenpilotTestCase
+from parameterized import parameterized
-from openpilot.common.parameterized import parameterized
+import pytest
 
-from openpilot.cereal import log
+from cereal import log, car
-from opendbc.car.structs import car
+import cereal.messaging as messaging
-import openpilot.cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
-from openpilot.cereal.services import SERVICE_LIST
 
 events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
 
@@ -23,10 +23,14 @@ def random_socks(num_socks=10):
 def random_bytes(length=1000):
   return bytes([random.randrange(0xFF) for _ in range(length)])
 
+def zmq_sleep(t=1):
+  if "ZMQ" in os.environ:
+    time.sleep(t)
+
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "steeringTorque", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:
@@ -47,7 +51,17 @@ def delayed_send(delay, sock, dat):
   threading.Timer(delay, send_func).start()
 
 
-class TestMessaging(OpenpilotTestCase):
+class TestMessaging:
+  def setUp(self):
+    # TODO: ZMQ tests are too slow; all sleeps will need to be
+    # replaced with logic to block on the necessary condition
+    if "ZMQ" in os.environ:
+      pytest.skip()
+
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep()
+
   @parameterized.expand(events)
   def test_new_message(self, evt):
     try:
@@ -74,6 +88,7 @@ class TestMessaging(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
 
     # no wait and no msgs in queue
     msgs = func(sub_sock)
@@ -81,11 +96,10 @@ class TestMessaging(OpenpilotTestCase):
     assert len(msgs) == 0
 
     # no wait but msgs are queued up
-    pub_sock.send(messaging.new_message(sock).to_bytes())
-    assert sub_sock.receive() is not None  # synchronize the PUB/SUB connection
     num_msgs = random.randrange(3, 10)
     for _ in range(num_msgs):
       pub_sock.send(messaging.new_message(sock).to_bytes())
+    time.sleep(0.1)
     msgs = func(sub_sock)
     assert isinstance(msgs, list)
     assert all(isinstance(msg, expected_type) for msg in msgs)
@@ -95,6 +109,7 @@ class TestMessaging(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock, timeout=100)
+    zmq_sleep()
 
     # no wait and no msg in queue, socket should timeout
     recvd = messaging.recv_sock(sub_sock)
@@ -112,7 +127,8 @@ class TestMessaging(OpenpilotTestCase):
   def test_recv_one(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=10)
+    sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
 
     # no msg in queue, socket should timeout
     recvd = messaging.recv_one(sub_sock)
@@ -125,10 +141,12 @@ class TestMessaging(OpenpilotTestCase):
     assert isinstance(recvd, capnp._DynamicStructReader)
     assert_carstate(msg.carState, recvd.carState)
 
+  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
   def test_recv_one_or_none(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock)
+    zmq_sleep()
 
     # no msg in queue, socket shouldn't block
     recvd = messaging.recv_one_or_none(sub_sock)
@@ -143,16 +161,19 @@ class TestMessaging(OpenpilotTestCase):
 
   def test_recv_one_retry(self):
     sock = "carState"
-    sock_timeout = 0.005
+    sock_timeout = 0.1
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
+    zmq_sleep()
 
-    # wait 5 socket timeouts and make sure it's still retrying
+    # this test doesn't work with ZMQ since multiprocessing interrupts it
-    p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
+    if "ZMQ" not in os.environ:
-    p.start()
+      # wait 5 socket timeouts and make sure it's still retrying
-    time.sleep(sock_timeout*5)
+      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
-    assert p.is_alive()
+      p.start()
-    p.terminate()
+      time.sleep(sock_timeout*5)
+      assert p.is_alive()
+      p.terminate()
 
     # wait 5 socket timeouts before sending
     msg = random_carstate()

cereal/messaging/tests/test_pub_sub_master.py

@@ -1,16 +1,20 @@
 import random
 import time
-from typing import cast
+from typing import Sized, cast
-from collections.abc import Sized
 
-from openpilot.common.test import OpenpilotTestCase
+import cereal.messaging as messaging
-import openpilot.cereal.messaging as messaging
+from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
-from openpilot.cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
+                                                  random_bytes, random_carstate, assert_carstate, \
-                                                  random_bytes, random_carstate, assert_carstate
+                                                  zmq_sleep
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
 
 
-class TestSubMaster(OpenpilotTestCase):
+class TestSubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
 
   def test_init(self):
     sm = messaging.SubMaster(events)
@@ -38,6 +42,7 @@ class TestSubMaster(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sm = messaging.SubMaster([sock,])
+    zmq_sleep()
 
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
@@ -49,6 +54,7 @@ class TestSubMaster(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sm = messaging.SubMaster([sock,])
+    zmq_sleep()
 
     for i in range(10):
       msg = messaging.new_message(sock)
@@ -60,17 +66,17 @@ class TestSubMaster(OpenpilotTestCase):
   def test_update_timeout(self):
     sock = random_sock()
     sm = messaging.SubMaster([sock,])
-    timeout = random.randrange(10, 30)
+    timeout = random.randrange(1000, 3000)
     start_time = time.monotonic()
     sm.update(timeout)
     t = time.monotonic() - start_time
     assert t >= timeout/1000.
-    assert t < 0.1
+    assert t < 3
     assert not any(sm.updated.values())
 
   def test_avg_frequency_checks(self):
     for poll in (True, False):
-      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "extrinsicsCalibration"],
+      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
                                poll=("modelV2" if poll else None),
                                frequency=(20. if not poll else None))
 
@@ -78,39 +84,50 @@ class TestSubMaster(OpenpilotTestCase):
         "carState": (20, 20),
         "modelV2": (20, 20 if poll else 10),
         "cameraOdometry": (20, 10),
-        "extrinsicsCalibration": (4, 4),
+        "liveCalibration": (4, 4),
         "carParams": (None, None),
         "userBookmark": (None, None),
       }
 
       for service, (max_freq, min_freq) in checks.items():
         if max_freq is not None:
-          assert min_freq is not None
           assert sm._check_avg_freq(service)
           assert sm.freq_tracker[service].max_freq == max_freq*1.2
           assert sm.freq_tracker[service].min_freq == min_freq*0.8
         else:
           assert not sm._check_avg_freq(service)
 
+  def test_alive(self):
+    pass
+
+  def test_ignore_alive(self):
+    pass
+
+  def test_valid(self):
+    pass
+
   # SubMaster should always conflate
   def test_conflate(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sm = messaging.SubMaster([sock,])
 
-    pub_sock.send(messaging.new_message(sock).to_bytes())
-    sm.update(1000)  # synchronize the PUB/SUB connection
-
     n = 10
     for i in range(n+1):
       msg = messaging.new_message(sock)
       msg.carState.vEgo = i
       pub_sock.send(msg.to_bytes())
+      time.sleep(0.01)
     sm.update(1000)
     assert sm[sock].vEgo == n
 
 
-class TestPubMaster(OpenpilotTestCase):
+class TestPubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
 
   def test_init(self):
     messaging.PubMaster(events)
@@ -119,6 +136,7 @@ class TestPubMaster(OpenpilotTestCase):
     socks = random_socks()
     pm = messaging.PubMaster(socks)
     sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
+    zmq_sleep()
 
     # PubMaster accepts either a capnp msg builder or bytes
     for capnp in [True, False]:

cereal/messaging/tests/test_services.py

@@ -0,0 +1,21 @@
+import os
+import tempfile
+from typing import Dict
+from parameterized import parameterized
+
+import cereal.services as services
+from cereal.services import SERVICE_LIST
+
+
+class TestServices:
+
+  @parameterized.expand(SERVICE_LIST.keys())
+  def test_services(self, s):
+    service = SERVICE_LIST[s]
+    assert service.frequency <= 104
+    assert service.decimation != 0
+
+  def test_generated_header(self):
+    with tempfile.NamedTemporaryFile(suffix=".h") as f:
+      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
+      assert ret == 0, "generated services header is not valid C"

cereal/messaging/tests/validate_sp_cereal_upstream.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+from typing import Any, List, Tuple
+
+DEBUG = False
+
+
+def print_debug(string: str) -> None:
+  if DEBUG:
+    print(string)
+
+
+def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
+  """
+  Create a new instance of a schema type, handling different field types.
+
+  Args:
+      struct: The Cap'n Proto schema structure
+      prop: A tuple containing the field name and field metadata
+
+  Returns:
+      A new initialized schema instance
+  """
+  struct_instance = struct.new_message()
+  field_name, field_metadata = prop
+
+  try:
+    field_type = field_metadata.proto.slot.type.which()
+
+    # Initialize different types of fields
+    if field_type in ('list', 'text', 'data'):
+      struct_instance.init(field_name, 1)
+      print_debug(f"Initialized list/text/data field: {field_name}")
+    elif field_type in ('struct', 'object'):
+      struct_instance.init(field_name)
+      print_debug(f"Initialized struct/object field: {field_name}")
+
+    return struct_instance
+
+  except Exception as e:
+    print(f"Error creating instance for {field_name}: {e}")
+    return None
+
+
+def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
+  """
+  Retrieve all fields from a given schema structure.
+
+  Args:
+      schema_struct: The Cap'n Proto schema structure
+
+  Returns:
+      A list of field names and their metadata
+  """
+  try:
+    # Get all fields from the schema
+    schema_fields = list(schema_struct.schema.fields.items())
+
+    print_debug("Discovered schema fields:")
+    for field_name, field_metadata in schema_fields:
+      print_debug(f"- {field_name}")
+
+    return schema_fields
+
+  except Exception as e:
+    print(f"Error retrieving schema fields: {e}")
+    return []
+
+
+def generate_schema_instances(schema_struct: Any) -> List[Any]:
+  """
+  Generate instances for all fields in a given schema.
+
+  Args:
+      schema_struct: The Cap'n Proto schema structure
+
+  Returns:
+      A list of schema instances
+  """
+  schema_fields = get_schema_fields(schema_struct)
+  instances = []
+
+  for field_prop in schema_fields:
+    try:
+      instance = create_schema_instance(schema_struct, field_prop)
+      if instance is not None:
+        instances.append(instance)
+    except Exception as e:
+      print(f"Skipping field due to error: {e}")
+
+  print(f"Generated {len(instances)} schema instances")
+  return instances
+
+
+def persist_instances(instances: List[Any], filename: str) -> None:
+  """
+  Write schema instances to a binary file.
+
+  Args:
+      instances: List of schema instances
+      filename: Output file path
+  """
+  try:
+    with open(filename, 'wb') as f:
+      for instance in instances:
+        f.write(instance.to_bytes())
+
+    print(f"Successfully wrote {len(instances)} instances to {filename}")
+
+  except Exception as e:
+    print(f"Error persisting instances: {e}")
+    sys.exit(1)
+
+
+def read_instances(filename: str, schema_type: Any) -> List[Any]:
+  """
+  Read schema instances from a binary file.
+
+  Args:
+      filename: Input file path
+      schema_type: The schema type to use for reading
+
+  Returns:
+      A list of read schema instances
+  """
+  try:
+    with open(filename, 'rb') as f:
+      data = f.read()
+
+    instances = list(schema_type.read_multiple_bytes(data))
+
+    print(f"Read {len(instances)} instances from {filename}")
+    return instances
+
+  except Exception as e:
+    print(f"Error reading instances: {e}")
+    sys.exit(1)
+
+
+def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
+  """
+  Compare original and read-back instances to detect potential breaking changes.
+
+  Args:
+      original_instances: List of originally generated instances
+      read_instances: List of instances read back from file
+
+  Returns:
+      Boolean indicating whether schemas appear compatible
+  """
+  if len(original_instances) != len(read_instances):
+    print("❌ Schema Compatibility Warning: Instance count mismatch")
+    return False
+
+  compatible = True
+  for struct in read_instances:
+    try:
+      getattr(struct, struct.which())  # Attempting to access the field to validate readability
+    except Exception as e:
+      print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
+      compatible = False
+
+  return compatible
+
+
+def main():
+  """
+  CLI entry point for schema compatibility testing.
+  """
+  # Setup argument parser
+  parser = argparse.ArgumentParser(
+    description='Cap\'n Proto Schema Compatibility Testing Tool',
+    epilog='Test schema compatibility by generating and reading back instances.'
+  )
+
+  # Add mutually exclusive group for generation or reading mode
+  mode_group = parser.add_mutually_exclusive_group(required=True)
+  mode_group.add_argument('-g', '--generate', action='store_true',
+                          help='Generate schema instances')
+  mode_group.add_argument('-r', '--read', action='store_true',
+                          help='Read and validate schema instances')
+
+  # Common arguments
+  parser.add_argument('-f', '--file',
+                      default='schema_instances.bin',
+                      help='Output/input binary file (default: schema_instances.bin)')
+
+  # Parse arguments
+  args = parser.parse_args()
+
+  # Import the schema dynamically 
+  try:
+    from cereal import log
+    schema_type = log.Event
+  except ImportError:
+    print("Error: Unable to import schema. Ensure 'cereal' is installed.")
+    sys.exit(1)
+
+  # Execute based on mode
+  if args.generate:
+    print("🔧 Generating Schema Instances")
+    instances = generate_schema_instances(schema_type)
+    persist_instances(instances, args.file)
+    print("✅ Instance generation complete")
+
+  elif args.read:
+    print("🔍 Reading and Validating Schema Instances")
+    generated_instances = generate_schema_instances(schema_type)
+    read_back_instances = read_instances(args.file, schema_type)
+
+    # Compare schemas
+    if compare_schemas(generated_instances, read_back_instances):
+      print("✅ Schema Compatibility: No breaking changes detected")
+      sys.exit(0)
+    else:
+      print("❌ Potential Schema Breaking Changes Detected")
+      sys.exit(1)
+
+
+if __name__ == "__main__":
+  main()

cereal/services.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 from enum import IntEnum
+from typing import Optional
 
 
 # TODO: this should be automatically determined using the capnp schema
@@ -10,7 +11,7 @@ class QueueSize(IntEnum):
 
 
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: int | None = None,
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
                queue_size: QueueSize = QueueSize.SMALL):
     self.should_log = should_log
     self.frequency = frequency
@@ -23,9 +24,11 @@ _services: dict[str, tuple] = {
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
   "accelerometer": (True, 104., 104),
+  "magnetometer": (True, 25.),
+  "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
+  "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "chestnutState": (True, 10., 10),
   "touch": (True, 20., 1),
   "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
   "controlsState": (True, 100., 10, QueueSize.MEDIUM),
@@ -33,37 +36,37 @@ _services: dict[str, tuple] = {
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
-  "narrowRoadEncodeIdx": (False, 20., 1),
+  "roadEncodeIdx": (False, 20., 1),
-  "radarTracks": (True, 20.),
+  "liveTracks": (True, 20.),
   "sendcan": (True, 100., 139, QueueSize.MEDIUM),
-  "logMessage": (True, 0., None, QueueSize.BIG),
+  "logMessage": (True, 0.),
-  "errorLogMessage": (True, 0., 1, QueueSize.BIG),
+  "errorLogMessage": (True, 0., 1),
-  "extrinsicsCalibration": (True, 4., 4),
+  "liveCalibration": (True, 4., 4),
-  "lateralTorqueParameters": (True, 4., 1),
+  "liveTorqueParameters": (True, 4., 1),
-  "lateralDelay": (True, 4., 1),
+  "liveDelay": (True, 4., 1),
-  "operatingSystemLog": (True, 0.),
+  "androidLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
-  "lateralManeuverPlan": (True, 20.),
   "driverAssistance": (True, 20., 20),
   "procLog": (True, 0.5, 15, QueueSize.BIG),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
   "qcomGnss": (True, 2.),
+  "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
-  "deviceMotion": (True, 20., 4),
+  "livePose": (True, 20., 4),
-  "vehicleParameters": (True, 20., 5),
+  "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 10),
   "thumbnail": (True, 1 / 60., 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
-  "narrowRoadCameraState": (True, 20., 20),
+  "roadCameraState": (True, 20., 20),
-  "cabinCameraState": (True, 20., 20),
+  "driverCameraState": (True, 20., 20),
-  "cabinEncodeIdx": (False, 20., 1),
+  "driverEncodeIdx": (False, 20., 1),
   "driverStateV2": (True, 20., 10),
   "driverMonitoringState": (True, 20., 10),
   "wideRoadEncodeIdx": (False, 20., 1),
@@ -71,15 +74,20 @@ _services: dict[str, tuple] = {
   "drivingModelData": (True, 20., 10),
   "modelV2": (True, 20., None, QueueSize.BIG),
   "managerState": (True, 2., 1),
-  "qNarrowRoadEncodeIdx": (False, 20.),
+  "uploaderState": (True, 0., 1),
+  "navInstruction": (True, 1., 10),
+  "navRoute": (True, 0.),
+  "navThumbnail": (True, 0.),
+  "qRoadEncodeIdx": (False, 20.),
   "userBookmark": (True, 0., 1),
   "soundPressure": (True, 10., 10),
   "rawAudioData": (False, 20.),
   "bookmarkButton": (True, 0., 1),
-  "narrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "audioFeedback": (True, 0., 1),
-  "cabinEncodeData": (False, 20., None, QueueSize.BIG),
+  "roadEncodeData": (False, 20., None, QueueSize.BIG),
+  "driverEncodeData": (False, 20., None, QueueSize.BIG),
   "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qNarrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
 
   # sunnypilot
   "modelManagerSP": (False, 1., 1, QueueSize.BIG),
@@ -99,12 +107,14 @@ _services: dict[str, tuple] = {
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
   "livestreamWideRoadEncodeIdx": (False, 20.),
-  "livestreamNarrowRoadEncodeIdx": (False, 20.),
+  "livestreamRoadEncodeIdx": (False, 20.),
-  "livestreamCabinEncodeIdx": (False, 20.),
+  "livestreamDriverEncodeIdx": (False, 20.),
   "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamNarrowRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamCabinEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
   "customReservedRawData0": (True, 0.),
+  "customReservedRawData1": (True, 0.),
+  "customReservedRawData2": (True, 0.),
 }
 SERVICE_LIST = {name: Service(*vals) for
                 idx, (name, vals) in enumerate(_services.items())}
@@ -124,7 +134,8 @@ def build_header():
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += f'  {{ "{k}", {{"{k}", {should_log}, {v.frequency:f}, {decimation:d}, {v.queue_size:d}}}}},\n'
+    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation, v.queue_size)
   h += "};\n"
 
   h += "#endif\n"

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -0,0 +1,29 @@
+Import('env', 'envCython', 'arch')
+
+common_libs = [
+  'params.cc',
+  'swaglog.cc',
+  'util.cc',
+  'ratekeeper.cc',
+  'clutil.cc',
+]
+
+_common = env.Library('common', common_libs, LIBS="json11")
+Export('_common')
+
+if GetOption('extras'):
+  env.Program('tests/test_common',
+              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+
+# Cython bindings
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
+
+Export('common_python')

common/api/__init__.py

@@ -18,8 +18,8 @@ class Api:
     return self.service.get_token(payload_extra, expiry_hours)
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
-  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
+  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, **params)
 
 
 def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:

common/api/base.py

@@ -3,8 +3,8 @@ import os
 import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
-from openpilot.common.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
-from openpilot.common.version import get_version
+from openpilot.system.version import get_version
 
 # name: jwt signature algorithm
 KEYS = {"id_rsa": "RS256",
@@ -38,7 +38,6 @@ class BaseApi:
     }
     if payload_extra is not None:
       payload.update(payload_extra)
-    assert self.private_key is not None
     token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
     if isinstance(token, bytes):
       token = token.decode('utf8')
@@ -52,7 +51,7 @@ class BaseApi:
     ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
     return ascii_encoded_text.decode()
 
-  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, json=None, **params):
     headers = {}
     if access_token is not None:
       headers['Authorization'] = "JWT " + access_token
@@ -60,9 +59,7 @@ class BaseApi:
     version = self.remove_non_ascii_chars(get_version())
     headers['User-Agent'] = self.user_agent + version
 
-    # TODO: add session to Api
+    return requests.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
-    req = requests if session is None else session
-    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
 
   @staticmethod
   def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:

common/basedir.py

@@ -1,4 +1,4 @@
 import os
 
 
-BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))

common/clutil.cc

@@ -0,0 +1,98 @@
+#include "common/clutil.h"
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+
+#include "common/util.h"
+#include "common/swaglog.h"
+
+namespace {  // helper functions
+
+template <typename Func, typename Id, typename Name>
+std::string get_info(Func get_info_func, Id id, Name param_name) {
+  size_t size = 0;
+  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
+  std::string info(size, '\0');
+  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
+  return info;
+}
+inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
+inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
+
+void cl_print_info(cl_platform_id platform, cl_device_id device) {
+  size_t work_group_size = 0;
+  cl_device_type device_type = 0;
+  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
+  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
+  const char *type_str = "Other...";
+  switch (device_type) {
+    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
+    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
+    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
+  }
+
+  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
+  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
+  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
+  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
+  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
+  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
+  LOGD("max work group size: %zu", work_group_size);
+  LOGD("type = %d, %s", (int)device_type, type_str);
+}
+
+void cl_print_build_errors(cl_program program, cl_device_id device) {
+  cl_build_status status;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
+  size_t log_size;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
+  std::string log(log_size, '\0');
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
+
+  LOGE("build failed; status=%d, log: %s", status, log.c_str());
+}
+
+}  // namespace
+
+cl_device_id cl_get_device_id(cl_device_type device_type) {
+  cl_uint num_platforms = 0;
+  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
+  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
+  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
+
+  for (size_t i = 0; i < num_platforms; ++i) {
+    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
+
+    // Get first device
+    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
+      cl_print_info(platform_ids[i], device_id);
+      return device_id;
+    }
+  }
+  LOGE("No valid openCL platform found");
+  assert(0);
+  return nullptr;
+}
+
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
+void cl_release_context(cl_context context) {
+  clReleaseContext(context);
+}
+
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
+  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
+}
+
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
+  const char *csrc = src.c_str();
+  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
+  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
+    cl_print_build_errors(prg, device_id);
+    assert(0);
+  }
+  return prg;
+}

common/clutil.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#include <string>
+
+#define CL_CHECK(_expr)          \
+  do {                           \
+    assert(CL_SUCCESS == (_expr)); \
+  } while (0)
+
+#define CL_CHECK_ERR(_expr)           \
+  ({                                  \
+    cl_int err = CL_INVALID_VALUE;    \
+    __typeof__(_expr) _ret = _expr;   \
+    assert(_ret&& err == CL_SUCCESS); \
+    _ret;                             \
+  })
+
+cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
+void cl_release_context(cl_context context);
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/filter_simple.py

@@ -28,7 +28,7 @@ class BounceFilter(FirstOrderFilter):
     scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
     self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
     self.velocity.update(0.0)
-    if abs(self.velocity.x) < 1e-3:
+    if abs(self.velocity.x) < 1e-5:
       self.velocity.x = 0.0
     self.x += self.velocity.x
     return self.x

common/git.py

@@ -4,30 +4,39 @@ from openpilot.common.utils import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str | None = None) -> str:
+def get_short_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str | None = None) -> str:
+def get_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str | None = None) -> str:
+def get_origin(cwd: str = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
     return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
   except subprocess.CalledProcessError:  # Not on a branch, fallback
     return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
+
+
+@cache
+def get_normalized_origin(cwd: str = None) -> str:
+  return get_origin(cwd) \
+    .replace("git@", "", 1) \
+    .replace(".git", "", 1) \
+    .replace("https://", "", 1) \
+    .replace(":", "/", 1)

common/mat.h

@@ -0,0 +1,85 @@
+#pragma once
+
+typedef struct vec3 {
+  float v[3];
+} vec3;
+
+typedef struct vec4 {
+  float v[4];
+} vec4;
+
+typedef struct mat3 {
+  float v[3*3];
+} mat3;
+
+typedef struct mat4 {
+  float v[4*4];
+} mat4;
+
+static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
+  mat3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      float v = 0.0;
+      for (int k=0; k<3; k++) {
+        v += a.v[r*3+k] * b.v[k*3+c];
+      }
+      ret.v[r*3+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
+  vec3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      ret.v[r] += a.v[r*3+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+static inline mat4 matmul(const mat4 &a, const mat4 &b) {
+  mat4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      float v = 0.0;
+      for (int k=0; k<4; k++) {
+        v += a.v[r*4+k] * b.v[k*4+c];
+      }
+      ret.v[r*4+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
+  vec4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      ret.v[r] += a.v[r*4+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+// scales the input and output space of a transformation matrix
+// that assumes pixel-center origin.
+static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
+  // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
+
+  mat3 transform_out = {{
+    1.0f/s, 0.0f, 0.5f,
+    0.0f, 1.0f/s, 0.5f,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  mat3 transform_in = {{
+    s,  0.0f, -0.5f*s,
+    0.0f, s, -0.5f*s,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  return matmul3(transform_in, matmul3(in, transform_out));
+}

common/mock/__init__.py

@@ -1,19 +1,19 @@
 """
 Utilities for generating mock messages for testing.
-example in openpilot/common/tests/test_mock.py
+example in common/tests/test_mock.py
 """
 
 
 import functools
 import threading
-from openpilot.cereal.messaging import PubMaster
+from cereal.messaging import PubMaster
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
-from openpilot.common.mock.generators import generate_deviceMotion
+from openpilot.common.mock.generators import generate_livePose
 from openpilot.common.realtime import Ratekeeper
 
 
 MOCK_GENERATOR = {
-  "deviceMotion": generate_deviceMotion
+  "livePose": generate_livePose
 }
 
 

common/mock/generators.py

@@ -0,0 +1,14 @@
+from cereal import messaging
+
+
+def generate_livePose():
+  msg = messaging.new_message('livePose')
+  meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
+  msg.livePose.orientationNED = meas
+  msg.livePose.velocityDevice = meas
+  msg.livePose.angularVelocityDevice = meas
+  msg.livePose.accelerationDevice = meas
+  msg.livePose.inputsOK = True
+  msg.livePose.posenetOK = True
+  msg.livePose.sensorsOK = True
+  return msg

common/model.h

@@ -0,0 +1 @@
+#define DEFAULT_MODEL "Dark Souls 2 (Default)"

common/params.cc

@@ -12,7 +12,7 @@
 #include "common/queue.h"
 #include "common/swaglog.h"
 #include "common/util.h"
-#include "common/hardware/hw.h"
+#include "system/hardware/hw.h"
 
 namespace {
 

common/params.py

@@ -0,0 +1,19 @@
+from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+assert Params
+assert ParamKeyFlag
+assert ParamKeyType
+assert UnknownKeyName
+
+if __name__ == "__main__":
+  import sys
+
+  params = Params()
+  key = sys.argv[1]
+  assert params.check_key(key), f"unknown param: {key}"
+
+  if len(sys.argv) == 3:
+    val = sys.argv[2]
+    print(f"SET: {key} = {val}")
+    params.put(key, val)
+  elif len(sys.argv) == 2:
+    print(f"GET: {key} = {params.get(key)}")

common/params_keys.h

@@ -3,7 +3,7 @@
 #include <string>
 #include <unordered_map>
 
-#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "cereal/gen/cpp/log.capnp.h"
 
 inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
@@ -37,7 +37,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
     {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
     {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"DriverLockoutCount", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, INT, "0"}},
     {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
     {"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
     {"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
@@ -59,11 +58,12 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsEngaged", {PERSISTENT, BOOL}},
     {"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
-    {"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsMetric", {PERSISTENT | BACKUP, BOOL}},
     {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsOnroad", {PERSISTENT, BOOL}},
     {"IsRhdDetected", {PERSISTENT, BOOL}},
     {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
     {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
@@ -80,21 +80,19 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
     {"LiveParameters", {PERSISTENT, JSON}},
     {"LiveParametersV2", {PERSISTENT, BYTES}},
-    {"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
-    {"LivestreamRequestKeyframe", {CLEAR_ON_MANAGER_START | DONT_LOG, BOOL}},
     {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
     {"LocationFilterInitialState", {PERSISTENT, BYTES}},
-    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
     {"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
     {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
     {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
     {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_ChestnutBranch", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
+    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
     {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
     {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
@@ -104,8 +102,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
     {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
     {"PrimeType", {PERSISTENT, INT}},
     {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
+    {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RecordFront", {PERSISTENT | BACKUP, BOOL}},
     {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
     {"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
@@ -114,7 +115,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
     {"TermsVersion", {PERSISTENT, STRING}},
-    {"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TrainingVersion", {PERSISTENT, STRING}},
     {"UbloxAvailable", {PERSISTENT, BOOL}},
     {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
@@ -130,15 +130,12 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
     {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //
     {"ApiCache_DriveStats", {PERSISTENT, JSON}},
     {"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
-    {"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}},  // seconds
     {"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}},  // MPH or km/h
     {"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
     {"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
@@ -148,7 +145,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"CarParamsSPPersistent", {PERSISTENT, BYTES}},
     {"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
     {"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
-    {"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
     {"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
     {"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
@@ -158,7 +154,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
     {"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
-    {"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
     {"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
     {"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
@@ -171,17 +166,12 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
     {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
     {"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
-    {"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
+    {"OnroadScreenOffControl", {PERSISTENT | BACKUP, BOOL}},
     {"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
-    {"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
     {"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RoadEdgeLaneChangeEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ScreenSaverEnabled", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"ScreenSaverTimeout", {PERSISTENT | BACKUP, INT, "300"}},
     {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -195,14 +185,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
 
     // Model Manager params
     {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
-    {"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}},
     {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
+    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT, "0"}},
     {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
     {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
-    {"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
     {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
-    {"ModelManager_ModelsCache_USBGPU", {PERSISTENT | BACKUP, JSON}},
 
     // Neural Network Lateral Control
     {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -210,7 +197,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     // sunnylink params
     {"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
     {"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"ParamsVersion", {PERSISTENT, INT}},
     {"SunnylinkCache_Roles", {PERSISTENT, STRING}},
     {"SunnylinkCache_Users", {PERSISTENT, STRING}},
     {"SunnylinkDongleId", {PERSISTENT, STRING}},
@@ -227,21 +213,17 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TeslaMadsScreenButton", {PERSISTENT | BACKUP, INT, "0"}},
     {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     {"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
 
     // sunnypilot model params
-    {"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
     {"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
     {"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
     {"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
-    {"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
 
     // mapd
     {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
@@ -262,7 +244,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OsmStateTitle", {PERSISTENT, STRING}},
     {"OsmWayTest", {PERSISTENT, STRING}},
     {"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RoadNameToggle", {PERSISTENT, STRING}},
 
     // Speed Limit
     {"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
@@ -278,10 +260,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     // Torque lateral control custom params
     {"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
     {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
-    {"LateralJerkTorqueController", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
     {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
-    {"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
     {"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
     {"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},

common/params_pyx.pyx

@@ -0,0 +1,197 @@
+# distutils: language = c++
+# cython: language_level = 3
+import builtins
+import datetime
+import json
+from libcpp cimport bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.optional cimport optional
+
+from openpilot.common.swaglog import cloudlog
+
+cdef extern from "common/params.h":
+  cpdef enum ParamKeyFlag:
+    PERSISTENT
+    CLEAR_ON_MANAGER_START
+    CLEAR_ON_ONROAD_TRANSITION
+    CLEAR_ON_OFFROAD_TRANSITION
+    DEVELOPMENT_ONLY
+    CLEAR_ON_IGNITION_ON
+    BACKUP
+    ALL
+
+  cpdef enum ParamKeyType:
+    STRING
+    BOOL
+    INT
+    FLOAT
+    TIME
+    JSON
+    BYTES
+
+  cdef cppclass c_Params "Params":
+    c_Params(string) except + nogil
+    string get(string, bool) nogil
+    bool getBool(string, bool) nogil
+    int remove(string) nogil
+    int put(string, string) nogil
+    void putNonBlocking(string, string) nogil
+    void putBoolNonBlocking(string, bool) nogil
+    int putBool(string, bool) nogil
+    bool checkKey(string) nogil
+    ParamKeyType getKeyType(string) nogil
+    optional[string] getKeyDefaultValue(string) nogil
+    string getParamPath(string) nogil
+    void clearAll(ParamKeyFlag)
+    vector[string] allKeys(ParamKeyFlag)
+
+PYTHON_2_CPP = {
+  (str, STRING): lambda v: v,
+  (builtins.bool, BOOL): lambda v: "1" if v else "0",
+  (int, INT): str,
+  (float, FLOAT): str,
+  (datetime.datetime, TIME): lambda v: v.isoformat(),
+  (dict, JSON): json.dumps,
+  (list, JSON): json.dumps,
+  (bytes, BYTES): lambda v: v,
+}
+CPP_2_PYTHON = {
+  STRING: lambda v: v.decode("utf-8"),
+  BOOL: lambda v: v == b"1",
+  INT: int,
+  FLOAT: float,
+  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
+  JSON: json.loads,
+  BYTES: lambda v: v,
+}
+
+def ensure_bytes(v):
+  return v.encode() if isinstance(v, str) else v
+
+class UnknownKeyName(Exception):
+  pass
+
+cdef class Params:
+  cdef c_Params* p
+  cdef str d
+
+  def __cinit__(self, d=""):
+    cdef string path = <string>d.encode()
+    with nogil:
+      self.p = new c_Params(path)
+    self.d = d
+
+  def __reduce__(self):
+    return (type(self), (self.d,))
+
+  def __dealloc__(self):
+    del self.p
+
+  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
+    self.p.clearAll(tx_flag)
+
+  def check_key(self, key):
+    key = ensure_bytes(key)
+    if not self.p.checkKey(key):
+      raise UnknownKeyName(key)
+    return key
+
+  def python2cpp(self, proposed_type, expected_type, value, key):
+    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
+    if cast:
+      return cast(value)
+    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
+
+  def _cpp2python(self, t, value, default, key):
+    if value is None:
+      return None
+    try:
+      return CPP_2_PYTHON[t](value)
+    except (KeyError, TypeError, ValueError):
+      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
+      return self._cpp2python(t, default, None, key)
+
+  def get(self, key, bool block=False, bool return_default=False):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    cdef string val
+    with nogil:
+      val = self.p.get(k, block)
+
+    default_val = (default.value() if default.has_value() else None) if return_default else None
+    if val == b"":
+      if block:
+        # If we got no value while running in blocked mode
+        # it means we got an interrupt while waiting
+        raise KeyboardInterrupt
+      else:
+        return self._cpp2python(t, default_val, None, key)
+    return self._cpp2python(t, val, default_val, key)
+
+  def get_bool(self, key, bool block=False):
+    cdef string k = self.check_key(key)
+    cdef bool r
+    with nogil:
+      r = self.p.getBool(k, block)
+    return r
+
+  def _put_cast(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
+
+  def put(self, key, dat):
+    """
+    Warning: This function blocks until the param is written to disk!
+    In very rare cases this can take over a second, and your code will hang.
+    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
+    in general try to avoid writing params as much as possible.
+    """
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = self._put_cast(key, dat)
+    with nogil:
+      self.p.put(k, dat_bytes)
+
+  def put_bool(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBool(k, val)
+
+  def put_nonblocking(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = self._put_cast(key, dat)
+    with nogil:
+      self.p.putNonBlocking(k, dat_bytes)
+
+  def put_bool_nonblocking(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBoolNonBlocking(k, val)
+
+  def remove(self, key):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.remove(k)
+
+  def get_param_path(self, key=""):
+    cdef string key_bytes = ensure_bytes(key)
+    return self.p.getParamPath(key_bytes).decode("utf-8")
+
+  def get_type(self, key):
+    return self.p.getKeyType(self.check_key(key))
+
+  def all_keys(self, flag=ParamKeyFlag.ALL):
+    return self.p.allKeys(flag)
+
+  def get_default_value(self, key):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
+
+  def cpp2python(self, key, value):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return self._cpp2python(t, value, None, key)

common/pid.py

@@ -1,13 +1,17 @@
 import numpy as np
-from collections.abc import Sequence
+from numbers import Number
-
-Gain = int | float | tuple[Sequence[float], Sequence[float]] | list[list[float]]
 
 class PIDController:
-  def __init__(self, k_p: Gain, k_i: Gain, k_d: Gain = 0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+  def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
-    self._k_p = ([0], [k_p]) if isinstance(k_p, (int, float)) else k_p
+    self._k_p = k_p
-    self._k_i = ([0], [k_i]) if isinstance(k_i, (int, float)) else k_i
+    self._k_i = k_i
-    self._k_d = ([0], [k_d]) if isinstance(k_d, (int, float)) else k_d
+    self._k_d = k_d
+    if isinstance(self._k_p, Number):
+      self._k_p = [[0], [self._k_p]]
+    if isinstance(self._k_i, Number):
+      self._k_i = [[0], [self._k_i]]
+    if isinstance(self._k_d, Number):
+      self._k_d = [[0], [self._k_d]]
 
     self.set_limits(pos_limit, neg_limit)
 

common/prefix.h

@@ -5,7 +5,7 @@
 
 #include "common/params.h"
 #include "common/util.h"
-#include "common/hardware/hw.h"
+#include "system/hardware/hw.h"
 
 class OpenpilotPrefix {
 public:
@@ -13,11 +13,7 @@ public:
     if (prefix.empty()) {
       prefix = util::random_string(15);
     }
-#ifdef __APPLE__
+    msgq_path = Path::shm_path() + "/" + prefix;
-    msgq_path = "/tmp/msgq_" + prefix;
-#else
-    msgq_path = "/dev/shm/msgq_" + prefix;
-#endif
     bool ret = util::create_directories(msgq_path, 0777);
     assert(ret);
     setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
@@ -27,14 +23,14 @@ public:
     auto param_path = Params().getParamPath();
     if (util::file_exists(param_path)) {
       std::string real_path = util::readlink(param_path);
-      util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
+      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
       unlink(param_path.c_str());
     }
     if (getenv("COMMA_CACHE") == nullptr) {
-      util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
+      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
     }
-    util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
+    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
-    util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
+    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
     unsetenv("OPENPILOT_PREFIX");
   }
 

common/prefix.py

@@ -1,19 +1,17 @@
 import os
-import platform
 import shutil
 import uuid
 
 
 from openpilot.common.params import Params
-from openpilot.common.hardware import PC
+from openpilot.system.hardware import PC
-from openpilot.common.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
-from openpilot.common.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
-  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+  def __init__(self, prefix: str = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
     self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
+    self.msgq_path = os.path.join(Paths.shm_path(), "msgq_" + self.prefix)
-    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
     self.create_dirs_on_enter = create_dirs_on_enter
     self.clean_dirs_on_exit = clean_dirs_on_exit
     self.shared_download_cache = shared_download_cache

common/ratekeeper.cc

@@ -6,9 +6,9 @@
 #include "common/timing.h"
 #include "common/util.h"
 
-RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
-    : name(name_),
+    : name(name),
-      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
   interval = 1 / rate;
   last_monitor_time = seconds_since_boot();
   next_frame_time = last_monitor_time + interval;

common/realtime.py

@@ -6,8 +6,8 @@ import time
 
 from setproctitle import getproctitle
 
-from openpilot.common.utils import MovingAverage
+from openpilot.common.util import MovingAverage
-from openpilot.common.hardware import PC
+from openpilot.system.hardware import PC
 
 
 # time step for each process
@@ -28,11 +28,6 @@ class Priority:
   CTRL_HIGH = 53
 
 
-def drop_realtime() -> None:
-  if sys.platform == 'linux' and not PC:
-    os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
-
-
 def set_core_affinity(cores: list[int]) -> None:
   if sys.platform == 'linux' and not PC:
     os.sched_setaffinity(0, cores)

common/spinner.py

@@ -8,7 +8,7 @@ class Spinner:
     try:
       self.spinner_proc = subprocess.Popen(["./spinner.py"],
                                            stdin=subprocess.PIPE,
-                                           cwd=os.path.join(BASEDIR, "openpilot/system", "ui"),
+                                           cwd=os.path.join(BASEDIR, "system", "ui"),
                                            close_fds=True)
     except OSError:
       self.spinner_proc = None