clustermaps

This reverts commit 15e5d2efb9.

.gitattributes

@@ -10,5 +10,8 @@
 *.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
-openpilot/selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
+selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-openpilot/common/hardware/comma/updater filter=lfs diff=lfs merge=lfs -text
+common/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+
+# Small vendored Leaflet UI icons are kept directly in Git.
+tools/clustermaps/web/vendor/leaflet/images/*.png -filter -diff -merge -text

.github/labeler.yaml

@@ -8,24 +8,24 @@ chore:
 
 car:
   - changed-files:
-    - any-glob-to-all-files: '{openpilot/selfdrive/car/**,opendbc_repo}'
+    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
 
 simulation:
   - changed-files:
-    - any-glob-to-all-files: 'openpilot/tools/sim/**'
+    - any-glob-to-all-files: 'tools/sim/**'
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{openpilot/selfdrive/assets/**,openpilot/selfdrive/ui/**,openpilot/system/ui/**}'
+    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
 
 tools:
   - changed-files:
-    - any-glob-to-all-files: 'openpilot/tools/**'
+    - any-glob-to-all-files: 'tools/**'
 
 multilanguage:
   - changed-files:
-    - any-glob-to-all-files: 'openpilot/selfdrive/ui/translations/**'
+    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
 
 autonomy:
   - changed-files:
-    - any-glob-to-all-files: "{openpilot/selfdrive/modeld/models/**,openpilot/selfdrive/test/process_replay/model_replay_ref_commit,openpilot/sunnypilot/modeld*/models/**}"
+    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit,sunnypilot/modeld*/models/**}"

.github/pull_request_template.md

@@ -0,0 +1,68 @@
+<!-- Please copy and paste the relevant template -->
+
+<!--- ***** Template: Fingerprint *****
+
+**Car**
+Which car (make, model, year) this fingerprint is for
+
+**Route**
+A route with the fingerprint
+
+-->
+
+<!--- ***** Template: Car Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+**Route**
+
+Route: [a route with the bug fix]
+
+
+-->
+
+<!--- ***** Template: Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+
+-->
+
+<!--- ***** Template: Car Port *****
+
+**Checklist**
+
+- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
+- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
+- [ ] route with openpilot:
+- [ ] route with stock system:
+- [ ] car harness used (if comma doesn't sell it, put N/A):
+
+
+-->
+
+<!--- ***** Template: Refactor *****
+
+**Description**
+
+A description of the refactor, including the goals it accomplishes. 
+
+**Verification**
+
+Explain how you tested the refactor for regressions. 
+
+
+-->
+

.github/workflows/auto_pr_review.yaml

@@ -12,7 +12,7 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
       with:
         submodules: false
 
@@ -24,7 +24,7 @@ jobs:
 
     # Check PR target branch
     - name: check branch
-      uses: Vankka/pr-target-branch-action@5da68a42bcb7b43d39104295a876a6f3f8d7908b
+      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
       if: github.repository == 'sunnypilot/sunnypilot'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/badges.yaml

@@ -0,0 +1,35 @@
+name: badges
+on:
+  schedule:
+    - cron: '0 * * * *'
+  workflow_dispatch:
+
+env:
+  PYTHONPATH: ${{ github.workspace }}
+
+jobs:
+  badges:
+    name: create badges
+    runs-on: ubuntu-latest
+    if: github.repository == 'sunnypilot/sunnypilot'
+    permissions:
+      contents: write
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Push badges
+      run: |
+        python3 selfdrive/ui/translations/create_badges.py
+
+        rm .gitattributes
+
+        git checkout --orphan badges
+        git rm -rf --cached .
+        git config user.email "badge-researcher@sunnypilot.ai"
+        git config user.name "Badge Researcher"
+
+        git add translation_badge.svg
+        git commit -m "Add/Update badges"
+        git push -f origin HEAD

.github/workflows/build-all-tinygrad-models.yaml

@@ -7,19 +7,6 @@ on:
         description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
         required: true
         type: string
-      target_hardware:
-        description: 'Hardware target to compile for (qcom or usbgpu)'
-        required: true
-        type: choice
-        default: 'qcom'
-        options:
-          - qcom
-          - usbgpu
-      hf_repo:
-        description: 'Hugging Face dataset repository'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot_models_v1'
 
 jobs:
   setup:
@@ -43,7 +30,7 @@ jobs:
         run: |
           cd sunnypilot
           export PYTHONPATH=$(pwd)
-          ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
+          ref=$(python3 sunnypilot/models/tinygrad_ref.py)
           echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
           echo "tinygrad_ref is $ref"
 
@@ -59,14 +46,13 @@ jobs:
         id: get-json
         run: |
           cd docs/docs
-          PREFIX="driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_' || '' }}v"
+          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
-          latest=$(ls ${PREFIX}*.json | sed -E "s/${PREFIX}([0-9]+)\.json/\1/" | sort -n | tail -1)
           next=$((latest+1))
-          json_file="${PREFIX}${next}.json"
+          json_file="driving_models_v${next}.json"
-          cp "${PREFIX}${latest}.json" "$json_file"
+          cp "driving_models_v${latest}.json" "$json_file"
           echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
           echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
-          echo "SRC_JSON_FILE=docs/docs/${PREFIX}${latest}.json" >> $GITHUB_ENV
+          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
 
       - name: Extract tinygrad models
         id: set-matrix
@@ -75,23 +61,45 @@ jobs:
           jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
           echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
-      - name: Get next recompiled dir number
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+      - run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      - name: Clone GitLab docs repo and create new recompiled dir
         id: create-recompiled-dir
         env:
-          HF_REPO: ${{ github.event.inputs.hf_repo }}
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
-          pip install huggingface_hub
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          recompiled_dir=$(python3 -c "
+          cd gitlab_docs
-          from huggingface_hub import HfApi
+          git checkout main
-          import re, sys
+          git sparse-checkout set --no-cone models/
-          api = HfApi()
+          cd models
-          files = api.list_repo_files(repo_id=sys.argv[1], repo_type='dataset')
+          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          dirs = [re.search(r'models/recompiled([0-9]+)', f) for f in files]
+          if [[ -z "$latest_dir" ]]; then
-          nums = [int(m.group(1)) for m in dirs if m]
+            next_dir=1
-          print(max(nums) + 1)
+          else
-          " "$HF_REPO")
+            next_dir=$((latest_dir+1))
+          fi
+          recompiled_dir="${next_dir}"
+          mkdir -p "recompiled${recompiled_dir}"
+          touch "recompiled${recompiled_dir}/.gitkeep"
+          cd ../..
           echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
 
+      - name: Push empty recompiled dir to GitLab
+        run: |
+          cd gitlab_docs
+          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
+          git push origin main
+
       - name: Push new JSON to GitHub docs repo
         run: |
           cd docs
@@ -115,30 +123,25 @@ jobs:
       is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
-      target_hardware: ${{ github.event.inputs.target_hardware }}
-      hf_repo: ${{ github.event.inputs.hf_repo }}
-      set_min_version: ${{ github.event.inputs.set_min_version }}
-      tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
     secrets: inherit
 
   retry_failed_models:
     needs: [setup, get_and_build]
     runs-on: ubuntu-latest
-    if: ${{ !cancelled() && needs.setup.result == 'success' && (needs.get_and_build.result == 'success' || needs.get_and_build.result == 'failure') }}
+    if: ${{ needs.setup.result != 'failure' && !cancelled() }}
     outputs:
       retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
     steps:
       - uses: actions/download-artifact@v4
         with:
-          pattern: artifact-name-*
+          pattern: model-*
           path: output
-        continue-on-error: true
 
       - id: set-retry-matrix
         run: |
           echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
-          built=(); while IFS= read -r line; do [ -n "$line" ] && built+=("$line"); done < <(
+          built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'artifact-name-*' -printf "%f\n" 2>/dev/null | sed -E 's/^artifact-name-//' | awk '{gsub(/^ +| +$/, ""); print}'
+            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
           )
           jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
             'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
@@ -146,7 +149,7 @@ jobs:
 
   retry_get_and_build:
     needs: [setup, get_and_build, retry_failed_models]
-    if: ${{ !cancelled() && needs.retry_failed_models.result == 'success' && needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '' }}
+    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
     strategy:
       matrix:
         model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
@@ -158,9 +161,146 @@ jobs:
       is_20hz: ${{ matrix.model.is_20hz }}
       recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
       json_version: ${{ needs.setup.outputs.json_version }}
-      target_hardware: ${{ github.event.inputs.target_hardware }}
       artifact_suffix: -retry
-      hf_repo: ${{ github.event.inputs.hf_repo }}
-      set_min_version: ${{ github.event.inputs.set_min_version }}
-      tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
     secrets: inherit
+
+  publish_models:
+    name: Publish models sequentially
+    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
+    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
+    env:
+      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
+      JSON_FILE: ${{ needs.setup.outputs.json_file }}
+      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
+    steps:
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+
+      - name: Add GitLab.com SSH key to known_hosts
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      - name: Clone GitLab docs repo
+        env:
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+        run: |
+          echo "Cloning GitLab"
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          cd gitlab_docs
+          echo "checkout models/${RECOMPILED_DIR}"
+          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
+          git checkout main
+          cd ..
+
+      - name: Checkout docs repo
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/sunnypilot-models
+          ref: gh-pages
+          path: docs
+          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
+
+      - name: Validate recompiled dir and JSON version
+        run: |
+          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
+            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
+            exit 1
+          fi
+          if [ ! -f "$JSON_FILE" ]; then
+            echo "JSON file $JSON_FILE does not exist!"
+            exit 1
+          fi
+
+      - name: Download artifact name file
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
+          path: artifact_name
+
+      - name: Read artifact name
+        id: read-artifact-name
+        run: |
+          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+
+      - name: Download model artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
+          path: output
+
+      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
+        run: |
+          find output -type f -name '*.onnx' -delete
+          find output -type f -name 'big_*.pkl' -delete
+          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
+
+      - name: Copy model artifacts to gitlab
+        env:
+          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+        run: |
+          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          mkdir -p "$ARTIFACT_DIR"
+          for path in output/*; do
+            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
+              continue
+            fi
+            name="$(basename "$path")"
+            if [ -d "$path" ]; then
+              mkdir -p "$ARTIFACT_DIR/$name"
+              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
+              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
+            else
+              cp "$path" "$ARTIFACT_DIR/"
+              echo "Copied file $name -> $ARTIFACT_DIR/"
+            fi
+          done
+
+      - name: Push recompiled dir to GitLab
+        env:
+          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+        run: |
+          cd gitlab_docs
+          git checkout main
+          git pull origin main
+          for d in models/"$RECOMPILED_DIR"/*/; do
+            git sparse-checkout add "$d"
+          done
+          git add models/"$RECOMPILED_DIR"
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
+          git push origin main
+      - run: |
+          cd docs
+          git pull origin gh-pages
+
+      - name: update json
+        run: |
+          ARGS=""
+          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
+          ARGS="$ARGS --sort-by-date"
+          ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
+          eval python3 docs/json_parser.py \
+            --json-path "$JSON_FILE" \
+            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
+            $ARGS
+
+      - name: Push updated json to GitHub
+        run: |
+          cd docs
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git checkout gh-pages
+          git add docs/"$(basename $JSON_FILE)"
+          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
+          git push origin gh-pages

.github/workflows/build-default-big-model.yaml

@@ -1,78 +0,0 @@
-name: Build default big model
-
-on:
-  workflow_dispatch:
-
-env:
-  HF_REPO: sunnypilot/sunnypilot_models_v1
-  HF_DEFAULTS_PATH: models/defaults/big
-
-jobs:
-  build_model:
-    uses: ./.github/workflows/sunnypilot-build-model.yaml
-    with:
-      upstream_branch: ${{ github.sha }}
-      custom_name: default-big-model
-      target_hardware: usbgpu
-    secrets: inherit
-
-  upload_defaults:
-    needs: build_model
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
-
-      - name: Install huggingface_hub
-        run: pip install --upgrade "huggingface_hub>=0.22.0"
-
-      - name: Download artifact name
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-default-big-model
-          path: artifact_name
-
-      - name: Read artifact name
-        id: artifact
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.artifact.outputs.artifact_name }}
-          path: output
-
-      - name: Upload model to HF defaults
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
-          ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
-        run: |
-          rm -f output/artifact_name.txt
-          hf upload ${{ env.HF_REPO }} \
-            output/ \
-            "${HF_DEFAULTS_PATH}/${ARTIFACT_NAME}/" \
-            --repo-type=dataset
-
-      - name: Get tinygrad ref and ONNX hash
-        id: meta
-        run: |
-          export PYTHONPATH=$(pwd)
-          echo "tinygrad_ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" >> $GITHUB_OUTPUT
-          echo "onnx_sha256=$(sha256sum openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx | cut -d' ' -f1)" >> $GITHUB_OUTPUT
-
-      - name: Update default_models.json on HF
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
-          ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
-        run: |
-          python3 release/ci/upload_default_model.py \
-            --hf-repo "${{ env.HF_REPO }}" \
-            --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \
-            --artifact-name "$ARTIFACT_NAME" \
-            --metadata-path "output/metadata.json" \
-            --onnx-sha256 "${{ steps.meta.outputs.onnx_sha256 }}" \
-            --tinygrad-ref "${{ steps.meta.outputs.tinygrad_ref }}"

.github/workflows/build-single-tinygrad-model.yaml

@@ -12,11 +12,11 @@ on:
         required: false
         type: string
       recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 1 for recompiled1)'
+        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
         required: true
         type: string
       json_version:
-        description: 'driving_models version number to update (e.g. 18 for driving_models_v18.json)'
+        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
         required: true
         type: string
       artifact_suffix:
@@ -29,24 +29,11 @@ on:
         required: false
         type: boolean
         default: true
-      target_hardware:
+      bypass_push:
-        description: 'Hardware target to compile for (qcom or usbgpu)'
+        description: 'Bypass pushing to GitLab for build-all'
         required: false
-        type: string
+        default: true
-        default: 'qcom'
+        type: boolean
-      hf_repo:
-        description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot_models_v1'
-      set_min_version:
-        description: 'Minimum selector version'
-        required: false
-        type: string
-      tinygrad_ref:
-        description: 'Tinygrad reference'
-        required: false
-        type: string
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -76,11 +63,12 @@ on:
         default: 'None'
         options:
           - None
-          - Master Models
+          - Simple Plan Models
-          - Release Models
+          - Space Lab Models
-          - 2026 World Models
+          - TR Models
-          - 2026 Deep RL Models
+          - DTR Models
           - Custom Merge Models
+          - FOF series models
           - Other
       custom_model_folder:
         description: 'Custom model folder name (if "Other" selected)'
@@ -94,22 +82,9 @@ on:
         description: 'Minimum selector version'
         required: false
         type: string
-      target_hardware:
-        description: 'Hardware target to compile for'
-        required: false
-        type: choice
-        default: 'qcom'
-        options:
-          - qcom
-          - usbgpu
-      hf_repo:
-        description: 'Hugging Face dataset repository'
-        required: false
-        type: string
-        default: 'sunnypilot/sunnypilot_models_v1'
 env:
   RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
-  JSON_FILE: docs/docs/driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_v' || 'v' }}${{ inputs.json_version }}.json
+  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
 
 jobs:
   build_model:
@@ -119,20 +94,38 @@ jobs:
       custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
       is_20hz: ${{ inputs.is_20hz }}
       artifact_suffix: ${{ inputs.artifact_suffix }}
-      target_hardware: ${{ inputs.target_hardware }}
     secrets: inherit
 
   publish_model:
-    if: ${{ !cancelled() && needs.build_model.result == 'success' }}
+    if: ${{ !inputs.bypass_push && !cancelled() }}
     concurrency:
-      group: hf-push-${{ inputs.recompiled_dir }}
+      group: gitlab-push-${{ inputs.recompiled_dir }}
       cancel-in-progress: false
     needs: build_model
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: write
     steps:
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+
+      - name: Add GitLab.com SSH key to known_hosts
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
+
+      - name: Clone GitLab docs repo
+        env:
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+        run: |
+          echo "Cloning GitLab"
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          cd gitlab_docs
+          echo "checkout models/${RECOMPILED_DIR}"
+          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
+          git checkout main
+          cd ..
+
       - name: Checkout docs repo
         uses: actions/checkout@v4
         with:
@@ -141,28 +134,16 @@ jobs:
           path: docs
           ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
 
-      - name: Install huggingface_hub
+      - name: Validate recompiled dir and JSON version
-        run: pip install --upgrade "huggingface_hub>=0.22.0"
-
-      - name: Validate hf_repo and JSON version
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
         run: |
+          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
+            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
+            exit 1
+          fi
           if [ ! -f "$JSON_FILE" ]; then
             echo "JSON file $JSON_FILE does not exist!"
             exit 1
           fi
-          python3 -c "
-          import sys
-          from huggingface_hub import HfApi
-          try:
-              api = HfApi()
-              api.repo_info(repo_id=sys.argv[1], repo_type='dataset')
-              print(f'Success: Repo {sys.argv[1]} exists.')
-          except Exception as e:
-              print('HF validation failed:', e)
-              sys.exit(1)
-          " "${{ inputs.hf_repo }}"
 
       - name: Download artifact name file
         uses: actions/download-artifact@v4
@@ -182,26 +163,49 @@ jobs:
           name: ${{ steps.read-artifact-name.outputs.artifact_name }}
           path: output
 
-      - name: Create models folder
+      - name: Remove unwanted files
+        run: |
+          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
+          find output -type f -name 'dmonitoring_model.onnx' -delete
+
+      - name: Copy model artifact(s) to GitLab recompiled dir
         env:
           ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
         run: |
-          mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/"
+          mkdir -p "$ARTIFACT_DIR"
-          rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt"
+          for path in output/*; do
+            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
+              continue
+            fi
+            name="$(basename "$path")"
+            if [ -d "$path" ]; then
+              mkdir -p "$ARTIFACT_DIR/$name"
+              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
+              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
+            else
+              cp "$path" "$ARTIFACT_DIR/"
+              echo "Copied file $name -> $ARTIFACT_DIR/"
+            fi
+          done
 
-      - name: Upload to Hugging Face
+      - name: Push recompiled dir to GitLab
         env:
-          HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
+          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
         run: |
-          hf upload ${{ inputs.hf_repo }} \
+          cd gitlab_docs
-            output/ \
+          git checkout main
-            "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \
+          git pull origin main
-            --repo-type=dataset
+          for d in models/"$RECOMPILED_DIR"/*/; do
+            git sparse-checkout add "$d"
+          done
+          git add models/"$RECOMPILED_DIR"
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
+          git push origin main
 
-      - name: Pull gh-pages
+      - run: |
-        run: |
           cd docs
           git pull origin gh-pages
 
@@ -217,11 +221,9 @@ jobs:
           fi
           [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
           [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
-          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
-          [ -n "${{ inputs.tinygrad_ref }}" ] && ARGS="$ARGS --tinygrad-ref \"${{ inputs.tinygrad_ref }}\""
           eval python3 docs/json_parser.py \
             --json-path "$JSON_FILE" \
-            --recompiled-dir "local_models/$RECOMPILED_DIR" \
+            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
             --sort-by-date \
             $ARGS
 

.github/workflows/cereal_validation.yaml

@@ -6,7 +6,7 @@ on:
       - master
   pull_request:
     paths:
-      - 'openpilot/cereal/**'
+      - 'cereal/**'
   workflow_dispatch:
   workflow_call:
     inputs:
@@ -30,25 +30,11 @@ jobs:
       - name: Checkout sunnypilot cereal
         uses: actions/checkout@v6
         with:
-          sparse-checkout: |
+          sparse-checkout: cereal
-            openpilot/cereal
-            uv.lock
-          submodules: false
 
       - name: Init sunnypilot opendbc submodule
         run: git submodule update --init --depth 1 opendbc_repo
 
-      - name: Locate sunnypilot capnp import paths
-        id: locate-sp-capnp
-        run: |
-          SP_IMPORT_ARGS=""
-          SP_CAR_CAPNP=$(find opendbc_repo -maxdepth 4 -name car.capnp -path '*/opendbc/car/car.capnp' -printf '%h\n' -quit)
-          if [ -n "$SP_CAR_CAPNP" ]; then
-            SP_IMPORT_ARGS="-I $SP_CAR_CAPNP"
-            echo "Found sunnypilot car.capnp at: $SP_CAR_CAPNP"
-          fi
-          echo "import_args=$SP_IMPORT_ARGS" >> "$GITHUB_OUTPUT"
-
       - name: Checkout upstream openpilot
         uses: actions/checkout@v6
         with:
@@ -79,19 +65,20 @@ jobs:
           fi
           echo "import_args=$IMPORT_ARGS" >> "$GITHUB_OUTPUT"
 
-      - name: Install pycapnp
+      - name: Install uv
-        run: |
+        run: pip install uv
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
-          pip install "pycapnp==${PYCAPNP_VER}"
 
       - name: Generate sunnypilot schema
         run: |
-          python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
+          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
-            -g -f /tmp/sp_schema.json --cereal-dir openpilot/cereal \
+          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
-            ${{ steps.locate-sp-capnp.outputs.import_args }}
+            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
+            -g -f /tmp/sp_schema.json --cereal-dir cereal
 
       - name: Validate against upstream
         run: |
-          python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
+          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
+            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
             -r -f /tmp/sp_schema.json --cereal-dir ${{ steps.locate-capnp.outputs.cereal_dir }} \
             ${{ steps.locate-capnp.outputs.import_args }}

.github/workflows/diff_report.yaml

@@ -17,42 +17,29 @@ jobs:
     - name: Wait for process replay
       id: wait
       continue-on-error: true
-      uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
+      uses: lewagon/wait-on-check-action@v1.3.4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
         check-name: process replay
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         allowed-conclusions: success,failure
         wait-interval: 20
-    - name: Get tests run ID
-      if: steps.wait.outcome == 'success'
-      id: tests-run
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        run_id=$(gh api \
-          "repos/${{ github.repository }}/actions/workflows/tests.yaml/runs?event=pull_request&head_sha=${{ github.event.pull_request.head.sha }}&per_page=1" \
-          --jq '.workflow_runs[0].id // empty')
-        if [ -z "$run_id" ]; then
-          echo "No tests.yaml run found for ${{ github.event.pull_request.head.sha }}" >&2
-          exit 1
-        fi
-        echo "run-id=$run_id" >> "$GITHUB_OUTPUT"
     - name: Download diff
       if: steps.wait.outcome == 'success'
-      continue-on-error: true
+      uses: dawidd6/action-download-artifact@v6
-      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
       with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
+        github_token: ${{ secrets.GITHUB_TOKEN }}
-        repository: ${{ github.repository }}
+        workflow: tests.yaml
-        run-id: ${{ steps.tests-run.outputs.run-id }}
+        workflow_conclusion: ''
+        pr: ${{ github.event.number }}
         name: diff_report_${{ github.event.number }}
         path: .
+        allow_forks: true
     - name: Comment on PR
-      if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != ''
+      if: steps.wait.outcome == 'success'
-      uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
+      uses: thollander/actions-comment-pull-request@v2
       with:
-        file-path: diff_report.txt
+        filePath: diff_report.txt
-        comment-tag: diff_report
+        comment_tag: diff_report
-        pr-number: ${{ github.event.number }}
+        pr_number: ${{ github.event.number }}
-        github-token: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -15,11 +15,6 @@ concurrency:
   group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
-env:
-  GIT_CONFIG_COUNT: 1
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
-
 jobs:
   docs:
     name: build docs
@@ -27,7 +22,7 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
       with:
         submodules: true
 
@@ -35,10 +30,11 @@ jobs:
     - name: Build docs
       run: |
         git lfs pull
-        python docs/serve.py --build
+        pip install zensical
+        python scripts/docs.py build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs
@@ -49,14 +45,14 @@ jobs:
       run: |
         set -x
 
-        source tools/release/identity.sh
+        source release/identity.sh
 
         cd openpilot-docs
         git checkout --orphan tmp
         git rm -rf .
 
         # copy over docs
-        cp -r ../docs/_site/ docs/
+        cp -r ../docs_site/ docs/
 
         # GitHub pages config
         touch docs/.nojekyll

.github/workflows/jenkins-pr-trigger.yaml

@@ -11,7 +11,7 @@ jobs:
       contents: write
     steps:
       - name: Delete stale Jenkins branches
-        uses: actions/github-script@v9
+        uses: actions/github-script@v8
         with:
           script: |
             const cutoff = Date.now() - 24 * 60 * 60 * 1000;
@@ -52,7 +52,7 @@ jobs:
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v9
+      uses: actions/github-script@v8
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +72,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v7
+      uses: actions/checkout@v6
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -86,7 +86,7 @@ jobs:
 
     - name: Delete trigger comment
       if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v9
+      uses: actions/github-script@v8
       with:
         script: |
           await github.rest.issues.deleteComment({

.github/workflows/lfs-maintenance.yaml

@@ -65,21 +65,8 @@ jobs:
           echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
           echo '    locksverify = false' >> .lfsconfig
 
-      - name: Configure LFS transfer settings
-        run: |
-          git config lfs.activitytimeout 300
-          git config lfs.transfer.maxretries 5
-          git config lfs.concurrenttransfers 4
-
       - name: Push LFS
         id: sync-and-commit
         run: |
           git lfs ls-files -l
-          for attempt in 1 2 3; do
+          git lfs push --all origin
-            echo "Push attempt $attempt..."
-            git lfs push --all origin && exit 0
-            echo "Attempt $attempt failed, retrying in 30s..."
-            sleep 30
-          done
-          echo "All push attempts failed"
-          exit 1

.github/workflows/model_review.yaml

@@ -0,0 +1,42 @@
+name: "model review"
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - 'selfdrive/modeld/models/*.onnx'
+  workflow_dispatch:
+
+jobs:
+  comment:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Checkout master
+        uses: actions/checkout@v6
+        with:
+          ref: master
+          path: base
+      - run: git lfs pull
+      - run: cd base && git lfs pull
+
+      - name: scripts/reporter.py
+        id: report
+        run: |
+          echo "content<<EOF" >> $GITHUB_OUTPUT
+          echo "## Model Review" >> $GITHUB_OUTPUT
+          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Post model report comment
+        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
+        with:
+          header: model-review
+          message: ${{ steps.report.outputs.content }}

.github/workflows/prebuilt.yaml

@@ -0,0 +1,39 @@
+name: prebuilt
+on:
+  schedule:
+    - cron: '0 * * * *'
+  workflow_dispatch:
+
+env:
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: release/ci/docker_build_sp.sh
+
+jobs:
+  build_prebuilt:
+    name: build prebuilt
+    runs-on: ubuntu-latest
+    if: github.repository == 'sunnypilot/sunnypilot'
+    env:
+      PUSH_IMAGE: true
+    permissions:
+      checks: read
+      contents: read
+      packages: write
+    steps:
+    - name: Wait for green check mark
+      if: ${{ github.event_name != 'workflow_dispatch' }}
+      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
+      with:
+        ref: master
+        wait-interval: 30
+        running-workflow-name: 'build prebuilt'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: git lfs pull
+    - name: Build and Push docker image
+      run: |
+        $DOCKER_LOGIN
+        eval "$BUILD"

.github/workflows/release.yaml

@@ -4,11 +4,6 @@ on:
     - cron: '0 9 * * *'
   workflow_dispatch:
 
-env:
-  GIT_CONFIG_COUNT: 1
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
-
 jobs:
   build___nightly:
     name: build __nightly
@@ -20,17 +15,17 @@ jobs:
     steps:
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
-      uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
+      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build __nightly'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
-    - uses: actions/checkout@v4 # checkout version > v4 breaks nightly release script
+    - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
     - run: ./tools/op.sh setup
     - name: Push __nightly
-      run: BRANCH=__nightly tools/release/build_stripped.sh
+      run: BRANCH=__nightly release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -3,25 +3,18 @@ name: repo maintenance
 on:
   schedule:
     - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
-  pull_request:
-    types: [closed]
   workflow_dispatch:
 
 env:
   PYTHONPATH: ${{ github.workspace }}
-  GIT_CONFIG_COUNT: 1
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
 
 jobs:
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
-    if: >-
+    if: github.repository == 'sunnypilot/sunnypilot'
-      github.repository == 'sunnypilot/sunnypilot' &&
-      (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
       with:
         submodules: true
     - run: ./tools/op.sh setup
@@ -57,10 +50,10 @@ jobs:
         git add .
     - name: update car docs
       run: |
-        python openpilot/selfdrive/car/docs.py
+        python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
+      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
       with:
         author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
         token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
@@ -82,51 +75,3 @@ jobs:
           ${{ steps.pip_tree.outputs.PIP_TREE }}
           ```
         labels: bot
-
-  cleanup_closed_branches:
-    if: github.repository == 'commaai/openpilot' && github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: read
-    steps:
-      - uses: actions/github-script@v9
-        with:
-          script: |
-            const { owner, repo } = context.repo;
-            const upstream = `${owner}/${repo}`;
-
-            const closed = context.payload.pull_request;
-            if (closed) {
-              if (closed.head.repo?.full_name === upstream) {
-                await github.rest.git.deleteRef({ owner, repo, ref: `heads/${closed.head.ref}` }).catch(console.log);
-              }
-              return;
-            }
-
-            for await (const response of github.paginate.iterator(github.rest.pulls.list, {
-              owner,
-              repo,
-              state: 'closed',
-              per_page: 100,
-            })) {
-              for (const pr of response.data) {
-                if (pr.head.repo?.full_name !== upstream) continue;
-
-                const branch = pr.head.ref;
-                try {
-                  await github.rest.git.deleteRef({
-                    owner,
-                    repo,
-                    ref: `heads/${branch}`,
-                  });
-                  console.log(`Deleted branch ${branch} (PR #${pr.number})`);
-                } catch (error) {
-                  if (error.status === 422 || error.status === 403) {
-                    console.log(`Skipping branch ${branch} (PR #${pr.number}): ${error.message}`);
-                    continue;
-                  }
-                  throw error;
-                }
-              }
-            }

.github/workflows/sunnypilot-build-model.yaml

@@ -6,7 +6,7 @@ env:
   SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   UPSTREAM_REPO: "commaai/openpilot"
   TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
-  MODELS_DIR: ${{ github.workspace }}/openpilot/selfdrive/modeld/models
+  MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
 
 on:
   workflow_call:
@@ -30,11 +30,6 @@ on:
         required: false
         type: string
         default: ''
-      target_hardware:
-        description: 'Hardware target to compile for (qcom or usbgpu)'
-        required: false
-        type: string
-        default: 'qcom'
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -51,14 +46,6 @@ on:
         required: false
         type: boolean
         default: true
-      target_hardware:
-        description: 'Hardware target to compile for'
-        required: true
-        type: choice
-        options:
-          - qcom
-          - usbgpu
-        default: 'qcom'
 
 
 run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
@@ -80,7 +67,6 @@ jobs:
         with:
           repository: commaai/openpilot
           ref: ${{ inputs.upstream_branch }}
-          fetch-depth: 1
           submodules: recursive
           path: openpilot
 
@@ -90,38 +76,26 @@ jobs:
         with:
           repository: sunnypilot/sunnypilot
           ref: ${{ inputs.upstream_branch }}
-          fetch-depth: 1
           submodules: recursive
           path: openpilot
       - name: Get commit date
         id: commit-date
         run: |
-          cd ${{ github.workspace }}/openpilot/openpilot
+          cd ${{ github.workspace }}/openpilot
           commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
           echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
       - run: |
-          cd ${{ github.workspace }}/openpilot/openpilot
+          cd ${{ github.workspace }}/openpilot
-          if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
+          git lfs pull
-            git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx"
-            rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
-          else
-            git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx" -X ""
-            find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
-          fi
-          if grep -lIF "version https://git-lfs.github.com/spec/v1" selfdrive/modeld/models/*.onnx; then
-            echo "::error::the ONNX files above are still LFS pointers, not real models"
-            exit 1
-          fi
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v4
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
-          path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
+          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
-          if-no-files-found: error
 
   build_model:
-    runs-on: [self-hosted, usbgpu]
+    runs-on: [self-hosted, tici]
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
@@ -129,9 +103,24 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 1
           submodules: recursive
 
+      - run: git lfs pull
+      - name: Cache SCons
+        uses: actions/cache@v4
+        with:
+          path: ${{env.SCONS_CACHE_DIR}}
+          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
+          restore-keys: |
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}
 
       - name: Set environment variables
         id: set-env
@@ -139,10 +128,7 @@ jobs:
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export UV_PYTHON_PREFERENCE=managed
-          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          uv sync --frozen
           printenv >> $GITHUB_ENV
           if [[ "${{ runner.debug }}" == "1" ]]; then
             cat $GITHUB_OUTPUT
@@ -162,71 +148,41 @@ jobs:
           if [[ "${{ runner.debug }}" == "1" ]]; then
             printenv
           fi
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
-          rm -rf ${{ env.MODELS_DIR }}/*.onnx*
+          rm -rf ${{ env.MODELS_DIR }}/*.onnx
 
       - name: Download model artifacts
         uses: actions/download-artifact@v4
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
-          path: ${{ env.MODELS_DIR }}
+          path: ${{ github.workspace }}/selfdrive/modeld/models
+      - run: |
+          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |
           source /etc/profile
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
 
-          COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
+          COMPILE_MODELD="${{ github.workspace }}/sunnypilot/modeld_v2/compile_modeld.py"
           MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
           CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
-
+          TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
-          TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
-          if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
-            echo "USBGPU build"
-            export USBGPU=1
-            TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
-            OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
-          else
-            echo "QCOM build"
-            TG_FLAGS="$TG_FLAGS_QCOM"
-            OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
-          fi
 
           # Generate metadata for all ONNX files
           find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
             echo "Generating metadata: $onnx_file"
-            env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+            env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
           done
 
           # Detect model type and build compile args
-          VISION_ONNX=""
+          VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
-          for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
+          POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
-            [ -f "$f" ] && VISION_ONNX="$f" && break
+          OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
-          done
+          ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
-
+          SUPERCOMBO_ONNX="${{ env.MODELS_DIR }}/supercombo.onnx"
-          POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
-            [ -f "$f" ] && POLICY_ONNX="$f" && break
-          done
-
-          OFF_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
-            [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
-          done
-
-          ON_POLICY_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
-            [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
-          done
-
-          SUPERCOMBO_ONNX=""
-          for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
-            [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
-          done
 
           MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
           if [ -f "$VISION_ONNX" ]; then
@@ -247,15 +203,24 @@ jobs:
           fi
 
           if [ -n "$MODEL_TYPE" ]; then
-            echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
+            echo "Detected: $MODEL_TYPE -> driving_tinygrad.pkl"
             env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
               --model-type $MODEL_TYPE \
               --model-size $MODEL_SIZE \
               --camera-resolutions $CAMERA_RES \
               $ONNX_ARGS \
-              --output "$OUTPUT_PKL"
+              --output "${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
           fi
 
+      - name: Validate Model Outputs
+        run: |
+          source /etc/profile
+          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
+          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
+          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
+            --validate-only \
+            --model-dir "${{ env.MODELS_DIR }}"
+
       - name: Prepare Output
         run: |
           sudo rm -rf ${{ env.OUTPUT_DIR }}
@@ -264,8 +229,10 @@ jobs:
           # Copy the model files
           rsync -avm \
             --include='*.dlc' \
+            --include='*.pkl' \
             --include='*.chunk*' \
             --include='*.chunkmanifest' \
+            --include='*.onnx' \
             --exclude='*' \
             --delete-excluded \
             --chown=comma:comma \
@@ -297,5 +264,4 @@ jobs:
       - name: Re-enable powersave
         if: always()
         run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -4,8 +4,12 @@ env:
   BUILD_DIR: "/data/openpilot"
   OUTPUT_DIR: ${{ github.workspace }}/output
   CI_DIR: ${{ github.workspace }}/release/ci
+  SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
   PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
 
+  # Branch configurations
+  STAGING_SOURCE_BRANCH: 'master'
+
   # Runtime configuration
   SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
 
@@ -75,7 +79,7 @@ jobs:
             is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
             echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
 
-            stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
+            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
             echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
             echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
           fi
@@ -105,6 +109,11 @@ jobs:
       group: build-${{ github.head_ref || github.ref_name }}
       cancel-in-progress: false
     runs-on: [self-hosted, tici]
+    outputs:
+      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
+      version: ${{ needs.prepare_strategy.outputs.version }}
+      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
+      commit_sha: ${{ github.sha }}
     if: ${{
       (always() && !cancelled() && !failure()) &&
       needs.prepare_strategy.result == 'success' &&
@@ -120,16 +129,34 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
       - run: git lfs pull
 
+      - name: Cache SCons
+        uses: actions/cache@v4
+        with:
+          path: ${{env.SCONS_CACHE_DIR}}
+          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
+          restore-keys: |
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}
+
       - name: Set environment variables
+        id: set-env
         run: |
+          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
+          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
+          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
+          echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
+
           # Set up common environment
           source /etc/profile;
           export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export UV_PYTHON_PREFERENCE=managed
-          export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
           export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          uv sync
           printenv >> $GITHUB_ENV
+          if [[ "${{ runner.debug }}" == "1" ]]; then
+            cat $GITHUB_OUTPUT
+          fi
 
       - name: Setup build environment
         run: |
@@ -138,56 +165,67 @@ jobs:
           echo "Starting build stage..."
           echo "BUILD_DIR: ${BUILD_DIR}"
           echo "CI_DIR: ${CI_DIR}"
-          echo "VERSION: ${{ needs.prepare_strategy.outputs.version }}"
+          echo "VERSION: ${{ steps.set-env.outputs.version }}"
           echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
           echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
           echo "-------"
           if [[ "${{ runner.debug }}" == "1" ]]; then
             printenv
           fi
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
 
       - name: Build Main Project
         run: |
-          export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
+          export PYTHONPATH="$BUILD_DIR"
-          ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" --
+          ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
-          # outside the checkout, which is wiped each run. /data/scons_cache is the device's, not ours.
-          SCONS_CACHE="$RUNNER_WORKSPACE/scons_cache"
-          mkdir -p "$SCONS_CACHE"
           cd $BUILD_DIR
-          echo "Building locationd..."
+          sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
-          # -j1: parallel rednose generators OOM the device
+          echo "Building sunnypilot's modeld_v2..."
-          scons -j1 cache_dir="$SCONS_CACHE" --minimal \
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
-            openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
+          echo "Building sunnypilot's locationd..."
+          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
+          echo "Building openpilot's locationd..."
+          scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
           echo "Building rest of sunnypilot"
-          /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
+          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
           touch ${BUILD_DIR}/prebuilt
           if [[ "${{ runner.debug }}" == "1" ]]; then
             ls -la ${BUILD_DIR}
           fi
 
-      - name: Strip release tree
+      - name: Prepare Output
         run: |
-          cd $BUILD_DIR
+          sudo rm -rf ${OUTPUT_DIR}
-          find . -name '*.a' -delete
+          mkdir -p ${OUTPUT_DIR}
-          find . -name '*.o' -delete
+          rsync -am${RUNNER_DEBUG:+v} \
-          find . -name '*.os' -delete
+            --exclude='.sconsign.dblite' \
-          find . -name '*.pyc' -delete
+            --exclude='*.a' \
-          find . -name 'moc_*' -delete
+            --exclude='*.o' \
-          find . -name '__pycache__' -type d -exec rm -rf {} +
+            --exclude='*.os' \
-          find . -name 'SConstruct' -delete
+            --exclude='*.pyc' \
-          find . -name 'SConscript' -delete
+            --exclude='moc_*' \
-          rm -rf .sconsign.dblite Jenkinsfile tools/release/ release/
+            --exclude='__pycache__' \
-          rm -f openpilot/selfdrive/modeld/models/*.onnx*
+            --exclude='Jenkinsfile' \
-          rm -f openpilot/sunnypilot/modeld*/models/*.onnx*
+            --exclude='**/release/' \
-          find openpilot/third_party/ -name '*x86*' -exec rm -r {} +
+            --exclude='**/.github/' \
-          find openpilot/third_party/ -name '*Darwin*' -exec rm -r {} +
+            --exclude='**/selfdrive/ui/replay/' \
-          cd -
+            --exclude='**/__pycache__/' \
+            --exclude='${{env.SCONS_CACHE_DIR}}' \
+            --exclude='**/.git/' \
+            --exclude='**/SConstruct' \
+            --exclude='**/SConscript' \
+            --exclude='**/.venv/' \
+            --exclude='selfdrive/modeld/models/*.onnx*' \
+            --exclude='sunnypilot/modeld*/models/*.onnx*' \
+            --exclude='third_party/*x86*' \
+            --exclude='third_party/*Darwin*' \
+            --delete-excluded \
+            --chown=comma:comma \
+            ${BUILD_DIR}/ ${OUTPUT_DIR}/
 
       - name: 'Tar.gz files'
         run: |
-          tar czf prebuilt.tar.gz -C ${{ env.BUILD_DIR }} .
+          tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} .
           ls -la prebuilt.tar.gz
 
       - name: 'Upload Artifact'
@@ -195,12 +233,10 @@ jobs:
         with:
           name: prebuilt
           path: prebuilt.tar.gz
-          compression-level: 0
 
       - name: Re-enable powersave
         if: always()
         run: |
-          source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
 
 
@@ -237,22 +273,29 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          echo '${{ toJSON(needs.prepare_strategy.outputs) }}'
+          echo '${{ toJSON(needs.build.outputs) }}'
           ls -la ${{ env.OUTPUT_DIR }}
 
           ${{ env.CI_DIR }}/publish.sh \
             "${{ github.workspace }}" \
             "${{ env.OUTPUT_DIR }}" \
-            "${{ needs.prepare_strategy.outputs.new_branch }}" \
+            "${{ needs.build.outputs.new_branch }}" \
-            "${{ needs.prepare_strategy.outputs.version }}" \
+            "${{ needs.build.outputs.version }}" \
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
+            "${{ needs.build.outputs.extra_version_identifier }}"
+
+          echo ""
+          echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
+          echo ""
+          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
+          echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
+          echo "3. Update as needed (JSON array with no spaces)"
 
       - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
         if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
         run: |
             TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
-            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
+            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
             git push -f origin ${TAG}
 
   notify:
@@ -271,6 +314,7 @@ jobs:
       - name: Prepare notification message
         id: message
         run: |
+          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
           export VERSION="${{ needs.prepare_strategy.outputs.version }}"
           export branch_name="${{ env.SOURCE_BRANCH }}"
           export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"

.github/workflows/tests.yaml

@@ -20,9 +20,7 @@ concurrency:
 env:
   CI: 1
   PYTHONPATH: ${{ github.workspace }}
-  GIT_CONFIG_COUNT: 1
+  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
-  GIT_CONFIG_KEY_0: lfs.fetchexclude
-  GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
 
 jobs:
   build_release:
@@ -37,27 +35,27 @@ jobs:
       STRIPPED_DIR: /tmp/releasepilot
       PYTHONPATH: /tmp/releasepilot
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
       with:
         submodules: true
     - name: Getting LFS files
-      uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
       with:
         timeout_minutes: 2
         max_attempts: 3
         command: git lfs pull
     - name: Build devel
-      timeout-minutes: 3
+      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
+      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
     - run: ./tools/op.sh setup
     - name: Build openpilot and run checks
       timeout-minutes: 30
       working-directory: ${{ env.STRIPPED_DIR }}
-      run: python3 openpilot/system/manager/build.py
+      run: python3 system/manager/build.py
     - name: Run tests
       timeout-minutes: 1
       working-directory: ${{ env.STRIPPED_DIR }}
-      run: tools/release/check-dirty.sh
+      run: release/check-dirty.sh
     - name: Check submodules
       if: github.repository == 'sunnypilot/sunnypilot'
       timeout-minutes: 3
@@ -77,13 +75,15 @@ jobs:
             export CHECK_PR_REFS=true
           fi
         fi
-        tools/release/check-submodules.sh
+        release/check-submodules.sh
 
   build_mac:
     name: build macOS
     runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
     - name: Remove Homebrew from environment
       run: |
         FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
@@ -101,7 +101,9 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
     - run: ./tools/op.sh setup
     - name: Static analysis
       timeout-minutes: 1
@@ -116,21 +118,23 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
     - run: ./tools/op.sh setup
-    - name: Install raylib headless dependencies
-      if: ${{ !contains(runner.name, 'nsc') }}
-      run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
     - name: Build openpilot
       run: scons
     - name: Run unit tests
       timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
-      env:
+      run: |
-        RAYLIB_BACKEND: headless
+        source selfdrive/test/setup_xvfb.sh
-      run: tools/op.sh test
+        # Pre-compile Python bytecode so each pytest worker doesn't need to
+        $PYTEST --collect-only -m 'not slow' -qq
+        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
 
   process_replay:
     name: process replay
+    if: false  # disable process_replay for forks
     runs-on: ${{
       (github.repository == 'commaai/openpilot') &&
       ((github.event_name != 'pull_request') ||
@@ -138,60 +142,68 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Run replay
       timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
       continue-on-error: ${{ github.ref == 'refs/heads/master' }}
-      run: openpilot/selfdrive/test/process_replay/test_processes.py -j$(nproc)
+      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
     - name: Print diff
       id: print-diff
       if: always()
-      run: cat openpilot/selfdrive/test/process_replay/diff.txt
+      run: cat selfdrive/test/process_replay/diff.txt
     - name: Print diff report
       if: always()
-      run: cat openpilot/selfdrive/test/process_replay/diff_report.txt
+      run: cat selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v7
+    - uses: actions/upload-artifact@v6
       if: always()
       continue-on-error: true
       with:
         name: process_replay_diff.txt
-        path: openpilot/selfdrive/test/process_replay/diff.txt
+        path: selfdrive/test/process_replay/diff.txt
     - name: Upload diff report
-      uses: actions/upload-artifact@v7
+      uses: actions/upload-artifact@v6
       if: always() && github.event_name == 'pull_request'
       continue-on-error: true
       with:
         name: diff_report_${{ github.event.number }}
-        path: openpilot/selfdrive/test/process_replay/diff_report.txt
+        path: selfdrive/test/process_replay/diff_report.txt
     - name: Checkout ci-artifacts
-      if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v7
+      uses: actions/checkout@v4
       with:
-        repository: sunnypilot/ci-artifacts
+        repository: commaai/ci-artifacts
         ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
         path: ${{ github.workspace }}/ci-artifacts
     - name: Prepare refs
-      if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
       working-directory: ${{ github.workspace }}/ci-artifacts
       run: |
         git config user.name "GitHub Actions Bot"
         git config user.email "<>"
         git fetch origin process-replay || true
         git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
-        cp ${{ github.workspace }}/openpilot/selfdrive/test/process_replay/fakedata/*.zst .
+        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
         echo "${{ github.sha }}" > ref_commit
         git add .
         git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
     - name: Push refs
-      if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
       with:
         timeout_minutes: 2
         max_attempts: 3
         command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force
+    - name: Run regen
+      if: false
+      timeout-minutes: 4
+      env:
+        ONNXCPU: 1
+      run: $PYTEST selfdrive/test/process_replay/test_regen.py
 
   simulator_driving:
     name: simulator driving
@@ -203,16 +215,17 @@ jobs:
       || fromJSON('["ubuntu-24.04"]') }}
     if: false  # FIXME: Started to timeout recently
     steps:
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Driving test
       timeout-minutes: 2
-      env:
+      run: |
-        # MetaDrive renders offscreen through panda3d's EGL pipe on llvmpipe
+        source selfdrive/test/setup_xvfb.sh
-        EGL_PLATFORM: surfaceless
+        pytest -s tools/sim/tests/test_metadrive_bridge.py
-      run: python openpilot/tools/sim/tests/test_metadrive_bridge.py
 
   create_ui_report:
     name: Create UI Report
@@ -223,21 +236,19 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
       - run: ./tools/op.sh setup
-      - name: Install raylib headless dependencies
-        if: ${{ !contains(runner.name, 'nsc') }}
-        run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
       - name: Build openpilot
         run: scons
       - name: Create UI Report
-        env:
-          RAYLIB_BACKEND: headless
         run: |
-          python3 openpilot/selfdrive/ui/tests/diff/replay.py
+          source selfdrive/test/setup_xvfb.sh
-          python3 openpilot/selfdrive/ui/tests/diff/replay.py --big
+          python3 selfdrive/ui/tests/diff/replay.py
+          python3 selfdrive/ui/tests/diff/replay.py --big
       - name: Upload UI Report
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v6
         with:
           name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: openpilot/selfdrive/ui/tests/diff/report
+          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -8,9 +8,9 @@ on:
     branches:
       - 'master'
     paths:
-      - 'openpilot/selfdrive/assets/**'
+      - 'selfdrive/assets/**'
-      - 'openpilot/selfdrive/ui/**'
+      - 'selfdrive/ui/**'
-      - 'openpilot/system/ui/**'
+      - 'system/ui/**'
   workflow_dispatch:
 
 env:
@@ -25,8 +25,7 @@ env:
 
 jobs:
   preview:
-    if: false # tmp disable due to GH API rate limiting flakiness
+    if: github.repository == 'sunnypilot/sunnypilot'
-    #if: github.repository == 'sunnypilot/sunnypilot'
     name: preview
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -35,12 +34,12 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
         with:
           submodules: true
 
       - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
+        uses: lewagon/wait-on-check-action@v1.3.4
         with:
           ref: ${{ env.SHA }}
           check-name: ${{ env.UI_JOB_NAME }}
@@ -54,7 +53,7 @@ jobs:
           echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
       - name: Getting proposed ui
-        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151
+        uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
@@ -63,7 +62,7 @@ jobs:
           path: ${{ github.workspace }}/pr_ui
 
       - name: Getting mici master ui
-        uses: actions/checkout@v7
+        uses: actions/checkout@v6
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -71,7 +70,7 @@ jobs:
           ref: openpilot_master_ui_mici_raylib
 
       - name: Getting big master ui
-        uses: actions/checkout@v7
+        uses: actions/checkout@v6
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -98,7 +97,7 @@ jobs:
           done
 
       - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@178e0b4a408f06ce40d896c3cd79f50fa6f8b0c3
+        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
 
       - name: Finding diffs
         if: github.event_name == 'pull_request_target'
@@ -116,13 +115,13 @@ jobs:
             cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
 
             diff_exit_code=0
-            python3 ${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/diff.py \
+            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
               "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
               "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
               "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
 
-            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
 
             REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
             if [ $diff_exit_code -eq 0 ]; then
@@ -157,7 +156,7 @@ jobs:
           for variant in $VARIANTS; do
             IFS=':' read -r name _ _ <<< "$variant"
             diff_name="${name}_diff"
-            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
             git add "${diff_name}_pr_${{ github.event.number }}.html"
           done
           git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
@@ -165,12 +164,12 @@ jobs:
 
       - name: Comment on PR
         if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
+        uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
             <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
             ## UI Preview
             ${{ steps.find_diff.outputs.COMMENT }}
-          comment-tag: run_id_ui_preview
+          comment_tag: run_id_ui_preview
-          pr-number: ${{ github.event.number }}
+          pr_number: ${{ github.event.number }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -15,13 +15,6 @@ a.out
 .cache/
 bin/
 
-# created at launch for comma hardware PYTHONPATH (PC uses editable installs via pyproject.toml)
-/msgq
-/opendbc
-/rednose
-/teleoprtc
-/tinygrad
-
 *.mp4
 *.dylib
 *.DSYM
@@ -39,8 +32,6 @@ bin/
 *.os-*
 *.so
 *.a
-st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
-*.unchunked
 *.clb
 *.class
 *.pyxbldc
@@ -53,18 +44,18 @@ st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
 config.json
 compile_commands.json
 compare_runtime*.html
-openpilot/selfdrive/modeld/models/tg_input_devices.json
+selfdrive/modeld/models/tg_input_devices.json
 
 # build artifacts
-docs/_site/
+docs_site/
-openpilot/selfdrive/pandad/pandad
+selfdrive/pandad/pandad
-openpilot/cereal/services.h
+cereal/services.h
-openpilot/cereal/gen
+cereal/gen
-openpilot/cereal/messaging/bridge
+cereal/messaging/bridge
-openpilot/selfdrive/ui/translations/tmp
+selfdrive/ui/translations/tmp
-openpilot/selfdrive/car/tests/cars_dump
+selfdrive/car/tests/cars_dump
-openpilot/system/camerad/camerad
+system/camerad/camerad
-openpilot/system/camerad/test/ae_gray_test
+system/camerad/test/ae_gray_test
 
 .coverage*
 coverage.xml

.gitmodules

@@ -17,5 +17,5 @@
   path = tinygrad_repo
   url = https://github.com/sunnypilot/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
-	path = openpilot/sunnypilot/neural_network_data
+  path = sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.vscode/launch.json

@@ -6,9 +6,9 @@
       "type": "pickString",
       "description": "Select the process to debug",
       "options": [
-        "openpilot/selfdrive/controls/controlsd.py",
+        "selfdrive/controls/controlsd.py",
-        "openpilot/system/timed/timed.py",
+        "system/timed/timed.py",
-        "openpilot/tools/sim/run_bridge.py"
+        "tools/sim/run_bridge.py"
       ]
     },
     {
@@ -16,7 +16,7 @@
       "type": "pickString",
       "description": "Select the process to debug",
       "options": [
-        "openpilot/selfdrive/ui/ui"
+        "selfdrive/ui/ui"
       ]
     },
     {

.vscode/settings.json

@@ -17,5 +17,10 @@
     "**/.git",
     "**/.venv",
     "**/__pycache__",
+    // exclude directories that should be using the symlinked version
+    "common/**",
+    "selfdrive/**",
+    "system/**",
+    "tools/**",
   ]
 }

Jenkinsfile

@@ -22,7 +22,6 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-export COMMA_CACHE=/data/tmp/comma_download_cache
 #export LOGPRINT=debug # this has gotten too spammy...
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
@@ -31,14 +30,14 @@ export GIT_COMMIT=${env.GIT_COMMIT}
 export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
 export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
-# only use 1 thread since most require real hardware that can't be shared
+# only use 1 thread for tici tests since most require HIL
 export PYTEST_ADDOPTS="-n0 -s"
 
 
 export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
 source ~/.bash_profile
-if [ -f /AGNOS ]; then
+if [ -f /TICI ]; then
   source /etc/profile
 
   rm -rf /tmp/tmp*
@@ -64,8 +63,6 @@ if [ -f /data/openpilot/launch_env.sh ]; then
   source /data/openpilot/launch_env.sh
 fi
 
-export LD_LIBRARY_PATH="\$(python -c 'import ffmpeg; print(ffmpeg.LIB_DIR)'):/usr/local/lib:\${LD_LIBRARY_PATH:-}"
-
 ln -snf ${env.TEST_DIR} /data/pythonpath
 
 cd ${env.TEST_DIR} || true
@@ -96,7 +93,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
           retry (3) {
             def date = sh(script: 'date', returnStdout: true).trim();
             device(device_ip, "set time", "date -s '" + date + "'")
-            device(device_ip, "git checkout", extra + "\n" + readFile("openpilot/selfdrive/test/setup_device_ci.sh"))
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }
           steps.each { item ->
             def name = item[0]
@@ -182,7 +179,7 @@ node {
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/tools/release/build_release.sh"),
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
@@ -190,12 +187,12 @@ node {
       parallel (
         'nightly': {
           deviceStage("build nightly", "tizi-needs-can", [], [
-            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/tools/release/build_release.sh"),
+            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
           deviceStage("build nightly-dev", "tizi-needs-can", [], [
-            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/tools/release/build_release.sh"),
+            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
       )
@@ -205,51 +202,51 @@ node {
     parallel (
       'onroad tests': {
         deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
-          step("build openpilot", "cd openpilot/system/manager && ./build.py"),
+          step("build openpilot", "cd system/manager && ./build.py"),
-          step("check dirty", "tools/release/check-dirty.sh"),
+          step("check dirty", "release/check-dirty.sh"),
-          step("onroad tests", "./openpilot/selfdrive/test/test_onroad.py", [timeout: 60]),
+          step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test power draw", "./openpilot/selfdrive/test/test_power_draw.py"),
+          step("test power draw", "pytest -s selfdrive/test//test_power_draw.py"),
-          step("test encoder", "./openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
+          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
-          step("test manager", "./openpilot/system/manager/test/test_manager.py"),
+          step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
       'camerad OX03C10': {
         deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'sensord': {
         deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py"),
+          step("build", "cd system/manager && ./build.py"),
-          step("test sensord", "./openpilot/system/sensord/tests/test_sensord.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
         deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
-          step("build", "cd openpilot/system/manager && ./build.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
+          step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
-          step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
+          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
         ])
       },
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
-          step("build openpilot", "cd openpilot/system/manager && ./build.py"),
+          step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "./openpilot/selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
-          step("test pandad spi", "./openpilot/selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test amp", "./openpilot/common/hardware/comma/tests/test_amplifier.py"),
+          step("test amp", "pytest common/hardware/tici/tests/test_amplifier.py"),
         ])
       },
 

RELEASES.md

@@ -1,14 +1,6 @@
-Version 0.11.2 (2026-08-12)
+Version 0.11.2 (2026-06-15)
-=======================
+========================
-* New driving model
+
-  * Big model with 880M parameters
-* Support for big models running on an external GPU
-* Live stream cameras from comma connect
-* Generate dashcam clips from comma connect
-* Remote comma body control from comma connect
-* New alert sounds
-* CUPRA Born 2021-2023 support thanks to DaHansi!
-* Volkswagen ID.4 2021-2025 support thanks to DaHansi!
 
 Version 0.11.1 (2026-05-18)
 ========================
@@ -1112,7 +1104,7 @@ Version 0.2.1  (2016-12-14)
 
 Version 0.2  (2016-12-12)
 =========================
- * Car/Radar abstraction layers have shipped, see openpilot/cereal/car.capnp
+ * Car/Radar abstraction layers have shipped, see cereal/car.capnp
  * controlsd has been refactored
  * Shipped plant model and testing maneuvers
  * visiond exits more gracefully now

SConstruct

@@ -10,7 +10,7 @@ import numpy as np
 import SCons.Errors
 from SCons.Defaults import _stripixes
 
-COMMA_HARDWARE = os.path.isfile('/AGNOS')
+TICI = os.path.isfile('/TICI')
 
 SCons.Warnings.warningAsException(True)
 
@@ -24,55 +24,25 @@ release = not os.path.exists(File('#.gitattributes').abspath) # file absent on r
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=(not COMMA_HARDWARE and not release),
+          default=(not TICI and not release),
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-submodule_python_paths = [
-  Dir("#").abspath,
-  Dir("#msgq_repo").abspath,
-  Dir("#opendbc_repo").abspath,
-  Dir("#rednose_repo").abspath,
-  Dir("#teleoprtc_repo").abspath,
-  Dir("#tinygrad_repo").abspath,
-]
-for p in reversed(submodule_python_paths):
-  if p not in sys.path:
-    sys.path.insert(0, p)
-
-if external_pythonpath := os.environ.get("PYTHONPATH"):
-  submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths]
-
 # Detect platform
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and COMMA_HARDWARE:
+elif arch == "aarch64" and TICI:
-  arch = "comma_arm64"
+  arch = "larch64"
 assert arch in [
-  "comma_arm64",  # linux comma hardware (AGNOS) arm64
+  "larch64",  # linux tici arm64
-  "aarch64",      # linux pc arm64
+  "aarch64",  # linux pc arm64
-  "x86_64",       # linux pc x64
+  "x86_64",   # linux pc x64
-  "Darwin",       # macOS arm64 (x86 not supported)
+  "Darwin",   # macOS arm64 (x86 not supported)
 ]
 
-pkg_names = ['acados', 'capnproto', 'eigen', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zstd']
+pkg_names = ['acados', 'bzip2', 'capnproto', 'catch2', 'eigen', 'ffmpeg', 'json11', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
 pkgs = [importlib.import_module(name) for name in pkg_names]
 acados = pkgs[pkg_names.index('acados')]
-ffmpeg = pkgs[pkg_names.index('ffmpeg')]
-# Shared package ships .so/.dylib; older device venvs still have static .a only.
-# Keep static link deps (x264/z/va/drm) when the installed package is static so
-# COMMA_HARDWARE CI works without upgrading the device venv yet.
-# TODO: drop the static fallback once device venvs have comma-deps-ffmpeg>=7.1.0.post94
-_ffmpeg_lib_names = os.listdir(ffmpeg.LIB_DIR) if os.path.isdir(ffmpeg.LIB_DIR) else []
-ffmpeg_shared = any(
-  n.startswith('libavcodec.so') or (n.startswith('libavcodec') and n.endswith('.dylib'))
-  for n in _ffmpeg_lib_names
-)
-ffmpeg_libs = ['avformat', 'avcodec', 'swresample', 'avutil']
-if not ffmpeg_shared:
-  ffmpeg_libs += ['x264', 'z']
-  if arch != "Darwin":
-    ffmpeg_libs += ['va', 'va-drm', 'drm']
 acados_include_dirs = [
   acados.INCLUDE_DIR,
   os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
@@ -121,7 +91,7 @@ def _libflags(target, source, env, for_signature):
 env = Environment(
   ENV={
     "PATH": os.environ['PATH'],
-    "PYTHONPATH": os.pathsep.join(submodule_python_paths),
+    "PYTHONPATH": Dir("#").abspath,
     "ACADOS_SOURCE_DIR": acados.DIR,
     "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
     "TERA_PATH": acados.TERA_PATH
@@ -129,11 +99,10 @@ env = Environment(
   CCFLAGS=[
     "-g",
     "-fPIC",
-    "-pipe",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow" if arch in ("Darwin", "comma_arm64") else "-Wshadow=local",
+    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
@@ -143,46 +112,36 @@ env = Environment(
   CFLAGS=["-std=gnu11"],
   CXXFLAGS=["-std=c++1z"],
   CPPPATH=[
-    "#openpilot",
+    "#",
-    "#msgq_repo",            # #include "msgq/..."
+    "#msgq",
-    "#opendbc_repo",         # #include "opendbc/..."
-    "#rednose_repo",         # #include "rednose/..."
-    "#rednose_repo/rednose", # #include "logger/..." (rednose package root)
-    "#openpilot/cereal/gen/cpp",
     acados_include_dirs,
     [x.INCLUDE_DIR for x in pkgs],
-    "#",
   ],
   LIBPATH=[
-    "#openpilot/common",
+    "#common",
     "#msgq_repo",
-    "#openpilot/selfdrive/pandad",
+    "#selfdrive/pandad",
-    "#rednose_repo/rednose/helpers",
+    "#rednose/helpers",
     [x.LIB_DIR for x in pkgs],
   ],
-  RPATH=[ffmpeg.LIB_DIR] if ffmpeg_shared else [],
+  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
-  REDNOSE_ROOT="#rednose_repo",
+  REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
-  toolpath=["#msgq_repo/site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
+  toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
-# SCons' Darwin linker tool doesn't define the variables used to expand RPATH.
+if arch != "larch64":
-if arch == "Darwin":
-  env["RPATHPREFIX"] = "-Wl,-rpath,"
-  env["RPATHSUFFIX"] = ""
-  env["_RPATH"] = "${_concat(RPATHPREFIX, RPATH, RPATHSUFFIX, __env__)}"
-if arch != "comma_arm64":
   env['_LIBFLAGS'] = _libflags
 
 # Arch-specific flags and paths
-if arch == "comma_arm64":
+if arch == "larch64":
   env["CC"] = "clang"
   env["CXX"] = "clang++"
   env.Append(LIBPATH=[
     "/usr/lib/aarch64-linux-gnu",
   ])
-  arch_flags = ["-D__COMMA_HARDWARE__", "-mcpu=cortex-a57"]
+  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
   env.Append(CCFLAGS=arch_flags)
   env.Append(CXXFLAGS=arch_flags)
 elif arch == "Darwin":
@@ -231,10 +190,10 @@ else:
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch', 'acados', 'release', 'ffmpeg_libs')
+Export('env', 'arch', 'acados', 'release')
 
 # Setup cache dir
-default_cache_dir = '/data/scons_cache' if arch == "comma_arm64" else '/tmp/scons_cache'
+default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
 cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
 cache_size_limit = 4e9 if "CI" in os.environ else 2e9
 CacheDir(cache_dir)
@@ -252,7 +211,7 @@ def prune_cache_dir(target=None, source=None, env=None):
 # ********** start building stuff **********
 
 # Build common module
-SConscript(['openpilot/common/SConscript'])
+SConscript(['common/SConscript'])
 Import('_common')
 common = [_common, 'json11', 'zmq']
 Export('common')
@@ -263,7 +222,7 @@ env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
 
-SConscript(['openpilot/cereal/SConscript'])
+SConscript(['cereal/SConscript'])
 
 Import('socketmaster', 'msgq')
 messaging = [socketmaster, msgq, 'capnp', 'kj',]
@@ -274,33 +233,34 @@ Export('messaging')
 SConscript(['panda/SConscript'])
 
 # Build rednose library
-SConscript(['rednose_repo/rednose/SConscript'])
+SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
-  'openpilot/system/loggerd/SConscript',
+  'system/loggerd/SConscript',
 ])
 
-if arch == "comma_arm64":
+if arch == "larch64":
-  SConscript(['openpilot/system/camerad/SConscript'])
+  SConscript(['system/camerad/SConscript'])
 
 # Build selfdrive
 SConscript([
-  'openpilot/selfdrive/pandad/SConscript',
+  'selfdrive/pandad/SConscript',
-  'openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
+  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
-  'openpilot/selfdrive/locationd/SConscript',
+  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'openpilot/selfdrive/modeld/SConscript',
+  'selfdrive/locationd/SConscript',
-  'openpilot/selfdrive/ui/SConscript',
+  'selfdrive/modeld/SConscript',
+  'selfdrive/ui/SConscript',
 ])
 
-SConscript(['openpilot/sunnypilot/SConscript'])
+SConscript(['sunnypilot/SConscript'])
 
 # Build desktop-only tools
-if GetOption('extras') and arch != "comma_arm64":
+if GetOption('extras') and arch != "larch64":
   SConscript([
-    'openpilot/tools/replay/SConscript',
+    'tools/replay/SConscript',
-    'openpilot/tools/cabana/SConscript',
+    'tools/cabana/SConscript',
-    'openpilot/tools/jotpluggler/SConscript',
+    'tools/jotpluggler/SConscript',
   ])
 
 

cereal/README.md

@@ -24,17 +24,17 @@ things are not. Read more details [here](https://capnproto.org/language.html).
 ### Custom forks
 
 Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
-spec, however this could conflict with future changes made in mainline openpilot's cereal spec. Rebasing against mainline openpilot
+spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
 then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
-[custom.capnp](custom.capnp) that we will leave empty in mainline openpilot's cereal spec. **If you only modify those, you can ensure your
+[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
 An example of compatible changes:
 ```diff
-diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp
+diff --git a/cereal/custom.capnp b/cereal/custom.capnp
 index 3348e859e..3365c7b98 100644
---- a/openpilot/cereal/custom.capnp
+--- a/cereal/custom.capnp
-+++ b/openpilot/cereal/custom.capnp
++++ b/cereal/custom.capnp
 @@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
  # DO rename the structs
  # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
@@ -48,10 +48,10 @@ index 3348e859e..3365c7b98 100644
  }
 
  struct CustomReserved1 @0xaedffd8f31e7b55d {
-diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp
+diff --git a/cereal/log.capnp b/cereal/log.capnp
 index 1209f3fd9..b189f58b6 100644
---- a/openpilot/cereal/log.capnp
+--- a/cereal/log.capnp
-+++ b/openpilot/cereal/log.capnp
++++ b/cereal/log.capnp
 @@ -2558,14 +2558,14 @@ struct Event {
 
      # DO change the name of the field
@@ -76,7 +76,7 @@ index 1209f3fd9..b189f58b6 100644
 Example
 ---
 ```python
-import openpilot.cereal.messaging as messaging
+import cereal.messaging as messaging
 
 # in subscriber
 sm = messaging.SubMaster(['sensorEvents'])

cereal/SConscript

@@ -0,0 +1,20 @@
+Import('env', 'common', 'msgq')
+
+cereal_dir = Dir('.')
+gen_dir = Dir('gen')
+
+# Build cereal
+schema_files = ['log.capnp', 'car.capnp', 'deprecated.capnp', 'custom.capnp']
+env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
+            schema_files,
+            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
+
+cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
+
+# Build messaging
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+
+socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
+
+Export('cereal', 'socketmaster')

cereal/__init__.py

@@ -0,0 +1,11 @@
+import os
+import capnp
+from importlib.resources import as_file, files
+
+capnp.remove_import_hook()
+
+with as_file(files("cereal")) as fspath:
+  CEREAL_PATH = fspath.as_posix()
+  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -0,0 +1 @@
+../opendbc_repo/opendbc/car/car.capnp

cereal/custom.capnp

@@ -1,4 +1,4 @@
-using Cxx = import "/include/c++.capnp";
+using Cxx = import "./include/c++.capnp";
 $Cxx.namespace("cereal");
 
 @0xb526ba661d550a59;
@@ -69,8 +69,6 @@ struct LeadData {
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
   mads @0 :ModularAssistiveDrivingSystem;
   intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
-  buttonsPressed @2 :UInt16;
-  buttonsReleaseToggle @3 :UInt16;
 
   enum AudibleAlert {
     none @0;
@@ -139,16 +137,10 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
     eta @2 :UInt32;
   }
 
-  struct Chunk {
-    fileName @0 :Text;
-    sha256 @1 :Text;
-  }
-
   struct Artifact {
     fileName @0 :Text;
     downloadUri @1 :DownloadUri;
     downloadProgress @2 :DownloadProgress;
-    chunks @3 :List(Chunk);
   }
 
   struct Model {
@@ -163,7 +155,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
       policy @3;
       offPolicy @4;
       onPolicy @5;
-      chunked @6;
     }
   }
 
@@ -351,7 +342,6 @@ struct OnroadEventSP @0xda96579883444c35 {
     speedLimitChanged @21;
     speedLimitPending @22;
     e2eChime @23;
-    laneChangeRoadEdge @24;
   }
 }
 
@@ -458,8 +448,6 @@ struct LiveMapDataSP @0xf416ec09499d9d19 {
 
 struct ModelDataV2SP @0xa1680744031fdb2d {
   laneTurnDirection @0 :TurnDirection;
-  leftLaneChangeEdgeBlock @1 :Bool;
-  rightLaneChangeEdgeBlock @2 :Bool;
 
   enum TurnDirection {
     none @0;

cereal/deprecated.capnp

@@ -1,4 +1,4 @@
-using Cxx = import "/include/c++.capnp";
+using Cxx = import "./include/c++.capnp";
 $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
@@ -775,13 +775,3 @@ struct GpsTrajectory @0x8cfeb072f5301000 {
   x @0 :List(Float32);
   y @1 :List(Float32);
 }
-
-struct AudioFeedbackDEPRECATED @0xed47e3c075be372a {
-  audio @0 :AudioData;
-  blockNum @1 :UInt16;
-
-  struct AudioData {
-    data @0 :Data;
-    sampleRate @1 :UInt32;
-  }
-}

cereal/log.capnp

@@ -1,7 +1,7 @@
-using Cxx = import "/include/c++.capnp";
+using Cxx = import "./include/c++.capnp";
 $Cxx.namespace("cereal");
 
-using Car = import "/car.capnp";
+using Car = import "car.capnp";
 using Deprecated = import "deprecated.capnp";
 using Custom = import "custom.capnp";
 
@@ -75,6 +75,7 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     driverUnresponsive2 @37;
     driverUnresponsive3 @38;
     belowSteerSpeed @39;
+    lowBattery @40;
     accFaulted @41;
     sensorDataInvalid @42;
     commIssue @43;
@@ -106,17 +107,18 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     noGps @68;
     wrongCruiseMode @69;
     modeldLagging @70;
+    deviceFalling @71;
     fanMalfunction @72;
     cameraMalfunction @73;
     cameraFrameRate @74;
     processNotRunning @75;
     dashcamMode @76;
     selfdriveInitializing @77;
+    usbError @78;
     cruiseMismatch @79;
     canBusMissing @80;
     selfdrivedLagging @81;
     resumeBlocked @82;
-    carNotReady @103;
     steerTimeLimit @83;
     vehicleSensorsInvalid @84;
     locationdTemporaryError @85;
@@ -129,15 +131,9 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     aeb @92;
     userBookmark @95;
     excessiveActuation @96;
-    bigModelLoading @100;
+    audioFeedback @97;
-    bigModelFailed @102;
 
-    lowBatteryDEPRECATED @40;
     soundsUnavailableDEPRECATED @47;
-    deviceFallingDEPRECATED @71;
-    usbErrorDEPRECATED @78;
-    audioFeedbackDEPRECATED @97;
-    bigModelReadyDEPRECATED @101;
   }
 }
 
@@ -180,13 +176,13 @@ struct InitData {
 
   enum DeviceType {
     unknown @0;
-    neo @1;   # NEO, EON, & comma two
+    neo @1;
     chffrAndroid @2;
     chffrIos @3;
-    tici @4;  # comma three
+    tici @4;
     pc @5;
-    tizi @6;  # comma 3X
+    tizi @6;
-    mici @7;  # comma four
+    mici @7;
   }
 
   struct PandaInfo {
@@ -418,10 +414,6 @@ struct CanData {
 struct DeviceState @0xa4d8b5af2aa492eb {
   deviceType @45 :InitData.DeviceType;
 
-  # usb
-  chestnutPresent @51 :Bool;
-  usbState @52 :UsbState;
-
   networkType @22 :NetworkType;
   networkInfo @31 :NetworkInfo;
   networkStrength @24 :NetworkStrength;
@@ -692,67 +684,33 @@ struct PeripheralState {
   }
 }
 
-struct UsbState {
-  devices @0 :List(Device);
-
-  struct Device {
-    busnum @0 :UInt8;
-    devnum @1 :UInt8;
-    vendorId @2 :UInt16;
-    productId @3 :UInt16;
-    speedMbps @4 :UInt16;
-    manufacturer @6 :Text;
-    product @5 :Text;
-    linkErrorCount @7 :UInt16;
-    usb3Lane @8 :Usb3Lane;
-
-    enum Usb3Lane {
-      unknown @0;
-      a @1;
-      b @2;
-    }
-  }
-}
-
-struct ChestnutState {
-  tempC @0 :Float32;
-  memoryTempC @1 :Float32;
-  powerDrawW @2 :Float32;
-  powerLimitW @3 :Float32;
-  gpuUsagePercent @4 :UInt8;
-  gpuClockMhz @5 :UInt16;
-  fanSpeedRpm @6 :UInt16;
-  pcieLtssm @7 :UInt8;
-  supplyVoltage @8 :UInt16;  # mV
-  supplyCurrent @9 :Int16;  # mA
-}
-
 struct RadarState @0x9a185389d6fdd05f {
-  mdMonoTime @6 :UInt64;  # for debugging
+  mdMonoTime @6 :UInt64;
+  carStateMonoTime @11 :UInt64;
   radarErrors @13 :Car.RadarData.Error;
 
   leadOne @3 :LeadData;
   leadTwo @4 :LeadData;
 
   struct LeadData {
-    dRel @0 :Float32;  # m from the front bumper of the car
+    dRel @0 :Float32;
-    yRel @1 :Float32;  # m in car frame, left positive
+    yRel @1 :Float32;
-    vRel @2 :Float32;  # m/s relative longitudinal speed
+    vRel @2 :Float32;
-    vLead @4 :Float32;  # m/s absolute lead speed
+    aRel @3 :Float32;
-    vLeadK @8 :Float32;  # kalman-filtered lead speed
+    vLead @4 :Float32;
-    aLeadK @9 :Float32;  # kalman-filtered lead accel
+    dPath @6 :Float32;
-    present @11 :Bool;  # true if a lead is present
+    vLat @7 :Float32;
-    aLeadTau @12 :Float32;  # lead accel time constant
+    vLeadK @8 :Float32;
-    modelProb @13 :Float32;  # vision model lead probability
+    aLeadK @9 :Float32;
-    radar @14 :Bool;  # true if lead is radar-matched (vs vision-only)
+    fcw @10 :Bool;
-    radarTrackId @15 :Int32 = -1;  # for debugging
+    status @11 :Bool;
+    aLeadTau @12 :Float32;
+    modelProb @13 :Float32;
+    radar @14 :Bool;
+    radarTrackId @15 :Int32 = -1;
 
     deprecated :group {
-      aRel @3 :Float32;
       aLead @5 :Float32;
-      dPath @6 :Float32;
-      vLat @7 :Float32;
-      fcw @10 :Bool;
     }
   }
 
@@ -765,12 +723,11 @@ struct RadarState @0x9a185389d6fdd05f {
     calPerc @9 :Int8;
     canMonoTimes @10 :List(UInt64);
     cumLagMs @5 :Float32;
-    carStateMonoTime @11 :UInt64;
     radarErrors @12 :List(Car.RadarData.ErrorDEPRECATED);
   }
 }
 
-struct ExtrinsicsCalibration @0x96df70754d8390bc {
+struct LiveCalibrationData {
   calStatus @11 :Status;
   calCycle @2 :Int32;
   calPerc @3 :Int8;
@@ -814,31 +771,13 @@ struct SelfdriveState {
   alertStatus @5 :AlertStatus;
   alertSize @6 :AlertSize;
   alertType @7 :Text;
-  alertSound @13 :AudibleAlert;
+  alertSound @8 :Car.CarControl.HUDControl.AudibleAlert;
   alertHudVisual @12 :Car.CarControl.HUDControl.VisualAlert;
 
   # configurable driving settings
   experimentalMode @10 :Bool;
   personality @11 :LongitudinalPersonality;
 
-  enum AudibleAlert {
-    none @0;
-
-    engage @1;
-    disengage @2;
-    refuse @3;
-
-    warningSoft @4;
-    warningImmediate @5;
-
-    prompt @6;
-    promptRepeat @7;
-    promptDistracted @8;
-
-    preAlert @9;
-    complete @10;
-  }
-
   enum OpenpilotState @0xdbe58b96d2d1ac61 {
     disabled @0;
     preEnabled @1;
@@ -859,10 +798,6 @@ struct SelfdriveState {
     mid @2;
     full @3;
   }
-
-  deprecated :group {
-    alertSound @8 :Car.CarControl.HUDControl.AudibleAlert;
-  }
 }
 
 struct ControlsState @0x97ff69c53601abf1 {
@@ -983,7 +918,7 @@ struct ControlsState @0x97ff69c53601abf1 {
     alertStatus @38 :SelfdriveState.AlertStatus;
     alertSize @39 :SelfdriveState.AlertSize;
     alertType @44 :Text;
-    alertSound2 @56 :SelfdriveState.AudibleAlert;
+    alertSound2 @56 :Car.CarControl.HUDControl.AudibleAlert;
     engageable @41 :Bool;  # can OP be engaged?
     state @31 :SelfdriveState.OpenpilotState;
     enabled @19 :Bool;
@@ -1050,7 +985,6 @@ struct ModelDataV2 {
   timestampEof @3 :UInt64;
   modelExecutionTime @15 :Float32;
   rawPredictions @16 :Data;
-  big @27 :Bool;
 
   # predicted future position, orientation, etc..
   position @4 :XYZTData;
@@ -1067,6 +1001,7 @@ struct ModelDataV2 {
   roadEdgeStds @14 :List(Float32);
 
   # predicted lead cars
+  leads @11 :List(LeadDataV2);
   leadsV3 @18 :List(LeadDataV3);
 
   meta @12 :MetaData;
@@ -1076,9 +1011,8 @@ struct ModelDataV2 {
   action @26: Action;
 
   lateralPlannerSolutionDEPRECATED @25: Deprecated.LateralPlannerSolution;
-  leadsDEPRECATED @11 :List(LeadDataV2DEPRECATED);
 
-  struct LeadDataV2DEPRECATED {
+  struct LeadDataV2 {
     prob @0 :Float32; # probability that car is your lead at time t
     t @1 :Float32;
 
@@ -1198,7 +1132,7 @@ struct EncodeIndex {
   }
 }
 
-struct OperatingSystemLogEntry {
+struct AndroidLogEntry {
   id @0 :UInt8;
   ts @1 :UInt64;
   priority @2 :UInt8;
@@ -1336,7 +1270,7 @@ struct LateralPlan @0xe1e9318e2ae8b51e {
 struct LiveLocationKalman {
 
   # More info on reference frames:
-  # https://github.com/commaai/openpilot/tree/master/openpilot/common/transformations
+  # https://github.com/commaai/openpilot/tree/master/common/transformations
 
   positionECEF @0 : Measurement;
   positionGeodetic @1 : Measurement;
@@ -1391,9 +1325,9 @@ struct LiveLocationKalman {
 }
 
 
-struct DeviceMotion @0xc24ca2b57206b44d {
+struct LivePose {
   # More info on reference frames:
-  # https://github.com/commaai/openpilot/tree/master/openpilot/common/transformations
+  # https://github.com/commaai/openpilot/tree/master/common/transformations
   orientationNED @0 :XYZMeasurement;
   velocityDevice @1 :XYZMeasurement;
   accelerationDevice @2 :XYZMeasurement;
@@ -2186,11 +2120,8 @@ struct DriverMonitoringStateDEPRECATED @0xb83cda094a1da284 {
 
 struct DriverMonitoringState {
   lockout @0 :Bool;
-  lockoutCount @15 :Int8;
+  alertCountLockoutPercent @1 :Int8;
-  lockoutMinutesRemaining @11 :Int8;
+  alertTimeLockoutPercent @2 :Int8;
-  alert3Count @12 :Int8;
-  noResponseCount @13 :Int8;
-  noResponseForceDecel @14 :Bool;
 
   alwaysOn @3 :Bool;
   alwaysOnLockout @4 :Bool;
@@ -2254,11 +2185,6 @@ struct DriverMonitoringState {
     calibratedPercent @0 :Int8;
     offset @1 :Float32;
   }
-
-  deprecated :group {
-    alertCountLockoutPercent @1 :Int8;
-    alertTimeLockoutPercent @2 :Int8;
-  }
 }
 
 struct Boot {
@@ -2273,7 +2199,7 @@ struct Boot {
   }
 }
 
-struct VehicleParameters @0xd9058dcb967c2753 {
+struct LiveParametersData {
   valid @0 :Bool;
   gyroBias @1 :Float32;
   angleOffsetDeg @2 :Float32;
@@ -2307,8 +2233,8 @@ struct VehicleParameters @0xd9058dcb967c2753 {
   }
 }
 
-struct LateralTorqueParameters @0xe61690eb0b091692 {
+struct LiveTorqueParametersData {
-  valid @0 :Bool;
+  liveValid @0 :Bool;
   latAccelFactorRaw @1 :Float32;
   latAccelOffsetRaw @2 :Float32;
   frictionCoefficientRaw @3 :Float32;
@@ -2324,7 +2250,7 @@ struct LateralTorqueParameters @0xe61690eb0b091692 {
   calPerc @13 :Int8;
 }
 
-struct LateralDelay @0x98dfdb22c44df8d4 {
+struct LiveDelayData {
   lateralDelay @0 :Float32;
   validBlocks @1 :Int32;
   status @2 :Status;
@@ -2510,6 +2436,11 @@ struct AudioData {
   sampleRate @1 :UInt32;
 }
 
+struct AudioFeedback {
+  audio @0 :AudioData;
+  blockNum @1 :UInt16;
+}
+
 struct Touch {
   sec @0 :Int64;
   usec @1 :Int64;
@@ -2542,9 +2473,9 @@ struct Event {
     pandaStates @81 :List(PandaState);
     peripheralState @80 :PeripheralState;
     radarState @13 :RadarState;
-    radarTracks @131 :Car.RadarData;
+    liveTracks @131 :Car.RadarData;
     sendcan @17 :List(CanData);
-    extrinsicsCalibration @19 :ExtrinsicsCalibration;
+    liveCalibration @19 :LiveCalibrationData;
     carState @22 :Car.CarState;
     carControl @23 :Car.CarControl;
     carOutput @127 :Car.CarOutput;
@@ -2555,43 +2486,42 @@ struct Event {
     qcomGnss @31 :QcomGnss;
     gpsLocationExternal @48 :GpsLocationData;
     gpsLocation @21 :GpsLocationData;
-    vehicleParameters @61 :VehicleParameters;
+    liveParameters @61 :LiveParametersData;
-    lateralTorqueParameters @94 :LateralTorqueParameters;
+    liveTorqueParameters @94 :LiveTorqueParametersData;
-    lateralDelay @146 : LateralDelay;
+    liveDelay @146 : LiveDelayData;
     cameraOdometry @63 :CameraOdometry;
     thumbnail @66: Thumbnail;
     onroadEvents @134: List(OnroadEvent);
     carParams @69: Car.CarParams;
     driverMonitoringState @151 :DriverMonitoringState;
-    deviceMotion @129 :DeviceMotion;
+    livePose @129 :LivePose;
     modelV2 @75 :ModelDataV2;
     drivingModelData @128 :DrivingModelData;
     driverStateV2 @92 :DriverStateV2;
 
     # camera stuff, each camera state has a matching encode idx
-    narrowRoadCameraState @2 :FrameData;
+    roadCameraState @2 :FrameData;
-    cabinCameraState @70: FrameData;
+    driverCameraState @70: FrameData;
     wideRoadCameraState @74: FrameData;
-    narrowRoadEncodeIdx @15 :EncodeIndex;
+    roadEncodeIdx @15 :EncodeIndex;
-    cabinEncodeIdx @76 :EncodeIndex;
+    driverEncodeIdx @76 :EncodeIndex;
     wideRoadEncodeIdx @77 :EncodeIndex;
-    qNarrowRoadEncodeIdx @90 :EncodeIndex;
+    qRoadEncodeIdx @90 :EncodeIndex;
 
-    livestreamNarrowRoadEncodeIdx @117 :EncodeIndex;
+    livestreamRoadEncodeIdx @117 :EncodeIndex;
     livestreamWideRoadEncodeIdx @118 :EncodeIndex;
-    livestreamCabinEncodeIdx @119 :EncodeIndex;
+    livestreamDriverEncodeIdx @119 :EncodeIndex;
 
     # microphone data
     soundPressure @103 :SoundPressure;
     rawAudioData @147 :AudioData;
 
     # systems stuff
-    operatingSystemLog @20 :OperatingSystemLogEntry;
+    androidLog @20 :AndroidLogEntry;
     managerState @78 :ManagerState;
     procLog @33 :ProcLog;
     clocks @35 :Clocks;
     deviceState @6 :DeviceState;
-    chestnutState @152 :ChestnutState;
     logMessage @18 :Text;
     errorLogMessage @85 :Text;
 
@@ -2604,20 +2534,21 @@ struct Event {
     # driving feedback
     userBookmark @93 :UserBookmark;
     bookmarkButton @148 :UserBookmark;
+    audioFeedback @149 :AudioFeedback;
 
     lateralManeuverPlan @150 :LateralManeuverPlan;
 
     # *********** debug ***********
     testJoystick @52 :Joystick;
-    narrowRoadEncodeData @86 :EncodeData;
+    roadEncodeData @86 :EncodeData;
-    cabinEncodeData @87 :EncodeData;
+    driverEncodeData @87 :EncodeData;
     wideRoadEncodeData @88 :EncodeData;
-    qNarrowRoadEncodeData @89 :EncodeData;
+    qRoadEncodeData @89 :EncodeData;
     alertDebug @133 :DebugAlert;
 
-    livestreamNarrowRoadEncodeData @120 :EncodeData;
+    livestreamRoadEncodeData @120 :EncodeData;
     livestreamWideRoadEncodeData @121 :EncodeData;
-    livestreamCabinEncodeData @122 :EncodeData;
+    livestreamDriverEncodeData @122 :EncodeData;
 
     # *********** Custom: reserved for forks ***********
 
@@ -2653,7 +2584,6 @@ struct Event {
 
     # *********** legacy + deprecated ***********
     model @9 :Deprecated.ModelData; # TODO: rename modelV2 and mark this as deprecated
-    audioFeedbackDEPRECATED @149 :Deprecated.AudioFeedbackDEPRECATED;
     liveMpcDEPRECATED @36 :Deprecated.LiveMpcData;
     liveLongitudinalMpcDEPRECATED @37 :Deprecated.LiveLongitudinalMpcData;
     liveLocationKalmanDeprecatedDEPRECATED @51 :Deprecated.LiveLocationData;

cereal/messaging/__init__.py

@@ -7,43 +7,12 @@ import os
 import capnp
 import time
 
-from typing import Union
+from typing import Optional, List, Union, Dict
 
-from openpilot.cereal import log
+from cereal import log
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
 from openpilot.common.utils import MovingAverage
 
-__all__ = (
-  "NO_TRAVERSAL_LIMIT",
-  "Context",
-  "FrequencyTracker",
-  "IpcError",
-  "MultiplePublishersError",
-  "Poller",
-  "PubMaster",
-  "PubSocket",
-  "SocketEventHandle",
-  "SubMaster",
-  "SubSocket",
-  "delete_fake_prefix",
-  "drain_sock",
-  "drain_sock_raw",
-  "fake_event_handle",
-  "get_fake_prefix",
-  "log_from_bytes",
-  "new_message",
-  "pub_sock",
-  "recv_one",
-  "recv_one_or_none",
-  "recv_one_retry",
-  "recv_sock",
-  "reset_context",
-  "set_fake_prefix",
-  "sub_sock",
-  "toggle_fake_events",
-  "wait_for_one_event",
-)
-
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
@@ -53,8 +22,8 @@ def pub_sock(endpoint: str) -> PubSocket:
   return msgq.pub_sock(endpoint, segment_size)
 
 
-def sub_sock(endpoint: str, poller: Poller | None = None, addr: str = "127.0.0.1",
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: int | None = None) -> SubSocket:
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
   service = SERVICE_LIST.get(endpoint)
   segment_size = service.queue_size if service else 0
   return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
@@ -70,7 +39,7 @@ def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event
     return msg
 
 
-def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
+def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
   args = {
     'valid': False,
     'logMonoTime': int(time.monotonic() * 1e9),
@@ -85,14 +54,14 @@ def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp
   return dat
 
 
-def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> list[capnp.lib.capnp._DynamicStructReader]:
+def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
   """Receive all message currently available on the queue"""
   msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
   return [log_from_bytes(m) for m in msgs]
 
 
 # TODO: print when we drop packets?
-def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicStructReader | None:
+def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
   """Same as drain sock, but only returns latest message. Consider using conflate instead."""
   dat = None
 
@@ -113,14 +82,14 @@ def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicSt
   return dat
 
 
-def recv_one(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
+def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive()
   if dat is not None:
     dat = log_from_bytes(dat)
   return dat
 
 
-def recv_one_or_none(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
+def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive(non_blocking=True)
   if dat is not None:
     dat = log_from_bytes(dat)
@@ -179,27 +148,27 @@ class FrequencyTracker:
 
 
 class SubMaster:
-  def __init__(self, services: list[str], poll: str | None = None,
+  def __init__(self, services: List[str], poll: Optional[str] = None,
-               ignore_alive: list[str] | None = None, ignore_avg_freq: list[str] | None = None,
+               ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
-               ignore_valid: list[str] | None = None, addr: str = "127.0.0.1", frequency: float | None = None):
+               ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
     self.frame = -1
     self.services = services
-    self.seen = dict.fromkeys(services, False)
+    self.seen = {s: False for s in services}
-    self.updated = dict.fromkeys(services, False)
+    self.updated = {s: False for s in services}
-    self.recv_time = dict.fromkeys(services, 0.0)
+    self.recv_time = {s: 0. for s in services}
-    self.recv_frame = dict.fromkeys(services, 0)
+    self.recv_frame = {s: 0 for s in services}
     self.sock = {}
     self.data = {}
-    self.logMonoTime = dict.fromkeys(services, 0)
+    self.logMonoTime = {s: 0 for s in services}
 
     # zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
     on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
-    self.static_freq_services = {s for s in services if not on_demand[s]}
+    self.static_freq_services = set(s for s in services if not on_demand[s])
     self.alive = {s: on_demand[s] for s in services}
     self.freq_ok = {s: on_demand[s] for s in services}
     self.valid = {s: on_demand[s] for s in services}
 
-    self.freq_tracker: dict[str, FrequencyTracker] = {}
+    self.freq_tracker: Dict[str, FrequencyTracker] = {}
     self.poller = Poller()
     polled_services = set([poll, ] if poll is not None else services)
     self.non_polled_services = set(services) - polled_services
@@ -242,7 +211,7 @@ class SubMaster:
       msgs.append(recv_one_or_none(self.sock[s]))
     self.update_msgs(time.monotonic(), msgs)
 
-  def update_msgs(self, cur_time: float, msgs: list[capnp.lib.capnp._DynamicStructReader]) -> None:
+  def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1
     self.updated = dict.fromkeys(self.services, False)
     for msg in msgs:
@@ -265,21 +234,21 @@ class SubMaster:
       self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
       self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
 
-  def all_alive(self, service_list: list[str] | None = None) -> bool:
+  def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
 
-  def all_freq_ok(self, service_list: list[str] | None = None) -> bool:
+  def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
 
-  def all_valid(self, service_list: list[str] | None = None) -> bool:
+  def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
 
-  def all_checks(self, service_list: list[str] | None = None) -> bool:
+  def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
     return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
 
 
 class PubMaster:
-  def __init__(self, services: list[str]):
+  def __init__(self, services: List[str]):
     self.sock = {}
     for s in services:
       self.sock[s] = pub_sock(s)

cereal/messaging/bridge.cc

@@ -1,7 +1,7 @@
 #include <cassert>
 
-#include "openpilot/cereal/messaging/msgq_to_zmq.h"
+#include "cereal/messaging/msgq_to_zmq.h"
-#include "openpilot/cereal/services.h"
+#include "cereal/services.h"
 #include "common/util.h"
 
 ExitHandler do_exit;

cereal/messaging/bridge_zmq.cc

@@ -1,4 +1,4 @@
-#include "openpilot/cereal/messaging/bridge_zmq.h"
+#include "cereal/messaging/bridge_zmq.h"
 
 #include <cassert>
 #include <cstring>

cereal/messaging/messaging.h

@@ -8,7 +8,7 @@
 
 #include <capnp/serialize.h>
 
-#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "cereal/gen/cpp/log.capnp.h"
 #include "common/timing.h"
 #include "msgq/ipc.h"
 

cereal/messaging/msgq_to_zmq.cc

@@ -1,8 +1,8 @@
-#include "openpilot/cereal/messaging/msgq_to_zmq.h"
+#include "cereal/messaging/msgq_to_zmq.h"
 
 #include <cassert>
 
-#include "openpilot/cereal/services.h"
+#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;

cereal/messaging/msgq_to_zmq.h

@@ -8,7 +8,7 @@
 #include <vector>
 
 #include "msgq/impl_msgq.h"
-#include "openpilot/cereal/messaging/bridge_zmq.h"
+#include "cereal/messaging/bridge_zmq.h"
 
 class MsgqToZmq {
 public:

cereal/messaging/socketmaster.cc

@@ -3,8 +3,8 @@
 #include <string>
 #include <mutex>
 
-#include "openpilot/cereal/services.h"
+#include "cereal/services.h"
-#include "openpilot/cereal/messaging/messaging.h"
+#include "cereal/messaging/messaging.h"
 
 const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
 

cereal/messaging/tests/test_messaging.py

@@ -1,16 +1,16 @@
+import os
 import capnp
 import multiprocessing
 import numbers
 import random
 import threading
 import time
-from openpilot.common.test import OpenpilotTestCase
 from openpilot.common.parameterized import parameterized
+import pytest
 
-from openpilot.cereal import log
+from cereal import log, car
-from opendbc.car.structs import car
+import cereal.messaging as messaging
-import openpilot.cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
-from openpilot.cereal.services import SERVICE_LIST
 
 events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
 
@@ -23,6 +23,10 @@ def random_socks(num_socks=10):
 def random_bytes(length=1000):
   return bytes([random.randrange(0xFF) for _ in range(length)])
 
+def zmq_sleep(t=1):
+  if "ZMQ" in os.environ:
+    time.sleep(t)
+
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
@@ -47,7 +51,17 @@ def delayed_send(delay, sock, dat):
   threading.Timer(delay, send_func).start()
 
 
-class TestMessaging(OpenpilotTestCase):
+class TestMessaging:
+  def setUp(self):
+    # TODO: ZMQ tests are too slow; all sleeps will need to be
+    # replaced with logic to block on the necessary condition
+    if "ZMQ" in os.environ:
+      pytest.skip()
+
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep()
+
   @parameterized.expand(events)
   def test_new_message(self, evt):
     try:
@@ -74,6 +88,7 @@ class TestMessaging(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
 
     # no wait and no msgs in queue
     msgs = func(sub_sock)
@@ -81,11 +96,10 @@ class TestMessaging(OpenpilotTestCase):
     assert len(msgs) == 0
 
     # no wait but msgs are queued up
-    pub_sock.send(messaging.new_message(sock).to_bytes())
-    assert sub_sock.receive() is not None  # synchronize the PUB/SUB connection
     num_msgs = random.randrange(3, 10)
     for _ in range(num_msgs):
       pub_sock.send(messaging.new_message(sock).to_bytes())
+    time.sleep(0.1)
     msgs = func(sub_sock)
     assert isinstance(msgs, list)
     assert all(isinstance(msg, expected_type) for msg in msgs)
@@ -95,6 +109,7 @@ class TestMessaging(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock, timeout=100)
+    zmq_sleep()
 
     # no wait and no msg in queue, socket should timeout
     recvd = messaging.recv_sock(sub_sock)
@@ -112,7 +127,8 @@ class TestMessaging(OpenpilotTestCase):
   def test_recv_one(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=10)
+    sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
 
     # no msg in queue, socket should timeout
     recvd = messaging.recv_one(sub_sock)
@@ -125,10 +141,12 @@ class TestMessaging(OpenpilotTestCase):
     assert isinstance(recvd, capnp._DynamicStructReader)
     assert_carstate(msg.carState, recvd.carState)
 
+  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
   def test_recv_one_or_none(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock)
+    zmq_sleep()
 
     # no msg in queue, socket shouldn't block
     recvd = messaging.recv_one_or_none(sub_sock)
@@ -143,16 +161,19 @@ class TestMessaging(OpenpilotTestCase):
 
   def test_recv_one_retry(self):
     sock = "carState"
-    sock_timeout = 0.005
+    sock_timeout = 0.1
     pub_sock = messaging.pub_sock(sock)
     sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
+    zmq_sleep()
 
-    # wait 5 socket timeouts and make sure it's still retrying
+    # this test doesn't work with ZMQ since multiprocessing interrupts it
-    p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
+    if "ZMQ" not in os.environ:
-    p.start()
+      # wait 5 socket timeouts and make sure it's still retrying
-    time.sleep(sock_timeout*5)
+      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
-    assert p.is_alive()
+      p.start()
-    p.terminate()
+      time.sleep(sock_timeout*5)
+      assert p.is_alive()
+      p.terminate()
 
     # wait 5 socket timeouts before sending
     msg = random_carstate()

cereal/messaging/tests/test_pub_sub_master.py

@@ -1,16 +1,20 @@
 import random
 import time
-from typing import cast
+from typing import Sized, cast
-from collections.abc import Sized
 
-from openpilot.common.test import OpenpilotTestCase
+import cereal.messaging as messaging
-import openpilot.cereal.messaging as messaging
+from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
-from openpilot.cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
+                                                  random_bytes, random_carstate, assert_carstate, \
-                                                  random_bytes, random_carstate, assert_carstate
+                                                  zmq_sleep
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
 
 
-class TestSubMaster(OpenpilotTestCase):
+class TestSubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
 
   def test_init(self):
     sm = messaging.SubMaster(events)
@@ -38,6 +42,7 @@ class TestSubMaster(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sm = messaging.SubMaster([sock,])
+    zmq_sleep()
 
     msg = random_carstate()
     pub_sock.send(msg.to_bytes())
@@ -49,6 +54,7 @@ class TestSubMaster(OpenpilotTestCase):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sm = messaging.SubMaster([sock,])
+    zmq_sleep()
 
     for i in range(10):
       msg = messaging.new_message(sock)
@@ -60,17 +66,17 @@ class TestSubMaster(OpenpilotTestCase):
   def test_update_timeout(self):
     sock = random_sock()
     sm = messaging.SubMaster([sock,])
-    timeout = random.randrange(10, 30)
+    timeout = random.randrange(1000, 3000)
     start_time = time.monotonic()
     sm.update(timeout)
     t = time.monotonic() - start_time
     assert t >= timeout/1000.
-    assert t < 0.1
+    assert t < 3
     assert not any(sm.updated.values())
 
   def test_avg_frequency_checks(self):
     for poll in (True, False):
-      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "extrinsicsCalibration"],
+      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
                                poll=("modelV2" if poll else None),
                                frequency=(20. if not poll else None))
 
@@ -78,39 +84,50 @@ class TestSubMaster(OpenpilotTestCase):
         "carState": (20, 20),
         "modelV2": (20, 20 if poll else 10),
         "cameraOdometry": (20, 10),
-        "extrinsicsCalibration": (4, 4),
+        "liveCalibration": (4, 4),
         "carParams": (None, None),
         "userBookmark": (None, None),
       }
 
       for service, (max_freq, min_freq) in checks.items():
         if max_freq is not None:
-          assert min_freq is not None
           assert sm._check_avg_freq(service)
           assert sm.freq_tracker[service].max_freq == max_freq*1.2
           assert sm.freq_tracker[service].min_freq == min_freq*0.8
         else:
           assert not sm._check_avg_freq(service)
 
+  def test_alive(self):
+    pass
+
+  def test_ignore_alive(self):
+    pass
+
+  def test_valid(self):
+    pass
+
   # SubMaster should always conflate
   def test_conflate(self):
     sock = "carState"
     pub_sock = messaging.pub_sock(sock)
     sm = messaging.SubMaster([sock,])
 
-    pub_sock.send(messaging.new_message(sock).to_bytes())
-    sm.update(1000)  # synchronize the PUB/SUB connection
-
     n = 10
     for i in range(n+1):
       msg = messaging.new_message(sock)
       msg.carState.vEgo = i
       pub_sock.send(msg.to_bytes())
+      time.sleep(0.01)
     sm.update(1000)
     assert sm[sock].vEgo == n
 
 
-class TestPubMaster(OpenpilotTestCase):
+class TestPubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
 
   def test_init(self):
     messaging.PubMaster(events)
@@ -119,6 +136,7 @@ class TestPubMaster(OpenpilotTestCase):
     socks = random_socks()
     pm = messaging.PubMaster(socks)
     sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
+    zmq_sleep()
 
     # PubMaster accepts either a capnp msg builder or bytes
     for capnp in [True, False]:

cereal/messaging/tests/test_services.py

@@ -1,14 +1,13 @@
-import subprocess
+import os
 import tempfile
-
+from typing import Dict
-from openpilot.common.test import OpenpilotTestCase
 from openpilot.common.parameterized import parameterized
 
-import openpilot.cereal.services as services
+import cereal.services as services
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
 
 
-class TestServices(OpenpilotTestCase):
+class TestServices:
 
   @parameterized.expand(SERVICE_LIST.keys())
   def test_services(self, s):
@@ -18,5 +17,5 @@ class TestServices(OpenpilotTestCase):
 
   def test_generated_header(self):
     with tempfile.NamedTemporaryFile(suffix=".h") as f:
-      ret = subprocess.run(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11", shell=True).returncode
+      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
       assert ret == 0, "generated services header is not valid C"

cereal/services.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 from enum import IntEnum
+from typing import Optional
 
 
 # TODO: this should be automatically determined using the capnp schema
@@ -10,7 +11,7 @@ class QueueSize(IntEnum):
 
 
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: int | None = None,
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
                queue_size: QueueSize = QueueSize.SMALL):
     self.should_log = should_log
     self.frequency = frequency
@@ -25,7 +26,6 @@ _services: dict[str, tuple] = {
   "accelerometer": (True, 104., 104),
   "temperatureSensor": (True, 2., 200),
   "deviceState": (True, 2., 1),
-  "chestnutState": (True, 10., 10),
   "touch": (True, 20., 1),
   "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
   "controlsState": (True, 100., 10, QueueSize.MEDIUM),
@@ -33,15 +33,15 @@ _services: dict[str, tuple] = {
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
-  "narrowRoadEncodeIdx": (False, 20., 1),
+  "roadEncodeIdx": (False, 20., 1),
-  "radarTracks": (True, 20.),
+  "liveTracks": (True, 20.),
   "sendcan": (True, 100., 139, QueueSize.MEDIUM),
   "logMessage": (True, 0., None, QueueSize.BIG),
   "errorLogMessage": (True, 0., 1, QueueSize.BIG),
-  "extrinsicsCalibration": (True, 4., 4),
+  "liveCalibration": (True, 4., 4),
-  "lateralTorqueParameters": (True, 4., 1),
+  "liveTorqueParameters": (True, 4., 1),
-  "lateralDelay": (True, 4., 1),
+  "liveDelay": (True, 4., 1),
-  "operatingSystemLog": (True, 0.),
+  "androidLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
@@ -55,15 +55,15 @@ _services: dict[str, tuple] = {
   "qcomGnss": (True, 2.),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
-  "deviceMotion": (True, 20., 4),
+  "livePose": (True, 20., 4),
-  "vehicleParameters": (True, 20., 5),
+  "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 10),
   "thumbnail": (True, 1 / 60., 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
-  "narrowRoadCameraState": (True, 20., 20),
+  "roadCameraState": (True, 20., 20),
-  "cabinCameraState": (True, 20., 20),
+  "driverCameraState": (True, 20., 20),
-  "cabinEncodeIdx": (False, 20., 1),
+  "driverEncodeIdx": (False, 20., 1),
   "driverStateV2": (True, 20., 10),
   "driverMonitoringState": (True, 20., 10),
   "wideRoadEncodeIdx": (False, 20., 1),
@@ -71,15 +71,16 @@ _services: dict[str, tuple] = {
   "drivingModelData": (True, 20., 10),
   "modelV2": (True, 20., None, QueueSize.BIG),
   "managerState": (True, 2., 1),
-  "qNarrowRoadEncodeIdx": (False, 20.),
+  "qRoadEncodeIdx": (False, 20.),
   "userBookmark": (True, 0., 1),
   "soundPressure": (True, 10., 10),
   "rawAudioData": (False, 20.),
   "bookmarkButton": (True, 0., 1),
-  "narrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "audioFeedback": (True, 0., 1),
-  "cabinEncodeData": (False, 20., None, QueueSize.BIG),
+  "roadEncodeData": (False, 20., None, QueueSize.BIG),
+  "driverEncodeData": (False, 20., None, QueueSize.BIG),
   "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qNarrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
 
   # sunnypilot
   "modelManagerSP": (False, 1., 1, QueueSize.BIG),
@@ -99,11 +100,11 @@ _services: dict[str, tuple] = {
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
   "livestreamWideRoadEncodeIdx": (False, 20.),
-  "livestreamNarrowRoadEncodeIdx": (False, 20.),
+  "livestreamRoadEncodeIdx": (False, 20.),
-  "livestreamCabinEncodeIdx": (False, 20.),
+  "livestreamDriverEncodeIdx": (False, 20.),
   "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamNarrowRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamCabinEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
   "customReservedRawData0": (True, 0.),
 }
 SERVICE_LIST = {name: Service(*vals) for
@@ -124,7 +125,8 @@ def build_header():
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += f'  {{ "{k}", {{"{k}", {should_log}, {v.frequency:f}, {decimation:d}, {v.queue_size:d}}}}},\n'
+    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation, v.queue_size)
   h += "};\n"
 
   h += "#endif\n"

common/SConscript

@@ -0,0 +1,23 @@
+Import('env', 'envCython')
+
+common_libs = [
+  'params.cc',
+  'swaglog.cc',
+  'util.cc',
+  'ratekeeper.cc',
+]
+
+_common = env.Library('common', common_libs, LIBS="json11")
+Export('_common')
+
+if GetOption('extras'):
+  env.Program('tests/test_common',
+              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+
+# Cython bindings
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+
+common_python = [params_python]
+
+Export('common_python')

common/api/base.py

@@ -4,7 +4,7 @@ import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
 from openpilot.common.hardware.hw import Paths
-from openpilot.common.version import get_version
+from openpilot.system.version import get_version
 
 # name: jwt signature algorithm
 KEYS = {"id_rsa": "RS256",
@@ -38,7 +38,6 @@ class BaseApi:
     }
     if payload_extra is not None:
       payload.update(payload_extra)
-    assert self.private_key is not None
     token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
     if isinstance(token, bytes):
       token = token.decode('utf8')

common/basedir.py

@@ -1,4 +1,4 @@
 import os
 
 
-BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))

common/esim/esim.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import sys
-import time
 from openpilot.common.hardware import HARDWARE
 from openpilot.common.esim.base import LPABase, Profile
 
@@ -29,17 +27,6 @@ def print_profiles(lpa: LPABase) -> None:
     print(f'{i}. {p.iccid} (nickname: {p.nickname or "<none provided>"}) (provider: {p.provider}) - {"enabled" if p.enabled else "disabled"}')
 
 
-def execute_and_process_notifications(lpa: LPABase, operation) -> None:
-  try:
-    operation()
-  finally:
-    time.sleep(1)  # Need to wait for 1s after the operation is finished so the eUICC/modem can settle down.
-    try:
-      lpa.process_notifications()
-    except Exception as e:
-      print(f'failed to process eSIM notifications: {e}', file=sys.stderr)
-
-
 if __name__ == '__main__':
   parser = argparse.ArgumentParser(prog='esim.py', description='manage eSIM profiles on your comma device', epilog='comma.ai')
   sub = parser.add_subparsers(dest='cmd')
@@ -63,21 +50,18 @@ if __name__ == '__main__':
   args = parser.parse_args()
 
   lpa = HARDWARE.get_sim_lpa()
-  if not lpa.is_euicc():
-    raise SystemExit("no eUICC detected")
   if args.cmd == 'switch':
-    iccid = resolve_iccid(lpa, args.profile)
+    lpa.switch_profile(resolve_iccid(lpa, args.profile))
-    execute_and_process_notifications(lpa, lambda: lpa.switch_profile(iccid))
   elif args.cmd == 'delete':
     iccid = resolve_iccid(lpa, args.profile)
     confirm = input(f'are you sure you want to delete profile {iccid}? (y/N) ')
     if confirm == 'y':
-      execute_and_process_notifications(lpa, lambda: lpa.delete_profile(iccid))
+      lpa.delete_profile(iccid)
     else:
       print('cancelled')
       exit(0)
   elif args.cmd == 'download':
-    execute_and_process_notifications(lpa, lambda: lpa.download_profile(args.qr, args.name))
+    lpa.download_profile(args.qr, args.name)
   elif args.cmd == 'nickname':
     lpa.nickname_profile(resolve_iccid(lpa, args.profile), args.name)
   else:

common/esim/lpa.py

@@ -6,6 +6,7 @@ import fcntl
 import hashlib
 import os
 import requests
+import serial
 import subprocess
 import sys
 import termios
@@ -19,7 +20,6 @@ from pathlib import Path
 
 from openpilot.common.time_helpers import system_time_valid
 from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
-from openpilot.common.serial import Serial, SerialException
 
 GSMA_CI_BUNDLE = str(Path(__file__).parent / "gsma_ci_bundle.pem")
 
@@ -133,7 +133,7 @@ class AtClient:
     self._device = device
     self._baud = baud
     self._timeout = timeout
-    self._serial: Serial | None = None
+    self._serial: serial.Serial | None = None
 
   def send_raw(self, data: bytes) -> None:
     self._ensure_serial()
@@ -185,14 +185,14 @@ class AtClient:
         pass
       self._serial = None
     if self._serial is None:
-      self._serial = Serial(self._device, baudrate=self._baud, timeout=self._timeout)
+      self._serial = serial.Serial(self._device, baudrate=self._baud, timeout=self._timeout)
 
   def query(self, cmd: str) -> list[str]:
     self._ensure_serial()
     try:
       self._send(cmd)
       return self._expect()
-    except SerialException:
+    except serial.SerialException:
       self._ensure_serial(reconnect=True)
       self._send(cmd)
       return self._expect()
@@ -208,7 +208,7 @@ class AtClient:
     if self._serial:
       try:
         self._serial.reset_input_buffer()
-      except (OSError, SerialException, termios.error):
+      except (OSError, serial.SerialException, termios.error):
         self._ensure_serial(reconnect=True)
     for line in self.query(f'AT+CCHO="{ISDR_AID}"'):
       if line.startswith("+CCHO:") and (ch := line.split(":", 1)[1].strip()):
@@ -230,7 +230,7 @@ class AtClient:
       try:
         self._open_isdr_once()
         return
-      except (RuntimeError, TimeoutError, termios.error, SerialException):
+      except (RuntimeError, TimeoutError, termios.error, serial.SerialException):
         time.sleep(OPEN_ISDR_RETRY_DELAY_S)
         if attempt == OPEN_ISDR_RESET_ATTEMPT:
           self._reset_modem()
@@ -451,7 +451,7 @@ def process_notifications(client: AtClient) -> None:
       response = es10x_command(client, request)
       content = require_tag(require_tag(response, TAG_RETRIEVE_NOTIFICATION, "RetrieveNotificationsListResponse"),
                             TAG_OK, "RetrieveNotificationsListResponse")
-      pending_notif = next((content[start:end] for t, _, start, end in iter_tlv(content, with_positions=True) if t in (TAG_PROFILE_INSTALL_RESULT, 0x30)), None)
+      pending_notif = next((v for t, v in iter_tlv(content) if t in (TAG_PROFILE_INSTALL_RESULT, 0x30)), None)
       if pending_notif is None:
         raise RuntimeError("Missing PendingNotification")
 
@@ -682,7 +682,7 @@ def download_profile(client: AtClient, activation_code: str) -> str:
     session.close()
 
 
-class LPA(LPABase):
+class TiciLPA(LPABase):
   def __init__(self):
     if hasattr(self, '_client'):
       return

common/file_chunker.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import io
 import sys
 import math
 import os
@@ -22,12 +21,13 @@ def get_chunk_targets(path, file_size):
 
 def chunk_file(path, targets):
   manifest_path, *chunk_paths = targets
-  actual_num_chunks = max(1, math.ceil(os.path.getsize(path) / CHUNK_SIZE))
-  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
   with open(path, 'rb') as f:
-    for chunk_path in chunk_paths:
+    data = f.read()
-      with open(chunk_path, 'wb') as out:
+  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
-        out.write(f.read(CHUNK_SIZE))
+  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
+  for i, chunk_path in enumerate(chunk_paths):
+    with open(chunk_path, 'wb') as f:
+      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
   Path(manifest_path).write_text(str(len(chunk_paths)))
   os.remove(path)
 
@@ -39,41 +39,14 @@ def get_existing_chunks(path):
     return _chunk_paths(path, num_chunks)
   raise FileNotFoundError(path)
 
-class ChunkStream(io.RawIOBase):
+def read_file_chunked(path):
-  def __init__(self, paths):
-    self._paths = iter(paths)
-    self._f = None
-
-  def readable(self):
-    return True
-
-  def readinto(self, b):
-    n = 0
-    view = memoryview(b)
-    while n < len(b):
-      if self._f is None:
-        p = next(self._paths, None)
-        if p is None:
-          break
-        self._f = open(p, 'rb')
-      count = self._f.readinto(view[n:])
-      if not count:
-        self._f.close()
-        self._f = None
-        continue
-      n += count
-    return n
-
-def open_file_chunked(path):
   manifest_path = get_manifest_path(path)
   if os.path.isfile(manifest_path):
     num_chunks = int(Path(manifest_path).read_text().strip())
-    paths = [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
+    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
-  elif os.path.isfile(path):
+  if os.path.isfile(path):
-    paths = [path]
+    return Path(path).read_bytes()
-  else:
+  raise FileNotFoundError(path)
-    raise FileNotFoundError(path)
-  return io.BufferedReader(ChunkStream(paths))
 
 
 if __name__ == "__main__":

common/git.py

@@ -31,3 +31,12 @@ def get_origin(cwd: str | None = None) -> str:
     return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
   except subprocess.CalledProcessError:  # Not on a branch, fallback
     return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
+
+
+@cache
+def get_normalized_origin(cwd: str | None = None) -> str:
+  return get_origin(cwd) \
+    .replace("git@", "", 1) \
+    .replace(".git", "", 1) \
+    .replace("https://", "", 1) \
+    .replace(":", "/", 1)

common/hardware/__init__.py

@@ -0,0 +1,16 @@
+import os
+from typing import cast
+
+from openpilot.common.hardware.base import HardwareBase
+from openpilot.common.hardware.tici.hardware import Tici
+from openpilot.common.hardware.pc.hardware import Pc
+
+TICI = os.path.isfile('/TICI')
+AGNOS = os.path.isfile('/AGNOS')
+PC = not TICI
+
+
+if TICI:
+  HARDWARE = cast(HardwareBase, Tici())
+else:
+  HARDWARE = cast(HardwareBase, Pc())

common/hardware/base.h

@@ -5,7 +5,7 @@
 #include <map>
 #include <string>
 
-#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "cereal/gen/cpp/log.capnp.h"
 
 // no-op base hw class
 class HardwareNone {
@@ -15,7 +15,7 @@ public:
 
   static std::string get_serial() { return "cccccc"; }
 
-  static std::map<std::string, std::string> get_init_logs(bool route_log = false) {
+  static std::map<std::string, std::string> get_init_logs() {
     return {};
   }
 

common/hardware/base.py

@@ -2,7 +2,7 @@ import os
 from abc import abstractmethod, ABC
 from dataclasses import dataclass, fields
 
-from openpilot.cereal import log
+from cereal import log
 from openpilot.common.esim.base import LPABase
 
 NetworkType = log.DeviceState.NetworkType

common/hardware/hw.h

@@ -5,9 +5,9 @@
 #include "common/hardware/base.h"
 #include "common/util.h"
 
-#if __COMMA_HARDWARE__
+#if __TICI__
-#include "common/hardware/comma/hardware.h"
+#include "common/hardware/tici/hardware.h"
-#define Hardware HardwareComma
+#define Hardware HardwareTici
 #else
 #include "common/hardware/pc/hardware.h"
 #define Hardware HardwarePC

common/hardware/pc/hardware.py

@@ -1,7 +1,7 @@
-from openpilot.cereal import log
+from cereal import log
 from openpilot.common.hardware.base import HardwareBase
 
-class HardwarePc(HardwareBase):
+class Pc(HardwareBase):
   def get_device_type(self):
     return "pc"
 

common/hardware/tici/agnos.json

@@ -23,14 +23,14 @@
   },
   {
     "name": "abl",
-    "url": "https://commadist.azureedge.net/agnosupdate/abl-29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/abl-b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c.img.xz",
-    "hash": "29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35",
+    "hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
-    "hash_raw": "29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35",
+    "hash_raw": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
     "size": 274432,
     "sparse": false,
     "full_check": true,
     "has_ab": true,
-    "ondevice_hash": "29fd7ed1c012e599420764840f9f11286d34dbff4adaf102a447f06d8c5e0b35"
+    "ondevice_hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c"
   },
   {
     "name": "aop",
@@ -56,29 +56,29 @@
   },
   {
     "name": "boot",
-    "url": "https://commadist.azureedge.net/agnosupdate/boot-b30f5eef65ec3878f3aa3dcaf2cc95c09e2c1e661cd3a38e94da37dee76f68bd.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8.img.xz",
-    "hash": "b30f5eef65ec3878f3aa3dcaf2cc95c09e2c1e661cd3a38e94da37dee76f68bd",
+    "hash": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
-    "hash_raw": "b30f5eef65ec3878f3aa3dcaf2cc95c09e2c1e661cd3a38e94da37dee76f68bd",
+    "hash_raw": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
-    "size": 46897152,
+    "size": 17487872,
     "sparse": false,
     "full_check": true,
     "has_ab": true,
-    "ondevice_hash": "6650e4c46df99ae6dfd6ee895a34b8a2a3cc490a8ce18e16cc3c451c3f822b6e"
+    "ondevice_hash": "edca8bee1531e66953d107eeceeed2dc7b3ca46417e49d55508f94e58bf95db8"
   },
   {
     "name": "system",
-    "url": "https://commadist.azureedge.net/agnosupdate/system-5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img.xz",
-    "hash": "b134fd04e9da27fa1d359ea0f2742c216fa21a08b5c47e9be22ab3b0563d9b9b",
+    "hash": "78acfe16a7b62a3a91fc7a81f40a693e4468cec1c69df7d0b1e550aacc646113",
-    "hash_raw": "5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3",
+    "hash_raw": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
     "size": 4718592000,
     "sparse": true,
     "full_check": false,
     "has_ab": true,
-    "ondevice_hash": "91242772af771ae96fe2eebc105f2b80a7e1dbaaf6003c2574b62d51b806f468",
+    "ondevice_hash": "743142c5a898f27b2a1029cca42c8a5d5d1fc0096414422b850fe84c8d0b8342",
     "alt": {
-      "hash": "5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3",
+      "hash": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
-      "url": "https://commadist.azureedge.net/agnosupdate/system-5b6ce7965904a157fd3a134ccfcb854f9ca5c1cc2a26b7cb80a4fa4e1cc4aaa3.img",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img",
       "size": 4718592000
     }
   }
-]
+]

common/hardware/tici/agnos.py

@@ -12,14 +12,14 @@ import requests
 
 SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
 
-AGNOS_MANIFEST_FILE = "openpilot/system/hardware/comma/agnos.json"
+AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json"
 
 
 class StreamingDecompressor:
   def __init__(self, url: str) -> None:
     self.buf = b""
 
-    self.req = requests.get(url, stream=True, headers={'Accept-Encoding': 'identity'}, timeout=60)
+    self.req = requests.get(url, stream=True, headers={'Accept-Encoding': None}, timeout=60)
     self.it = self.req.iter_content(chunk_size=1024 * 1024)
     self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
     self.eof = False
@@ -228,7 +228,7 @@ def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, st
   cloudlog.info(f"Target slot {target_slot_number}")
 
   # set target slot as unbootable
-  subprocess.run(f"abctl --set_unbootable {target_slot_number}", shell=True)
+  os.system(f"abctl --set_unbootable {target_slot_number}")
 
   for partition in update:
     success = False

common/hardware/tici/all-partitions.json

@@ -339,51 +339,51 @@
   },
   {
     "name": "boot",
-    "url": "https://commadist.azureedge.net/agnosupdate/boot-5463317ca7a231f2ddd55a667a8c2b0d6692982359773ca875a99a0e6fd67fb4.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8.img.xz",
-    "hash": "5463317ca7a231f2ddd55a667a8c2b0d6692982359773ca875a99a0e6fd67fb4",
+    "hash": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
-    "hash_raw": "5463317ca7a231f2ddd55a667a8c2b0d6692982359773ca875a99a0e6fd67fb4",
+    "hash_raw": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
-    "size": 46897152,
+    "size": 17487872,
     "sparse": false,
     "full_check": true,
     "has_ab": true,
-    "ondevice_hash": "daefa6f897bdcc277e50dbf562fd63e57d50d9fc7a599f2d171208a490e725d9"
+    "ondevice_hash": "edca8bee1531e66953d107eeceeed2dc7b3ca46417e49d55508f94e58bf95db8"
   },
   {
     "name": "system",
-    "url": "https://commadist.azureedge.net/agnosupdate/system-2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img.xz",
-    "hash": "14ab660bb955604ab6c482b3f59e5239e2c40c54e798201afcaed43d5cb0dcfd",
+    "hash": "78acfe16a7b62a3a91fc7a81f40a693e4468cec1c69df7d0b1e550aacc646113",
-    "hash_raw": "2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d",
+    "hash_raw": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
     "size": 4718592000,
     "sparse": true,
     "full_check": false,
     "has_ab": true,
-    "ondevice_hash": "18666ca3b529304f5e7a8fc2e3c9dce715b55941e3965a7af43c613f8e18cfce",
+    "ondevice_hash": "743142c5a898f27b2a1029cca42c8a5d5d1fc0096414422b850fe84c8d0b8342",
     "alt": {
-      "hash": "2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d",
+      "hash": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
-      "url": "https://commadist.azureedge.net/agnosupdate/system-2e1ca22762e66898aae384fcc21f488cb015c020b6b588862a64fc8128f0ad1d.img",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img",
       "size": 4718592000
     }
   },
   {
     "name": "userdata_90",
-    "url": "https://commadist.azureedge.net/agnosupdate/userdata_90-871adca25cb254ad1704ce427f3faaae1027f74aff43a3e150188c8a1c3d3cb8.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_90-14a3fc6e9bd148b9deebf6ae9df2f1b3b759e629b337e41b6895864cdd51f630.img.xz",
-    "hash": "e4f59e5dcf4758759da9284479163e420febf1c118bf33c83b402e80e241608d",
+    "hash": "52160dd01b30b3dc572226e8d549a034b03bc328e80f1f4cd6a857b6dd447687",
-    "hash_raw": "871adca25cb254ad1704ce427f3faaae1027f74aff43a3e150188c8a1c3d3cb8",
+    "hash_raw": "14a3fc6e9bd148b9deebf6ae9df2f1b3b759e629b337e41b6895864cdd51f630",
     "size": 96636764160,
     "sparse": true,
     "full_check": true,
     "has_ab": false,
-    "ondevice_hash": "17b4ec7449e62c52be5df551542684b953e5c7ffc95103008043602f79ae7112"
+    "ondevice_hash": "3bbc052c7793087946b0cd668c1778f930084f6b00896aeebd193dfce53fa518"
   },
   {
     "name": "userdata_89",
-    "url": "https://commadist.azureedge.net/agnosupdate/userdata_89-9d84bd64ef237a32675f936594c16932f8acbb52cf2dd82373a0e718cd32dfcf.img.xz",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_89-425c69d021f4ee2f767963bf7b991d1a492485fa465c5f5d001e1cf7de3d62a0.img.xz",
-    "hash": "652e61ecb5b12250dbfeb9f12fd74af16b867b5ea9bf8a08b5731ffd63a76317",
+    "hash": "02a8c5512754d7781d930d242be3fad01fbce652297c69dfe8722dc92b18dc09",
-    "hash_raw": "9d84bd64ef237a32675f936594c16932f8acbb52cf2dd82373a0e718cd32dfcf",
+    "hash_raw": "425c69d021f4ee2f767963bf7b991d1a492485fa465c5f5d001e1cf7de3d62a0",
     "size": 95563022336,
     "sparse": true,
     "full_check": true,
     "has_ab": false,
-    "ondevice_hash": "acad63821bb48823c871910b1a363ab924d2635a7574cc5045e877c438d3012f"
+    "ondevice_hash": "3667d500f91b08a4671d7c09623eb6ae8fc905d9273893b50689e214005c7fa6"
   }
 ]

common/hardware/tici/hardware.h

@@ -9,7 +9,7 @@
 #include "common/util.h"
 #include "common/hardware/base.h"
 
-class HardwareComma : public HardwareNone {
+class HardwareTici : public HardwareNone {
 public:
   static std::string get_name() {
     static const std::string name = []() {
@@ -59,7 +59,7 @@ public:
     std::ofstream("/sys/class/leds/led:switch_2/brightness") << value << "\n";
   }
 
-  static std::map<std::string, std::string> get_init_logs(bool route_log = false) {
+  static std::map<std::string, std::string> get_init_logs() {
     std::map<std::string, std::string> ret = {
       {"/BUILD", util::read_file("/BUILD")},
       {"lsblk", util::check_output("lsblk -o NAME,SIZE,STATE,VENDOR,MODEL,REV,SERIAL")},
@@ -73,14 +73,12 @@ public:
     temp.erase(temp.find_last_not_of(std::string("\0\r\n", 3))+1);
     ret["boot temp"] = temp;
 
-    // TODO: these are too slow to do on route log inits. need to do it async?
+    // TODO: log something from system and boot
-    if (!route_log) {
+    for (std::string part : {"xbl", "abl", "aop", "devcfg", "xbl_config"}) {
-      for (std::string part : {"xbl", "abl", "aop", "devcfg", "xbl_config"}) {
+      for (std::string slot : {"a", "b"}) {
-        for (std::string slot : {"a", "b"}) {
+        std::string partition = part + "_" + slot;
-          std::string partition = part + "_" + slot;
+        std::string hash = util::check_output("sha256sum /dev/disk/by-partlabel/" + partition);
-          std::string hash = util::check_output("sha256sum /dev/disk/by-partlabel/" + partition);
+        ret[partition] = hash.substr(0, hash.find_first_of(" "));
-          ret[partition] = hash.substr(0, hash.find_first_of(" "));
-        }
       }
     }
 

common/hardware/tici/hardware.py

@@ -7,13 +7,14 @@ import time
 from functools import cached_property, lru_cache
 from pathlib import Path
 
-from openpilot.cereal import log
+from cereal import log
 from openpilot.common.utils import sudo_read, sudo_write
 from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
 from openpilot.common.esim.base import LPABase
 from openpilot.common.hardware.base import HardwareBase, ThermalConfig, ThermalZone
-from openpilot.common.hardware.comma.pins import GPIO
+from openpilot.common.esim.lpa import TiciLPA
-from openpilot.common.hardware.comma.amplifier import Amplifier
+from openpilot.common.hardware.tici.pins import GPIO
+from openpilot.common.hardware.tici.amplifier import Amplifier
 
 MODEM_STATE_PATH = "/dev/shm/modem"
 
@@ -57,17 +58,7 @@ def get_default_route_iface():
     routes = [(int(route[6]), route[0]) for line in f.readlines()[1:] if (route := line.split())[1] == "00000000" and int(route[3], 16) & 0x1]
   return min(routes)[1] if routes else None
 
-class HardwareComma(HardwareBase):
+class Tici(HardwareBase):
-  """
-    This platform covers the Snapdragon 845-based comma devices:
-    - tici = comma three
-    - tizi = comma 3X
-    - mici = comma four
-
-    We strictly use only the device codenames in this codebase for
-    consistency, though all user-facing UI should use the product names.
-  """
-
   @cached_property
   def amplifier(self):
     if self.get_device_type() == "mici":
@@ -154,8 +145,7 @@ class HardwareComma(HardwareBase):
     }
 
   def get_sim_lpa(self) -> LPABase:
-    from openpilot.common.esim.lpa import LPA
+    return TiciLPA()
-    return LPA()
 
   def get_imei(self):
     return self.get_modem_state().get('imei', '')
@@ -251,7 +241,7 @@ class HardwareComma(HardwareBase):
     return (self.read_param_file("/sys/class/power_supply/bms/voltage_now", int) * self.read_param_file("/sys/class/power_supply/bms/current_now", int) / 1e12)
 
   def shutdown(self):
-    subprocess.run("sudo poweroff", shell=True)
+    os.system("sudo poweroff")
 
   def get_thermal_config(self):
     intake, exhaust, gnss, bottomSoc = None, None, None, None
@@ -345,11 +335,11 @@ class HardwareComma(HardwareBase):
       self.amplifier.initialize_configuration()
 
     # Allow hardwared to write engagement status to kmsg
-    subprocess.run("sudo chmod a+w /dev/kmsg", shell=True)
+    os.system("sudo chmod a+w /dev/kmsg")
 
     # Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
     gpio_init(GPIO.SOM_ST_IO, True)
-    gpio_set(GPIO.SOM_ST_IO, True)
+    gpio_set(GPIO.SOM_ST_IO, 1)
 
     # *** IRQ config ***
 
@@ -399,21 +389,21 @@ class HardwareComma(HardwareBase):
     gpio_init(GPIO.STM_RST_N, True)
     gpio_init(GPIO.STM_BOOT0, True)
 
-    gpio_set(GPIO.STM_RST_N, True)
+    gpio_set(GPIO.STM_RST_N, 1)
-    gpio_set(GPIO.STM_BOOT0, False)
+    gpio_set(GPIO.STM_BOOT0, 0)
     time.sleep(0.01)
-    gpio_set(GPIO.STM_RST_N, False)
+    gpio_set(GPIO.STM_RST_N, 0)
 
   def recover_internal_panda(self):
     gpio_init(GPIO.STM_RST_N, True)
     gpio_init(GPIO.STM_BOOT0, True)
 
-    gpio_set(GPIO.STM_RST_N, True)
+    gpio_set(GPIO.STM_RST_N, 1)
-    gpio_set(GPIO.STM_BOOT0, True)
+    gpio_set(GPIO.STM_BOOT0, 1)
     time.sleep(0.01)
-    gpio_set(GPIO.STM_RST_N, False)
+    gpio_set(GPIO.STM_RST_N, 0)
     time.sleep(0.01)
-    gpio_set(GPIO.STM_BOOT0, False)
+    gpio_set(GPIO.STM_BOOT0, 0)
 
   def booted(self):
     # this normally boots within 8s, but on rare occasions takes 30+s
@@ -423,7 +413,7 @@ class HardwareComma(HardwareBase):
     return True
 
 if __name__ == "__main__":
-  t = HardwareComma()
+  t = Tici()
   t.initialize_hardware()
   t.set_power_save(False)
   print(t.get_sim_info())

common/hardware/tici/modem.py

@@ -3,15 +3,12 @@ import fcntl
 import json
 import logging
 import os
-import select
+import serial
 import signal
-import struct
 import subprocess
 import tempfile
-import termios
 import time
 
-from contextlib import contextmanager
 from ipaddress import IPv4Address, AddressValueError
 
 from enum import Enum
@@ -54,7 +51,7 @@ PPPD_CMD = [
   "novj", "novjccomp", "ipcp-accept-local", "ipcp-accept-remote", "nomagic",
   "user", '""', "password", '""',
 ]
-INITIAL_STATE: dict[str, object] = {
+INITIAL_STATE = {
   "seconds_since_boot": 0,
   "state": "INITIALIZING",
   "connected": False, "ip_address": "",
@@ -66,39 +63,6 @@ INITIAL_STATE: dict[str, object] = {
 }
 
 
-@contextmanager
-def _serial_port(port: str, baudrate: int):
-  fd = os.open(port, os.O_RDWR | os.O_NOCTTY)
-  try:
-    attrs = termios.tcgetattr(fd)
-    attrs[0] = 0
-    attrs[1] = 0
-    attrs[2] = termios.CLOCAL | termios.CREAD | termios.CS8
-    attrs[3] = 0
-    attrs[4] = attrs[5] = getattr(termios, f"B{baudrate}")
-    attrs[6][termios.VMIN] = 0
-    attrs[6][termios.VTIME] = 0
-    termios.tcsetattr(fd, termios.TCSANOW, attrs)
-    yield fd
-  finally:
-    os.close(fd)
-
-
-def _read_line(fd: int, timeout: float) -> bytes:
-  data = bytearray()
-  deadline = time.monotonic() + timeout
-  while True:
-    readable, _, _ = select.select([fd], [], [], max(0.0, deadline - time.monotonic()))
-    if not readable:
-      return bytes(data)
-    byte = os.read(fd, 1)
-    if not byte:
-      return bytes(data)
-    data.extend(byte)
-    if byte == b"\n":
-      return bytes(data)
-
-
 class State(Enum):
   INITIALIZING = "INITIALIZING"
   SEARCHING = "SEARCHING"
@@ -132,11 +96,10 @@ class PPPSession:
   def reset_data_port():
     """Drop DTR on PPP_PORT so the modem terminates any stuck PPP session."""
     try:
-      with _serial_port(PPP_PORT, 460800) as fd:
+      with serial.Serial(PPP_PORT, 460800, timeout=1) as s:
-        dtr = struct.pack("I", termios.TIOCM_DTR)
+        s.dtr = False
-        fcntl.ioctl(fd, termios.TIOCMBIC, dtr)
         time.sleep(0.2)
-        fcntl.ioctl(fd, termios.TIOCMBIS, dtr)
+        s.dtr = True
     except Exception as e:
       logging.warning(f"data port reset failed: {e}")
 
@@ -257,14 +220,12 @@ class Modem:
       os.close(fd)
       return []
     try:
-      with _serial_port(AT_PORT, 9600) as serial_fd:
+      with serial.Serial(AT_PORT, 9600, timeout=5) as ser:
-        termios.tcflush(serial_fd, termios.TCIFLUSH)
+        ser.reset_input_buffer()
-        command = (cmd + "\r").encode()
+        ser.write((cmd + "\r").encode())
-        while command:
-          command = command[os.write(serial_fd, command):]
         lines = []
         while True:
-          raw = _read_line(serial_fd, 5)
+          raw = ser.readline()
           if not raw:
             raise TimeoutError("AT timeout")
           line = raw.decode(errors="ignore").strip()
@@ -544,11 +505,7 @@ class Modem:
       except (AddressValueError, ValueError):
         pass
     if not dns_servers:
-      dns_servers = [
+      logging.warning(f"no cellular DNS servers reported by modem: {v!r}")
-        "8.8.8.8",  # Google
-        "1.1.1.1",  # Cloudflare
-      ]
-      logging.warning(f"no cellular DNS servers reported by modem: {v!r}; using fallback {dns_servers}")
     return dns_servers
 
   def _poll_byte_counters(self) -> dict:

common/hardware/tici/tests/test_agnos_updater.py

@@ -6,8 +6,7 @@ TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
 MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
 
 
-from openpilot.common.test import OpenpilotTestCase
+class TestAgnosUpdater:
-class TestAgnosUpdater(OpenpilotTestCase):
 
   def test_manifest(self):
     with open(MANIFEST) as f:

common/hardware/tici/tests/test_amplifier.py

@@ -1,17 +1,19 @@
-#!/usr/bin/env python3
+import pytest
-
 import time
+import random
 import subprocess
-import unittest
 
 from panda import Panda
-from openpilot.common.test import OpenpilotTestCase
+from openpilot.common.hardware import TICI, HARDWARE
-from openpilot.common.hardware import HARDWARE
+from openpilot.common.hardware.tici.amplifier import Amplifier
-from openpilot.common.hardware.comma.amplifier import Amplifier
 
 
-class TestAmplifier(OpenpilotTestCase):
+class TestAmplifier:
-  COMMA_HARDWARE_TEST = True
+
+  @classmethod
+  def setup_class(cls):
+    if not TICI:
+      pytest.skip()
 
   def setup_method(self):
     # clear dmesg
@@ -55,7 +57,7 @@ class TestAmplifier(OpenpilotTestCase):
       time.sleep(0.1)
 
       self.panda.set_siren(True)
-      time.sleep(0.1)
+      time.sleep(random.randint(0, 5))
 
       amp = Amplifier(debug=True)
       r = amp.initialize_configuration()
@@ -64,8 +66,4 @@ class TestAmplifier(OpenpilotTestCase):
       if self._check_for_i2c_errors(True):
         break
     else:
-      self.fail("didn't hit any i2c errors")
+      pytest.fail("didn't hit any i2c errors")
-
-
-if __name__ == "__main__":
-  unittest.main()

common/mock/__init__.py

@@ -1,19 +1,19 @@
 """
 Utilities for generating mock messages for testing.
-example in openpilot/common/tests/test_mock.py
+example in common/tests/test_mock.py
 """
 
 
 import functools
 import threading
-from openpilot.cereal.messaging import PubMaster
+from cereal.messaging import PubMaster
-from openpilot.cereal.services import SERVICE_LIST
+from cereal.services import SERVICE_LIST
-from openpilot.common.mock.generators import generate_deviceMotion
+from openpilot.common.mock.generators import generate_livePose
 from openpilot.common.realtime import Ratekeeper
 
 
 MOCK_GENERATOR = {
-  "deviceMotion": generate_deviceMotion
+  "livePose": generate_livePose
 }
 
 

common/mock/generators.py

@@ -0,0 +1,14 @@
+from cereal import messaging
+
+
+def generate_livePose():
+  msg = messaging.new_message('livePose')
+  meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
+  msg.livePose.orientationNED = meas
+  msg.livePose.velocityDevice = meas
+  msg.livePose.angularVelocityDevice = meas
+  msg.livePose.accelerationDevice = meas
+  msg.livePose.inputsOK = True
+  msg.livePose.posenetOK = True
+  msg.livePose.sensorsOK = True
+  return msg

common/parameterized.py

@@ -0,0 +1,54 @@
+import re
+import sys
+import pytest
+import inspect
+
+
+def _to_safe_name(s):
+  return re.sub(r"[^a-zA-Z0-9_]+", "_", str(s)).strip("_")
+
+
+class parameterized:
+  @staticmethod
+  def expand(cases):
+    cases = list(cases)
+
+    if not cases:
+      return lambda func: pytest.mark.skip("no parameterized cases")(func)
+
+    def decorator(func):
+      params = [p for p in inspect.signature(func).parameters if p != 'self']
+      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
+      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
+      expand_params = params[: len(normalized[0])]
+      if len(expand_params) == 1:
+        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
+      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
+
+    return decorator
+
+
+def parameterized_class(attrs, input_list=None):
+  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
+    params_list = attrs
+  else:
+    assert input_list is not None
+    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
+    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
+
+  def decorator(cls):
+    globs = sys._getframe(1).f_globals
+    for i, params in enumerate(params_list):
+      # append sanitized string param values so pytest -k can filter by them
+      suffix = "_".join(filter(None, (_to_safe_name(v) for v in params.values() if isinstance(v, str))))
+      name = f"{cls.__name__}_{i}" + (f"_{suffix}" if suffix else "")
+      new_cls = type(name, (cls,), dict(params))
+      new_cls.__module__ = cls.__module__
+      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
+      globs[name] = new_cls
+    # Don't collect the un-parametrised base, but return it so outer decorators
+    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
+    cls.__test__ = False
+    return cls
+
+  return decorator

common/params.py

@@ -0,0 +1,19 @@
+from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+assert Params
+assert ParamKeyFlag
+assert ParamKeyType
+assert UnknownKeyName
+
+if __name__ == "__main__":
+  import sys
+
+  params = Params()
+  key = sys.argv[1]
+  assert params.check_key(key), f"unknown param: {key}"
+
+  if len(sys.argv) == 3:
+    val = sys.argv[2]
+    print(f"SET: {key} = {val}")
+    params.put(key, val, block=True)
+  elif len(sys.argv) == 2:
+    print(f"GET: {key} = {params.get(key)}")

common/params_keys.h

@@ -3,7 +3,7 @@
 #include <string>
 #include <unordered_map>
 
-#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "cereal/gen/cpp/log.capnp.h"
 
 inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
@@ -37,7 +37,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
     {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
     {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"DriverLockoutCount", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, INT, "0"}},
     {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
     {"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
     {"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
@@ -62,8 +61,10 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsMetric", {PERSISTENT | BACKUP, BOOL}},
     {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsOnroad", {PERSISTENT, BOOL}},
     {"IsRhdDetected", {PERSISTENT, BOOL}},
     {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
     {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
@@ -91,10 +92,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
     {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
     {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_ChestnutBranch", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
+    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
     {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
     {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
     {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
@@ -106,6 +108,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"PrimeType", {PERSISTENT, INT}},
     {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
+    {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RecordFront", {PERSISTENT | BACKUP, BOOL}},
     {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
     {"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
@@ -130,8 +133,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
     {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //
@@ -178,10 +181,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"RoadEdgeLaneChangeEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"ScreenSaverEnabled", {PERSISTENT | BACKUP, BOOL, "1"}},
-    {"ScreenSaverTimeout", {PERSISTENT | BACKUP, INT, "300"}},
     {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -195,14 +195,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
 
     // Model Manager params
     {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
-    {"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}},
     {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
     {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
     {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
     {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
-    {"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
     {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
-    {"ModelManager_ModelsCache_USBGPU", {PERSISTENT | BACKUP, JSON}},
 
     // Neural Network Lateral Control
     {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -227,7 +224,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
-    {"TeslaMadsScreenButton", {PERSISTENT | BACKUP, INT, "0"}},
     {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
 
@@ -278,7 +274,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     // Torque lateral control custom params
     {"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
     {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
-    {"LateralJerkTorqueController", {PERSISTENT | BACKUP, BOOL, "0"}},
     {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
     {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
     {"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},