sync: fix upstream conflicts

# Conflicts:
#	.github/labeler.yaml
#	.github/workflows/auto_pr_review.yaml
#	.github/workflows/docs.yaml
#	.github/workflows/release.yaml
#	.github/workflows/repo-maintenance.yaml
#	.github/workflows/tests.yaml
#	.github/workflows/ui_preview.yaml
#	README.md
#	SConstruct
#	conftest.py
#	docs/CARS.md
#	msgq_repo
#	opendbc_repo
#	openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py
#	openpilot/common/api.py
#	openpilot/common/hardware/hw.py
#	openpilot/common/version.py
#	openpilot/selfdrive/assets/fonts/Audiowide-Regular.ttf
#	openpilot/selfdrive/assets/sounds/prompt_single_high.wav
#	openpilot/selfdrive/assets/sounds/prompt_single_low.wav
#	openpilot/selfdrive/car/card.py
#	openpilot/selfdrive/car/helpers.py
#	openpilot/selfdrive/car/tests/test_car_interfaces.py
#	openpilot/selfdrive/car/tests/test_cruise_speed.py
#	openpilot/selfdrive/controls/lib/desire_helper.py
#	openpilot/selfdrive/controls/lib/longcontrol.py
#	openpilot/selfdrive/controls/plannerd.py
#	openpilot/selfdrive/controls/radard.py
#	openpilot/selfdrive/controls/tests/test_longcontrol.py
#	openpilot/selfdrive/modeld/compile_warp.py
#	openpilot/selfdrive/modeld/modeld.py
#	openpilot/selfdrive/monitoring/policy.py
#	openpilot/selfdrive/monitoring/test_monitoring.py
#	openpilot/selfdrive/selfdrived/events.py
#	openpilot/selfdrive/selfdrived/selfdrived.py
#	openpilot/selfdrive/ui/layouts/onboarding.py
#	openpilot/selfdrive/ui/mici/layouts/onboarding.py
#	openpilot/selfdrive/ui/soundd.py
#	openpilot/selfdrive/ui/tests/diff/replay.py
#	openpilot/selfdrive/ui/translations/app.pot
#	openpilot/system/athena/athenad.py
#	openpilot/system/athena/manage_athenad.py
#	openpilot/system/hardware/hardwared.py
#	openpilot/system/loggerd/config.py
#	openpilot/system/manager/github_runner.sh
#	openpilot/system/manager/manager.py
#	openpilot/system/updated/updated.py
#	panda
#	pyproject.toml
#	scripts/lint/lint.sh
#	system/manager/process_config.py
#	system/statsd.py
#	tinygrad_repo
#	tools/release/build_release.sh
#	tools/release/build_stripped.sh
#	uv.lock

.dockerignore

@@ -1,18 +0,0 @@
-**/.git
-.DS_Store
-*.dylib
-*.DSYM
-*.d
-*.pyc
-*.pyo
-.*.swp
-.*.swo
-.*.un~
-*.tmp
-*.o
-*.o-*
-*.os
-*.os-*
-
-venv/
-.venv/

.gitattributes

@@ -10,5 +10,5 @@
 *.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
-selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
+openpilot/selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+openpilot/common/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text

.github/labeler.yaml

@@ -8,24 +8,24 @@ chore:
 
 car:
   - changed-files:
-    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
+    - any-glob-to-all-files: '{openpilot/selfdrive/car/**,opendbc_repo}'
 
 simulation:
   - changed-files:
-    - any-glob-to-all-files: 'tools/sim/**'
+    - any-glob-to-all-files: 'openpilot/tools/sim/**'
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
+    - any-glob-to-all-files: '{openpilot/selfdrive/assets/**,openpilot/selfdrive/ui/**,openpilot/system/ui/**}'
 
 tools:
   - changed-files:
-    - any-glob-to-all-files: 'tools/**'
+    - any-glob-to-all-files: 'openpilot/tools/**'
 
 multilanguage:
   - changed-files:
-    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
+    - any-glob-to-all-files: 'openpilot/selfdrive/ui/translations/**'
 
 autonomy:
   - changed-files:
-    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit,sunnypilot/modeld*/models/**}"
+    - any-glob-to-all-files: "{openpilot/selfdrive/modeld/models/**,openpilot/selfdrive/test/process_replay/model_replay_ref_commit,openpilot/sunnypilot/modeld*/models/**}"

.github/pull_request_template.md

@@ -44,8 +44,8 @@ Explain how you tested this bug fix.
 
 **Checklist**
 
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
+- [ ] added entry to CAR in openpilot/selfdrive/car/*/values.py and ran `openpilot/selfdrive/car/docs.py` to generate new docs
-- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
+- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/openpilot/selfdrive/car/tests/routes.py)
 - [ ] route with openpilot:
 - [ ] route with stock system:
 - [ ] car harness used (if comma doesn't sell it, put N/A):

.github/workflows/auto_pr_review.yaml

@@ -12,7 +12,7 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: false
 
@@ -24,7 +24,7 @@ jobs:
 
     # Check PR target branch
     - name: check branch
-      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
+      uses: Vankka/pr-target-branch-action@5da68a42bcb7b43d39104295a876a6f3f8d7908b
       if: github.repository == 'sunnypilot/sunnypilot'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cereal_validation.yaml

@@ -6,7 +6,7 @@ on:
       - master
   pull_request:
     paths:
-      - 'cereal/**'
+      - 'openpilot/cereal/**'
   workflow_dispatch:
   workflow_call:
     inputs:
@@ -30,11 +30,25 @@ jobs:
       - name: Checkout sunnypilot cereal
         uses: actions/checkout@v6
         with:
-          sparse-checkout: cereal
+          sparse-checkout: |
+            openpilot/cereal
+            uv.lock
+          submodules: false
 
       - name: Init sunnypilot opendbc submodule
         run: git submodule update --init --depth 1 opendbc_repo
 
+      - name: Locate sunnypilot capnp import paths
+        id: locate-sp-capnp
+        run: |
+          SP_IMPORT_ARGS=""
+          SP_CAR_CAPNP=$(find opendbc_repo -maxdepth 4 -name car.capnp -path '*/opendbc/car/car.capnp' -printf '%h\n' -quit)
+          if [ -n "$SP_CAR_CAPNP" ]; then
+            SP_IMPORT_ARGS="-I $SP_CAR_CAPNP"
+            echo "Found sunnypilot car.capnp at: $SP_CAR_CAPNP"
+          fi
+          echo "import_args=$SP_IMPORT_ARGS" >> "$GITHUB_OUTPUT"
+
       - name: Checkout upstream openpilot
         uses: actions/checkout@v6
         with:
@@ -65,20 +79,19 @@ jobs:
           fi
           echo "import_args=$IMPORT_ARGS" >> "$GITHUB_OUTPUT"
 
-      - name: Install uv
+      - name: Install pycapnp
-        run: pip install uv
+        run: |
+          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          pip install "pycapnp==${PYCAPNP_VER}"
 
       - name: Generate sunnypilot schema
         run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
-          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
+            -g -f /tmp/sp_schema.json --cereal-dir openpilot/cereal \
-            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
+            ${{ steps.locate-sp-capnp.outputs.import_args }}
-            -g -f /tmp/sp_schema.json --cereal-dir cereal
 
       - name: Validate against upstream
         run: |
-          PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
+          python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
-          uv run --isolated --with "pycapnp==${PYCAPNP_VER}" \
-            python3 cereal/messaging/tests/validate_sp_cereal_upstream.py \
             -r -f /tmp/sp_schema.json --cereal-dir ${{ steps.locate-capnp.outputs.cereal_dir }} \
             ${{ steps.locate-capnp.outputs.import_args }}

.github/workflows/diff_report.yaml

@@ -17,29 +17,41 @@ jobs:
     - name: Wait for process replay
       id: wait
       continue-on-error: true
-      uses: lewagon/wait-on-check-action@v1.3.4
+      uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
       with:
         ref: ${{ github.event.pull_request.head.sha }}
         check-name: process replay
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         allowed-conclusions: success,failure
         wait-interval: 20
+    - name: Get tests run ID
+      if: steps.wait.outcome == 'success'
+      id: tests-run
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        run_id=$(gh api \
+          "repos/${{ github.repository }}/actions/workflows/tests.yaml/runs?event=pull_request&head_sha=${{ github.event.pull_request.head.sha }}&per_page=1" \
+          --jq '.workflow_runs[0].id // empty')
+        if [ -z "$run_id" ]; then
+          echo "No tests.yaml run found for ${{ github.event.pull_request.head.sha }}" >&2
+          exit 1
+        fi
+        echo "run-id=$run_id" >> "$GITHUB_OUTPUT"
     - name: Download diff
       if: steps.wait.outcome == 'success'
-      uses: dawidd6/action-download-artifact@v6
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
       with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
+        github-token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: tests.yaml
+        repository: ${{ github.repository }}
-        workflow_conclusion: ''
+        run-id: ${{ steps.tests-run.outputs.run-id }}
-        pr: ${{ github.event.number }}
         name: diff_report_${{ github.event.number }}
         path: .
-        allow_forks: true
     - name: Comment on PR
       if: steps.wait.outcome == 'success'
-      uses: thollander/actions-comment-pull-request@v2
+      uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
       with:
-        filePath: diff_report.txt
+        file-path: diff_report.txt
-        comment_tag: diff_report
+        comment-tag: diff_report
-        pr_number: ${{ github.event.number }}
+        pr-number: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -22,7 +22,7 @@ jobs:
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: true
 
@@ -34,7 +34,7 @@ jobs:
         python scripts/docs.py build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs
@@ -45,7 +45,7 @@ jobs:
       run: |
         set -x
 
-        source release/identity.sh
+        source tools/release/identity.sh
 
         cd openpilot-docs
         git checkout --orphan tmp

.github/workflows/jenkins-pr-trigger.yaml

@@ -11,7 +11,7 @@ jobs:
       contents: write
     steps:
       - name: Delete stale Jenkins branches
-        uses: actions/github-script@v8
+        uses: actions/github-script@v9
         with:
           script: |
             const cutoff = Date.now() - 24 * 60 * 60 * 1000;
@@ -52,7 +52,7 @@ jobs:
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v8
+      uses: actions/github-script@v9
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +72,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -86,7 +86,7 @@ jobs:
 
     - name: Delete trigger comment
       if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v8
+      uses: actions/github-script@v9
       with:
         script: |
           await github.rest.issues.deleteComment({

.github/workflows/model_review.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: [opened, reopened, synchronize]
     paths:
-      - 'selfdrive/modeld/models/*.onnx'
+      - 'openpilot/selfdrive/modeld/models/*.onnx'
   workflow_dispatch:
 
 jobs:
@@ -16,11 +16,11 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           submodules: true
       - name: Checkout master
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           ref: master
           path: base
@@ -36,7 +36,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Post model report comment
-        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
+        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0
         with:
           header: model-review
           message: ${{ steps.report.outputs.content }}

.github/workflows/release.yaml

@@ -15,17 +15,17 @@ jobs:
     steps:
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
-      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
+      uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build __nightly'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v4 # checkout version > v4 breaks nightly release script
       with:
         submodules: true
         fetch-depth: 0
     - run: ./tools/op.sh setup
     - name: Push __nightly
-      run: BRANCH=__nightly release/build_stripped.sh
+      run: BRANCH=__nightly tools/release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -3,6 +3,8 @@ name: repo maintenance
 on:
   schedule:
     - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
+  pull_request:
+    types: [closed]
   workflow_dispatch:
 
 env:
@@ -12,9 +14,11 @@ jobs:
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
-    if: github.repository == 'sunnypilot/sunnypilot'
+    if: >-
+      github.repository == 'sunnypilot/sunnypilot' &&
+      (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: true
     - run: ./tools/op.sh setup
@@ -50,10 +54,10 @@ jobs:
         git add .
     - name: update car docs
       run: |
-        python selfdrive/car/docs.py
+        python openpilot/selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+      uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
       with:
         author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
         token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
@@ -75,3 +79,43 @@ jobs:
           ${{ steps.pip_tree.outputs.PIP_TREE }}
           ```
         labels: bot
+
+  cleanup_closed_branches:
+    if: github.repository == 'commaai/openpilot'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: read
+    steps:
+      - uses: actions/github-script@v9
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const upstream = `${owner}/${repo}`;
+
+            for await (const response of github.paginate.iterator(github.rest.pulls.list, {
+              owner,
+              repo,
+              state: 'closed',
+              per_page: 100,
+            })) {
+              for (const pr of response.data) {
+                if (pr.head.repo?.full_name !== upstream) continue;
+
+                const branch = pr.head.ref;
+                try {
+                  await github.rest.git.deleteRef({
+                    owner,
+                    repo,
+                    ref: `heads/${branch}`,
+                  });
+                  console.log(`Deleted branch ${branch} (PR #${pr.number})`);
+                } catch (error) {
+                  if (error.status === 422 || error.status === 403) {
+                    console.log(`Skipping branch ${branch} (PR #${pr.number}): ${error.message}`);
+                    continue;
+                  }
+                  throw error;
+                }
+              }
+            }

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -79,7 +79,7 @@ jobs:
             is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
             echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
 
-            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
+            stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
             echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
             echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
           fi
@@ -177,7 +177,7 @@ jobs:
       - name: Build Main Project
         run: |
           export PYTHONPATH="$BUILD_DIR"
-          ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
+          ./tools/release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
           cd $BUILD_DIR
           sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
           echo "Building sunnypilot's modeld_v2..."
@@ -208,17 +208,17 @@ jobs:
             --exclude='Jenkinsfile' \
             --exclude='**/release/' \
             --exclude='**/.github/' \
-            --exclude='**/selfdrive/ui/replay/' \
+            --exclude='**/openpilot/selfdrive/ui/replay/' \
             --exclude='**/__pycache__/' \
             --exclude='${{env.SCONS_CACHE_DIR}}' \
             --exclude='**/.git/' \
             --exclude='**/SConstruct' \
             --exclude='**/SConscript' \
             --exclude='**/.venv/' \
-            --exclude='selfdrive/modeld/models/*.onnx*' \
+            --exclude='openpilot/selfdrive/modeld/models/*.onnx*' \
-            --exclude='sunnypilot/modeld*/models/*.onnx*' \
+            --exclude='openpilot/sunnypilot/modeld*/models/*.onnx*' \
-            --exclude='third_party/*x86*' \
+            --exclude='openpilot/third_party/*x86*' \
-            --exclude='third_party/*Darwin*' \
+            --exclude='openpilot/third_party/*Darwin*' \
             --delete-excluded \
             --chown=comma:comma \
             ${BUILD_DIR}/ ${OUTPUT_DIR}/

.github/workflows/tests.yaml

@@ -35,27 +35,27 @@ jobs:
       STRIPPED_DIR: /tmp/releasepilot
       PYTHONPATH: /tmp/releasepilot
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: true
     - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
       with:
         timeout_minutes: 2
         max_attempts: 3
         command: git lfs pull
     - name: Build devel
-      timeout-minutes: 1
+      timeout-minutes: 3
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
+      run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
     - run: ./tools/op.sh setup
     - name: Build openpilot and run checks
       timeout-minutes: 30
       working-directory: ${{ env.STRIPPED_DIR }}
-      run: python3 system/manager/build.py
+      run: python3 openpilot/system/manager/build.py
     - name: Run tests
       timeout-minutes: 1
       working-directory: ${{ env.STRIPPED_DIR }}
-      run: release/check-dirty.sh
+      run: tools/release/check-dirty.sh
     - name: Check submodules
       if: github.repository == 'sunnypilot/sunnypilot'
       timeout-minutes: 3
@@ -75,15 +75,13 @@ jobs:
             export CHECK_PR_REFS=true
           fi
         fi
-        release/check-submodules.sh
+        tools/release/check-submodules.sh
 
   build_mac:
     name: build macOS
     runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
-      with:
-        submodules: true
     - name: Remove Homebrew from environment
       run: |
         FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
@@ -101,9 +99,7 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
-      with:
-        submodules: true
     - run: ./tools/op.sh setup
     - name: Static analysis
       timeout-minutes: 1
@@ -118,16 +114,18 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
-      with:
-        submodules: true
     - run: ./tools/op.sh setup
+    - name: Install raylib headless dependencies
+      if: ${{ !contains(runner.name, 'nsc') }}
+      run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
     - name: Build openpilot
       run: scons
     - name: Run unit tests
       timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
+      env:
+        RAYLIB_BACKEND: headless
       run: |
-        source selfdrive/test/setup_xvfb.sh
         # Pre-compile Python bytecode so each pytest worker doesn't need to
         $PYTEST --collect-only -m 'not slow' -qq
         MAX_EXAMPLES=1 $PYTEST -m 'not slow'
@@ -142,39 +140,37 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
-      with:
-        submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Run replay
       timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
       continue-on-error: ${{ github.ref == 'refs/heads/master' }}
-      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
+      run: openpilot/selfdrive/test/process_replay/test_processes.py -j$(nproc)
     - name: Print diff
       id: print-diff
       if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
+      run: cat openpilot/selfdrive/test/process_replay/diff.txt
     - name: Print diff report
       if: always()
-      run: cat selfdrive/test/process_replay/diff_report.txt
+      run: cat openpilot/selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v6
+    - uses: actions/upload-artifact@v7
       if: always()
       continue-on-error: true
       with:
         name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
+        path: openpilot/selfdrive/test/process_replay/diff.txt
     - name: Upload diff report
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       if: always() && github.event_name == 'pull_request'
       continue-on-error: true
       with:
         name: diff_report_${{ github.event.number }}
-        path: selfdrive/test/process_replay/diff_report.txt
+        path: openpilot/selfdrive/test/process_replay/diff_report.txt
     - name: Checkout ci-artifacts
       if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7
       with:
         repository: commaai/ci-artifacts
         ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -187,13 +183,13 @@ jobs:
         git config user.email "<>"
         git fetch origin process-replay || true
         git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
-        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
+        cp ${{ github.workspace }}/openpilot/selfdrive/test/process_replay/fakedata/*.zst .
         echo "${{ github.sha }}" > ref_commit
         git add .
         git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
     - name: Push refs
       if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
       with:
         timeout_minutes: 2
         max_attempts: 3
@@ -203,7 +199,7 @@ jobs:
       timeout-minutes: 4
       env:
         ONNXCPU: 1
-      run: $PYTEST selfdrive/test/process_replay/test_regen.py
+      run: $PYTEST openpilot/selfdrive/test/process_replay/test_regen.py
 
   simulator_driving:
     name: simulator driving
@@ -215,17 +211,16 @@ jobs:
       || fromJSON('["ubuntu-24.04"]') }}
     if: false  # FIXME: Started to timeout recently
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
-      with:
-        submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Driving test
       timeout-minutes: 2
-      run: |
+      env:
-        source selfdrive/test/setup_xvfb.sh
+        # MetaDrive renders offscreen through panda3d's EGL pipe on llvmpipe
-        pytest -s tools/sim/tests/test_metadrive_bridge.py
+        EGL_PLATFORM: surfaceless
+      run: pytest -s openpilot/tools/sim/tests/test_metadrive_bridge.py
 
   create_ui_report:
     name: Create UI Report
@@ -236,19 +231,21 @@ jobs:
       && fromJSON('["namespace-profile-amd64-8x16"]')
       || fromJSON('["ubuntu-24.04"]') }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
-        with:
-          submodules: true
       - run: ./tools/op.sh setup
+      - name: Install raylib headless dependencies
+        if: ${{ !contains(runner.name, 'nsc') }}
+        run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
       - name: Build openpilot
         run: scons
       - name: Create UI Report
+        env:
+          RAYLIB_BACKEND: headless
         run: |
-          source selfdrive/test/setup_xvfb.sh
+          python3 openpilot/selfdrive/ui/tests/diff/replay.py
-          python3 selfdrive/ui/tests/diff/replay.py
+          python3 openpilot/selfdrive/ui/tests/diff/replay.py --big
-          python3 selfdrive/ui/tests/diff/replay.py --big
       - name: Upload UI Report
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/diff/report
+          path: openpilot/selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -8,9 +8,9 @@ on:
     branches:
       - 'master'
     paths:
-      - 'selfdrive/assets/**'
+      - 'openpilot/selfdrive/assets/**'
-      - 'selfdrive/ui/**'
+      - 'openpilot/selfdrive/ui/**'
-      - 'system/ui/**'
+      - 'openpilot/system/ui/**'
   workflow_dispatch:
 
 env:
@@ -34,12 +34,12 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           submodules: true
 
       - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@v1.3.4
+        uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
         with:
           ref: ${{ env.SHA }}
           check-name: ${{ env.UI_JOB_NAME }}
@@ -53,7 +53,7 @@ jobs:
           echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
       - name: Getting proposed ui
-        uses: dawidd6/action-download-artifact@v6
+        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
@@ -62,7 +62,7 @@ jobs:
           path: ${{ github.workspace }}/pr_ui
 
       - name: Getting mici master ui
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -70,7 +70,7 @@ jobs:
           ref: openpilot_master_ui_mici_raylib
 
       - name: Getting big master ui
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -97,7 +97,7 @@ jobs:
           done
 
       - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
+        uses: AnimMouse/setup-ffmpeg@178e0b4a408f06ce40d896c3cd79f50fa6f8b0c3
 
       - name: Finding diffs
         if: github.event_name == 'pull_request_target'
@@ -115,13 +115,13 @@ jobs:
             cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
 
             diff_exit_code=0
-            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
+            python3 ${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/diff.py \
               "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
               "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
               "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
 
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
+            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
+            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
 
             REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
             if [ $diff_exit_code -eq 0 ]; then
@@ -156,7 +156,7 @@ jobs:
           for variant in $VARIANTS; do
             IFS=':' read -r name _ _ <<< "$variant"
             diff_name="${name}_diff"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
+            cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
             git add "${diff_name}_pr_${{ github.event.number }}.html"
           done
           git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
@@ -164,12 +164,12 @@ jobs:
 
       - name: Comment on PR
         if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@v2
+        uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
         with:
           message: |
             <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
             ## UI Preview
             ${{ steps.find_diff.outputs.COMMENT }}
-          comment_tag: run_id_ui_preview
+          comment-tag: run_id_ui_preview
-          pr_number: ${{ github.event.number }}
+          pr-number: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -15,6 +15,13 @@ a.out
 .cache/
 bin/
 
+# created at launch for TICI PYTHONPATH (PC uses editable installs via pyproject.toml)
+/msgq
+/opendbc
+/rednose
+/teleoprtc
+/tinygrad
+
 *.mp4
 *.dylib
 *.DSYM
@@ -44,18 +51,18 @@ bin/
 config.json
 compile_commands.json
 compare_runtime*.html
-selfdrive/modeld/models/tg_input_devices.json
+openpilot/selfdrive/modeld/models/tg_input_devices.json
 
 # build artifacts
 docs_site/
-selfdrive/pandad/pandad
+openpilot/selfdrive/pandad/pandad
-cereal/services.h
+openpilot/cereal/services.h
-cereal/gen
+openpilot/cereal/gen
-cereal/messaging/bridge
+openpilot/cereal/messaging/bridge
-selfdrive/ui/translations/tmp
+openpilot/selfdrive/ui/translations/tmp
-selfdrive/car/tests/cars_dump
+openpilot/selfdrive/car/tests/cars_dump
-system/camerad/camerad
+openpilot/system/camerad/camerad
-system/camerad/test/ae_gray_test
+openpilot/system/camerad/test/ae_gray_test
 
 .coverage*
 coverage.xml

.gitmodules

@@ -6,7 +6,7 @@
   url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/commaai/msgq.git
+  url = https://github.com/sunnypilot/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git
@@ -17,5 +17,5 @@
   path = tinygrad_repo
   url = https://github.com/sunnypilot/tinygrad.git
 [submodule "sunnypilot/neural_network_data"]
-  path = sunnypilot/neural_network_data
+	path = openpilot/sunnypilot/neural_network_data
   url = https://github.com/sunnypilot/neural-network-data.git

.vscode/launch.json

@@ -6,9 +6,9 @@
       "type": "pickString",
       "description": "Select the process to debug",
       "options": [
-        "selfdrive/controls/controlsd.py",
+        "openpilot/selfdrive/controls/controlsd.py",
-        "system/timed/timed.py",
+        "openpilot/system/timed/timed.py",
-        "tools/sim/run_bridge.py"
+        "openpilot/tools/sim/run_bridge.py"
       ]
     },
     {
@@ -16,7 +16,7 @@
       "type": "pickString",
       "description": "Select the process to debug",
       "options": [
-        "selfdrive/ui/ui"
+        "openpilot/selfdrive/ui/ui"
       ]
     },
     {

.vscode/settings.json

@@ -17,10 +17,5 @@
     "**/.git",
     "**/.venv",
     "**/__pycache__",
-    // exclude directories that should be using the symlinked version
-    "common/**",
-    "selfdrive/**",
-    "system/**",
-    "tools/**",
   ]
 }

Dockerfile.openpilot

@@ -1,38 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-ENV OPENPILOT_PATH=/home/$USER/openpilot
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
-
-COPY --chown=$USER . ${OPENPILOT_PATH}/
-
-ENV UV_BIN="/home/$USER/.local/bin/"
-ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
-ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
-RUN tools/setup_dependencies.sh && \
-    sudo rm -rf /var/lib/apt/lists/*
-
-USER root
-RUN git config --global --add safe.directory '*'

Jenkinsfile

@@ -12,7 +12,7 @@ def retryWithDelay(int maxRetries, int delay, Closure body) {
 def device(String ip, String step_label, String cmd) {
   withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
     def ssh_cmd = """
-ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
+ssh -o ControlMaster=auto -o ControlPath=/tmp/ssh_control_%C -o ControlPersist=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
 
 set -e
 
@@ -63,6 +63,8 @@ if [ -f /data/openpilot/launch_env.sh ]; then
   source /data/openpilot/launch_env.sh
 fi
 
+export LD_LIBRARY_PATH="\$(python -c 'import ffmpeg; print(ffmpeg.LIB_DIR)'):/usr/local/lib:\${LD_LIBRARY_PATH:-}"
+
 ln -snf ${env.TEST_DIR} /data/pythonpath
 
 cd ${env.TEST_DIR} || true
@@ -93,7 +95,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
           retry (3) {
             def date = sh(script: 'date', returnStdout: true).trim();
             device(device_ip, "set time", "date -s '" + date + "'")
-            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+            device(device_ip, "git checkout", extra + "\n" + readFile("openpilot/selfdrive/test/setup_device_ci.sh"))
           }
           steps.each { item ->
             def name = item[0]
@@ -179,7 +181,7 @@ node {
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/release/build_release.sh"),
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/tools/release/build_release.sh"),
       ])
     }
 
@@ -187,12 +189,12 @@ node {
       parallel (
         'nightly': {
           deviceStage("build nightly", "tizi-needs-can", [], [
-            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
+            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/tools/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
           deviceStage("build nightly-dev", "tizi-needs-can", [], [
-            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
+            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/tools/release/build_release.sh"),
           ])
         },
       )
@@ -202,51 +204,51 @@ node {
     parallel (
       'onroad tests': {
         deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
-          step("build openpilot", "cd system/manager && ./build.py"),
+          step("build openpilot", "cd openpilot/system/manager && ./build.py"),
-          step("check dirty", "release/check-dirty.sh"),
+          step("check dirty", "tools/release/check-dirty.sh"),
-          step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
+          step("onroad tests", "pytest openpilot/selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
+          step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
+          step("test power draw", "pytest -s openpilot/selfdrive/test//test_power_draw.py"),
-          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
+          step("test encoder", "pytest openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
-          step("test manager", "pytest system/manager/test/test_manager.py"),
+          step("test manager", "pytest openpilot/system/manager/test/test_manager.py"),
         ])
       },
       'camerad OX03C10': {
         deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
+          step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test pandad", "pytest openpilot/selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
+          step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test pandad", "pytest openpilot/selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'sensord': {
         deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
+          step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+          step("test sensord", "pytest openpilot/system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
         deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+          step("build", "cd openpilot/system/manager && ./build.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
-          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+          step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
         ])
       },
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
-          step("build openpilot", "cd system/manager && ./build.py"),
+          step("build openpilot", "cd openpilot/system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "pytest openpilot/selfdrive/pandad/tests/test_pandad_loopback.py"),
-          step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad spi", "pytest openpilot/selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          step("test amp", "pytest openpilot/common/hardware/tici/tests/test_amplifier.py"),
         ])
       },
 

RELEASES.md

@@ -1104,7 +1104,7 @@ Version 0.2.1  (2016-12-14)
 
 Version 0.2  (2016-12-12)
 =========================
- * Car/Radar abstraction layers have shipped, see cereal/car.capnp
+ * Car/Radar abstraction layers have shipped, see openpilot/cereal/car.capnp
  * controlsd has been refactored
  * Shipped plant model and testing maneuvers
  * visiond exits more gracefully now

SConstruct

@@ -27,6 +27,21 @@ AddOption('--minimal',
           default=(not TICI and not release),
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
+submodule_python_paths = [
+  Dir("#").abspath,
+  Dir("#msgq_repo").abspath,
+  Dir("#opendbc_repo").abspath,
+  Dir("#rednose_repo").abspath,
+  Dir("#teleoprtc_repo").abspath,
+  Dir("#tinygrad_repo").abspath,
+]
+for p in reversed(submodule_python_paths):
+  if p not in sys.path:
+    sys.path.insert(0, p)
+
+if external_pythonpath := os.environ.get("PYTHONPATH"):
+  submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths]
+
 # Detect platform
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
@@ -40,9 +55,24 @@ assert arch in [
   "Darwin",   # macOS arm64 (x86 not supported)
 ]
 
-pkg_names = ['acados', 'bzip2', 'capnproto', 'catch2', 'eigen', 'ffmpeg', 'json11', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
+pkg_names = ['acados', 'bzip2', 'capnproto', 'catch2', 'eigen', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zstd']
 pkgs = [importlib.import_module(name) for name in pkg_names]
 acados = pkgs[pkg_names.index('acados')]
+ffmpeg = pkgs[pkg_names.index('ffmpeg')]
+# Shared package ships .so/.dylib; older device venvs still have static .a only.
+# Keep static link deps (x264/z/va/drm) when the installed package is static so
+# TICI CI works without upgrading the device venv yet.
+# TODO: drop the static fallback once device venvs have comma-deps-ffmpeg>=7.1.0.post94
+_ffmpeg_lib_names = os.listdir(ffmpeg.LIB_DIR) if os.path.isdir(ffmpeg.LIB_DIR) else []
+ffmpeg_shared = any(
+  n.startswith('libavcodec.so') or (n.startswith('libavcodec') and n.endswith('.dylib'))
+  for n in _ffmpeg_lib_names
+)
+ffmpeg_libs = ['avformat', 'avcodec', 'swresample', 'avutil']
+if not ffmpeg_shared:
+  ffmpeg_libs += ['x264', 'z']
+  if arch != "Darwin":
+    ffmpeg_libs += ['va', 'va-drm', 'drm']
 acados_include_dirs = [
   acados.INCLUDE_DIR,
   os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
@@ -91,7 +121,7 @@ def _libflags(target, source, env, for_signature):
 env = Environment(
   ENV={
     "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath,
+    "PYTHONPATH": os.pathsep.join(submodule_python_paths),
     "ACADOS_SOURCE_DIR": acados.DIR,
     "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
     "TERA_PATH": acados.TERA_PATH
@@ -112,22 +142,27 @@ env = Environment(
   CFLAGS=["-std=gnu11"],
   CXXFLAGS=["-std=c++1z"],
   CPPPATH=[
-    "#",
+    "#openpilot",
-    "#msgq",
+    "#msgq_repo",            # #include "msgq/..."
+    "#opendbc_repo",         # #include "opendbc/..."
+    "#rednose_repo",         # #include "rednose/..."
+    "#rednose_repo/rednose", # #include "logger/..." (rednose package root)
+    "#openpilot/cereal/gen/cpp",
     acados_include_dirs,
     [x.INCLUDE_DIR for x in pkgs],
+    "#",
   ],
   LIBPATH=[
-    "#common",
+    "#openpilot/common",
     "#msgq_repo",
-    "#selfdrive/pandad",
+    "#openpilot/selfdrive/pandad",
-    "#rednose/helpers",
+    "#rednose_repo/rednose/helpers",
     [x.LIB_DIR for x in pkgs],
   ],
-  RPATH=[],
+  RPATH=[ffmpeg.LIB_DIR] if ffmpeg_shared else [],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
-  REDNOSE_ROOT="#",
+  REDNOSE_ROOT="#rednose_repo",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
@@ -190,7 +225,7 @@ else:
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch', 'acados', 'release')
+Export('env', 'arch', 'acados', 'release', 'ffmpeg_libs')
 
 # Setup cache dir
 default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
@@ -211,7 +246,7 @@ def prune_cache_dir(target=None, source=None, env=None):
 # ********** start building stuff **********
 
 # Build common module
-SConscript(['common/SConscript'])
+SConscript(['openpilot/common/SConscript'])
 Import('_common')
 common = [_common, 'json11', 'zmq']
 Export('common')
@@ -222,7 +257,7 @@ env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
 
-SConscript(['cereal/SConscript'])
+SConscript(['openpilot/cereal/SConscript'])
 
 Import('socketmaster', 'msgq')
 messaging = [socketmaster, msgq, 'capnp', 'kj',]
@@ -233,34 +268,33 @@ Export('messaging')
 SConscript(['panda/SConscript'])
 
 # Build rednose library
-SConscript(['rednose/SConscript'])
+SConscript(['rednose_repo/rednose/SConscript'])
 
 # Build system services
 SConscript([
-  'system/loggerd/SConscript',
+  'openpilot/system/loggerd/SConscript',
 ])
 
 if arch == "larch64":
-  SConscript(['system/camerad/SConscript'])
+  SConscript(['openpilot/system/camerad/SConscript'])
 
 # Build selfdrive
 SConscript([
-  'selfdrive/pandad/SConscript',
+  'openpilot/selfdrive/pandad/SConscript',
-  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
+  'openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
+  'openpilot/selfdrive/locationd/SConscript',
-  'selfdrive/locationd/SConscript',
+  'openpilot/selfdrive/modeld/SConscript',
-  'selfdrive/modeld/SConscript',
+  'openpilot/selfdrive/ui/SConscript',
-  'selfdrive/ui/SConscript',
 ])
 
-SConscript(['sunnypilot/SConscript'])
+SConscript(['openpilot/sunnypilot/SConscript'])
 
 # Build desktop-only tools
 if GetOption('extras') and arch != "larch64":
   SConscript([
-    'tools/replay/SConscript',
+    'openpilot/tools/replay/SConscript',
-    'tools/cabana/SConscript',
+    'openpilot/tools/cabana/SConscript',
-    'tools/jotpluggler/SConscript',
+    'openpilot/tools/jotpluggler/SConscript',
   ])
 
 
@@ -276,6 +310,8 @@ def count_scons_nodes(nodes):
     if node in seen:
       continue
     seen.add(node)
+    if hasattr(node, 'has_builder') and node.has_builder():
+      build_product_nodes.add(node)
     executor = node.get_executor()
     if executor is not None:
       stack += executor.get_all_prerequisites() + executor.get_all_children()
@@ -284,6 +320,7 @@ def count_scons_nodes(nodes):
 
 progress_interval = 5
 progress_count = 0
+build_product_nodes = set()
 progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
 
 def progress_function(node):
@@ -299,3 +336,11 @@ def progress_function(node):
 
 Progress(progress_function, interval=progress_interval)
 AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)
+
+def check_build_product_size(target, source, env):
+  limit = 50 * 1024 * 1024  # GitHub max size
+  for t in target:
+    if hasattr(t, 'isfile') and t.isfile() and (size := os.path.getsize(t.abspath)) > limit:
+      raise SCons.Errors.UserError(f"{t} is {size / (1024 * 1024):.1f} MiB, exceeding the {limit / (1024 * 1024):.1f} MiB limit")
+if not GetOption('extras'):
+  AddPostAction(list(build_product_nodes), Action(check_build_product_size, None))

cereal/SConscript

@@ -1,20 +0,0 @@
-Import('env', 'common', 'msgq')
-
-cereal_dir = Dir('.')
-gen_dir = Dir('gen')
-
-# Build cereal
-schema_files = ['log.capnp', 'car.capnp', 'deprecated.capnp', 'custom.capnp']
-env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
-            schema_files,
-            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
-
-cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
-
-# Build messaging
-services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
-
-socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
-
-Export('cereal', 'socketmaster')

cereal/__init__.py

@@ -1,11 +0,0 @@
-import os
-import capnp
-from importlib.resources import as_file, files
-
-capnp.remove_import_hook()
-
-with as_file(files("cereal")) as fspath:
-  CEREAL_PATH = fspath.as_posix()
-  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
-  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
-  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -1 +0,0 @@
-../opendbc_repo/opendbc/car/car.capnp

common/tests/test_params.cc

@@ -1,27 +0,0 @@
-#include "catch2/catch.hpp"
-#define private public
-#include "common/params.h"
-#include "common/util.h"
-
-TEST_CASE("params_nonblocking_put") {
-  char tmp_path[] = "/tmp/asyncWriter_XXXXXX";
-  const std::string param_path = mkdtemp(tmp_path);
-  auto param_names = {"CarParams", "IsMetric"};
-  {
-    Params params(param_path);
-    for (const auto &name : param_names) {
-      params.putNonBlocking(name, "1");
-      // param is empty
-      REQUIRE(params.get(name).empty());
-    }
-
-    // check if thread is running
-    REQUIRE(params.future.valid());
-    REQUIRE(params.future.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout);
-  }
-  // check results
-  Params p(param_path);
-  for (const auto &name : param_names) {
-    REQUIRE(p.get(name) == "1");
-  }
-}

conftest.py

@@ -5,21 +5,27 @@ import pytest
 
 from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.system.manager import manager
-from openpilot.system.hardware import TICI, HARDWARE
+from openpilot.common.hardware import TICI, HARDWARE
 
 # these are heavy CI-only tests, invoked explicitly in .github/workflows/tests.yaml
 collect_ignore = [
-  "selfdrive/test/process_replay/test_processes.py",
+  "openpilot/selfdrive/test/process_replay/test_processes.py",
-  "selfdrive/test/process_replay/test_regen.py",
+  "openpilot/selfdrive/test/process_replay/test_regen.py",
+
+  "openpilot/tools/sim/",
+
   # tinygrad JIT has process-global state. Other test files import modeld → tinygrad,
   # which corrupts JIT captures for test_warp.py in the same process. Run separately in CI.
-  "sunnypilot/modeld_v2/tests/test_warp.py",
+  "openpilot/sunnypilot/modeld_v2/tests/test_warp.py",
-]
-collect_ignore_glob = [
-  "selfdrive/debug/*.py",
 ]
 
 
+def pytest_sessionstart(session):
+  # TODO: fix tests and enable test order randomization
+  if session.config.pluginmanager.hasplugin('randomly'):
+    session.config.option.randomly_reorganize = False
+
+
 @pytest.hookimpl(hookwrapper=True, trylast=True)
 def pytest_runtest_call(item):
   # ensure we run as a hook after capturemanager's

docs/CONTRIBUTING.md

@@ -68,6 +68,6 @@ A good pull request has all of the following:
 ### A guide for forks
 
 In order for your fork's data to be eligible for the training set:
-* **Your cereal messaging structs must be [compatible](../cereal#custom-forks)**
+* **Your cereal messaging structs must be [compatible](../openpilot/cereal#custom-forks)**
 * **The definitions of all the stock messaging structs must not change**: Do not change how any of the fields are set, including everything from `selfdriveState.enabled` to `carState.steeringAngleDeg`. Instead, create your own structs and set them however you'd like.
 * **Do not include cars that are not supported in upstream platforms**: Instead, create new opendbc platforms for cars that you'd like to support outside of upstream, even if it's just a trim-level difference.

docs/SAFETY.md

@@ -35,8 +35,8 @@ For additional safety implementation details, refer to [panda safety model](http
 
 ### Forks of openpilot
 
-* Do not disable or nerf [driver monitoring](https://github.com/commaai/openpilot/tree/master/selfdrive/monitoring)
+* Do not disable or nerf [driver monitoring](https://github.com/commaai/openpilot/tree/master/openpilot/selfdrive/monitoring)
-* Do not disable or nerf [excessive actuation checks](https://github.com/commaai/openpilot/tree/master/selfdrive/selfdrived/helpers.py)
+* Do not disable or nerf [excessive actuation checks](https://github.com/commaai/openpilot/tree/master/openpilot/selfdrive/selfdrived/helpers.py)
 * If your fork modifies any of the code in `opendbc/safety/`:
    * your fork cannot use the openpilot trademark
    * your fork must preserve the full [safety test suite](https://github.com/commaai/opendbc/tree/master/opendbc/safety/tests) and all tests must pass, including any new coverage required by the fork's changes

docs/concepts/logs.md

@@ -2,13 +2,13 @@
 
 openpilot records routes in one minute chunks called segments. A route starts on the rising edge of ignition and ends on the falling edge.
 
-Check out our [Python library](https://github.com/commaai/openpilot/blob/master/tools/lib/logreader.py) for reading openpilot logs. Also checkout our [tools](https://github.com/commaai/openpilot/tree/master/tools) to replay and view your data. These are the same tools we use to debug and develop openpilot.
+Check out our [Python library](https://github.com/commaai/openpilot/blob/master/openpilot/tools/lib/logreader.py) for reading openpilot logs. Also checkout our [tools](https://github.com/commaai/openpilot/tree/master/openpilot/tools) to replay and view your data. These are the same tools we use to debug and develop openpilot.
 
 For each segment, openpilot records the following log types:
 
 ## rlog.zst
 
-rlogs contain all the messages passed amongst openpilot's processes. See [cereal/services.py](https://github.com/commaai/openpilot/blob/master/cereal/services.py) for a list of all the logged services. They're a zstd archive of the serialized [Cap’n Proto](https://capnproto.org/) messages.
+rlogs contain all the messages passed amongst openpilot's processes. See [openpilot/cereal/services.py](https://github.com/commaai/openpilot/blob/master/openpilot/cereal/services.py) for a list of all the logged services. They're a zstd archive of the serialized [Cap’n Proto](https://capnproto.org/) messages.
 
 ## {f,e,d}camera.hevc
 
@@ -20,7 +20,7 @@ Each camera stream is H.265 encoded and written to its respective file.
 
 ## qlog.zst & qcamera.ts
 
-qlogs are a decimated subset of the rlogs. Check out [cereal/services.py](https://github.com/commaai/cereal/blob/master/services.py) for the decimation.
+qlogs are a decimated subset of the rlogs. Check out [openpilot/cereal/services.py](https://github.com/commaai/openpilot/blob/master/openpilot/cereal/services.py) for the decimation.
 
 qcameras are H.264 encoded, lower res versions of the fcamera.hevc. The video shown in [comma connect](https://connect.comma.ai/) is from the qcameras.
 

docs/how-to/car-port.md

@@ -30,7 +30,7 @@ Each car brand is supported by a standard interface structure in `opendbc/car/[b
 
 For historical reasons, openpilot still contains a small amount of car-specific logic. This will eventually be migrated to opendbc or otherwise removed.
 
-* `selfdrive/car/car_specific.py`: Brand-specific event logic
+* `openpilot/selfdrive/car/car_specific.py`: Brand-specific event logic
 
 # How do I port car?
 

docs/how-to/replay-a-drive.md

@@ -5,7 +5,7 @@ Replaying is a critical tool for openpilot development and debugging.
 ## Replaying a route
 *Hardware required: none*
 
-Just run `tools/replay/replay --demo`.
+Just run `openpilot/tools/replay/replay --demo`.
 
 ## Replaying CAN data
 *Hardware required: jungle and comma four*

docs/how-to/turn-the-speed-blue.md

@@ -28,10 +28,10 @@ scons
 We'll run the `replay` tool with the demo route to get data streaming for testing our UI changes.
 ```bash
 # in terminal 1
-tools/replay/replay --demo
+openpilot/tools/replay/replay --demo
 
 # in terminal 2
-./selfdrive/ui/ui.py
+./openpilot/selfdrive/ui/ui.py
 ```
 
 The openpilot UI should launch and show a replay of the demo route.
@@ -45,10 +45,10 @@ Now let’s update the speed display color in the UI.
 
 Search for the function responsible for rendering the current speed:
 ```bash
-git grep "_draw_current_speed" selfdrive/ui/onroad/hud_renderer.py
+git grep "_draw_current_speed" openpilot/selfdrive/ui/onroad/hud_renderer.py
 ```
 
-You'll find the relevant code inside `selfdrive/ui/onroad/hud_renderer.py`, in this function:
+You'll find the relevant code inside `openpilot/selfdrive/ui/onroad/hud_renderer.py`, in this function:
 
 ```python
 def _draw_current_speed(self, rect: rl.Rectangle) -> None:
@@ -72,7 +72,7 @@ Change `COLORS.white` to make it **blue** instead of white. A nice soft blue is
 
 After making changes, re-run the UI to see your new UI:
 ```bash
-./selfdrive/ui/ui.py
+./openpilot/selfdrive/ui/ui.py
 ```
 ![](https://blog.comma.ai/img/blue_speed_ui.png)
 

launch_chffrplus.sh

@@ -19,12 +19,12 @@ function agnos_init {
 
   # Check if AGNOS update is required
   if [ $(< /VERSION) != "$AGNOS_VERSION" ]; then
-    AGNOS_PY="$DIR/system/hardware/tici/agnos.py"
+    AGNOS_PY="$DIR/openpilot/common/hardware/tici/agnos.py"
-    MANIFEST="$DIR/system/hardware/tici/agnos.json"
+    MANIFEST="$DIR/openpilot/system/hardware/tici/agnos.json"
     if $AGNOS_PY --verify $MANIFEST; then
       sudo reboot
     fi
-    $DIR/system/hardware/tici/updater $AGNOS_PY $MANIFEST
+    $DIR/openpilot/common/hardware/tici/updater $AGNOS_PY $MANIFEST
   fi
 }
 
@@ -70,6 +70,14 @@ function launch {
   ln -sfn $(pwd) /data/pythonpath
   export PYTHONPATH="$PWD"
 
+  # submodule package symlinks for PYTHONPATH imports on device.
+  # on PC these come from editable installs via pyproject.toml / uv.
+  ln -sfn msgq_repo/msgq msgq
+  ln -sfn opendbc_repo/opendbc opendbc
+  ln -sfn rednose_repo/rednose rednose
+  ln -sfn teleoprtc_repo/teleoprtc teleoprtc
+  ln -sfn tinygrad_repo/tinygrad tinygrad
+
   # hardware specific init
   if [ -f /AGNOS ]; then
     agnos_init
@@ -79,7 +87,7 @@ function launch {
   tmux capture-pane -pq -S-1000 > /tmp/launch_log
 
   # start manager
-  cd system/manager
+  cd openpilot/system/manager
   if [ ! -f $DIR/prebuilt ]; then
     ./build.py
   fi

msgq

@@ -1 +0,0 @@
-msgq_repo/msgq

opendbc

@@ -1 +0,0 @@
-opendbc_repo/opendbc

openpilot/cereal/README.md

@@ -24,17 +24,17 @@ things are not. Read more details [here](https://capnproto.org/language.html).
 ### Custom forks
 
 Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
-spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
+spec, however this could conflict with future changes made in mainline openpilot's cereal spec. Rebasing against mainline openpilot
 then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
-[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
+[custom.capnp](custom.capnp) that we will leave empty in mainline openpilot's cereal spec. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
 An example of compatible changes:
 ```diff
-diff --git a/cereal/custom.capnp b/cereal/custom.capnp
+diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp
 index 3348e859e..3365c7b98 100644
---- a/cereal/custom.capnp
+--- a/openpilot/cereal/custom.capnp
-+++ b/cereal/custom.capnp
++++ b/openpilot/cereal/custom.capnp
 @@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
  # DO rename the structs
  # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
@@ -48,10 +48,10 @@ index 3348e859e..3365c7b98 100644
  }
 
  struct CustomReserved1 @0xaedffd8f31e7b55d {
-diff --git a/cereal/log.capnp b/cereal/log.capnp
+diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp
 index 1209f3fd9..b189f58b6 100644
---- a/cereal/log.capnp
+--- a/openpilot/cereal/log.capnp
-+++ b/cereal/log.capnp
++++ b/openpilot/cereal/log.capnp
 @@ -2558,14 +2558,14 @@ struct Event {
 
      # DO change the name of the field
@@ -76,7 +76,7 @@ index 1209f3fd9..b189f58b6 100644
 Example
 ---
 ```python
-import cereal.messaging as messaging
+import openpilot.cereal.messaging as messaging
 
 # in subscriber
 sm = messaging.SubMaster(['sensorEvents'])

openpilot/cereal/SConscript

@@ -0,0 +1,22 @@
+Import('env', 'common', 'msgq')
+
+cereal_dir = Dir('.')
+gen_dir = Dir('gen')
+
+# Build cereal
+schema_files = ['log.capnp', 'deprecated.capnp', 'custom.capnp']
+car_capnp = '#opendbc_repo/opendbc/car/car.capnp'
+all_output = schema_files + ['car.capnp']
+env.Command([f'gen/cpp/{s}.c++' for s in all_output] + [f'gen/cpp/{s}.h' for s in all_output],
+            schema_files + [car_capnp],
+            f"capnpc --src-prefix={cereal_dir.path} --src-prefix=opendbc_repo/opendbc/car --import-path=opendbc_repo/opendbc/car $SOURCES -o c++:{gen_dir.path}/cpp/")
+
+cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in all_output])
+
+# Build messaging
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+
+socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
+
+Export('cereal', 'socketmaster')

openpilot/cereal/__init__.py

@@ -0,0 +1,11 @@
+import os
+import capnp
+from importlib.resources import as_file, files
+
+capnp.remove_import_hook()
+
+with as_file(files("openpilot.cereal")) as fspath, as_file(files("opendbc")) as opendbc_path:
+  CEREAL_PATH = fspath.as_posix()
+  opendbc_import_path = os.path.join(os.path.realpath(opendbc_path.as_posix()), 'car')
+  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"), imports=[opendbc_import_path])
+  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"), imports=[opendbc_import_path])

openpilot/cereal/custom.capnp

@@ -1,4 +1,4 @@
-using Cxx = import "./include/c++.capnp";
+using Cxx = import "/include/c++.capnp";
 $Cxx.namespace("cereal");
 
 @0xb526ba661d550a59;

openpilot/cereal/deprecated.capnp

@@ -1,4 +1,4 @@
-using Cxx = import "./include/c++.capnp";
+using Cxx = import "/include/c++.capnp";
 $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
@@ -760,18 +760,6 @@ struct LateralLQRState @0x9024e2d790c82ade {
   steeringAngleDesiredDeg @6 :Float32;
 }
 
-struct LateralCurvatureState @0xad9d8095c06f7c61 {
-  active @0 :Bool;
-  actualCurvature @1 :Float32;
-  desiredCurvature @2 :Float32;
-  error @3 :Float32;
-  p @4 :Float32;
-  i @5 :Float32;
-  f @6 :Float32;
-  output @7 :Float32;
-  saturated @8 :Bool;
-}
-
 struct LateralPlannerSolution @0x84caeca5a6b4acfe {
   x @0 :List(Float32);
   y @1 :List(Float32);

openpilot/cereal/log.capnp

@@ -1,7 +1,7 @@
-using Cxx = import "./include/c++.capnp";
+using Cxx = import "/include/c++.capnp";
 $Cxx.namespace("cereal");
 
-using Car = import "car.capnp";
+using Car = import "/car.capnp";
 using Deprecated = import "deprecated.capnp";
 using Custom = import "custom.capnp";
 
@@ -414,6 +414,10 @@ struct CanData {
 struct DeviceState @0xa4d8b5af2aa492eb {
   deviceType @45 :InitData.DeviceType;
 
+  # usb
+  chestnutPresent @51 :Bool;
+  usbState @52 :UsbState;
+
   networkType @22 :NetworkType;
   networkInfo @31 :NetworkInfo;
   networkStrength @24 :NetworkStrength;
@@ -684,33 +688,47 @@ struct PeripheralState {
   }
 }
 
+struct UsbState {
+  devices @0 :List(Device);
+
+  struct Device {
+    busnum @0 :UInt8;
+    devnum @1 :UInt8;
+    vendorId @2 :UInt16;
+    productId @3 :UInt16;
+    speedMbps @4 :UInt16;
+    manufacturer @6 :Text;
+    product @5 :Text;
+    linkErrorCount @7 :UInt16;
+  }
+}
+
 struct RadarState @0x9a185389d6fdd05f {
-  mdMonoTime @6 :UInt64;
+  mdMonoTime @6 :UInt64;  # for debugging
-  carStateMonoTime @11 :UInt64;
   radarErrors @13 :Car.RadarData.Error;
 
   leadOne @3 :LeadData;
   leadTwo @4 :LeadData;
 
   struct LeadData {
-    dRel @0 :Float32;
+    dRel @0 :Float32;  # m from the front bumper of the car
-    yRel @1 :Float32;
+    yRel @1 :Float32;  # m in car frame, left positive
-    vRel @2 :Float32;
+    vRel @2 :Float32;  # m/s relative longitudinal speed
-    aRel @3 :Float32;
+    vLead @4 :Float32;  # m/s absolute lead speed
-    vLead @4 :Float32;
+    vLeadK @8 :Float32;  # kalman-filtered lead speed
-    dPath @6 :Float32;
+    aLeadK @9 :Float32;  # kalman-filtered lead accel
-    vLat @7 :Float32;
+    present @11 :Bool;  # true if a lead is present
-    vLeadK @8 :Float32;
+    aLeadTau @12 :Float32;  # lead accel time constant
-    aLeadK @9 :Float32;
+    modelProb @13 :Float32;  # vision model lead probability
-    fcw @10 :Bool;
+    radar @14 :Bool;  # true if lead is radar-matched (vs vision-only)
-    status @11 :Bool;
+    radarTrackId @15 :Int32 = -1;  # for debugging
-    aLeadTau @12 :Float32;
-    modelProb @13 :Float32;
-    radar @14 :Bool;
-    radarTrackId @15 :Int32 = -1;
 
     deprecated :group {
+      aRel @3 :Float32;
       aLead @5 :Float32;
+      dPath @6 :Float32;
+      vLat @7 :Float32;
+      fcw @10 :Bool;
     }
   }
 
@@ -723,6 +741,7 @@ struct RadarState @0x9a185389d6fdd05f {
     calPerc @9 :Int8;
     canMonoTimes @10 :List(UInt64);
     cumLagMs @5 :Float32;
+    carStateMonoTime @11 :UInt64;
     radarErrors @12 :List(Car.RadarData.ErrorDEPRECATED);
   }
 }
@@ -771,13 +790,30 @@ struct SelfdriveState {
   alertStatus @5 :AlertStatus;
   alertSize @6 :AlertSize;
   alertType @7 :Text;
-  alertSound @8 :Car.CarControl.HUDControl.AudibleAlert;
+  alertSound @13 :AudibleAlert;
   alertHudVisual @12 :Car.CarControl.HUDControl.VisualAlert;
 
   # configurable driving settings
   experimentalMode @10 :Bool;
   personality @11 :LongitudinalPersonality;
 
+  enum AudibleAlert {
+    none @0;
+
+    engage @1;
+    disengage @2;
+    refuse @3;
+
+    warningSoft @4;
+    warningImmediate @5;
+
+    prompt @6;
+    promptRepeat @7;
+    promptDistracted @8;
+
+    preAlert @9;
+  }
+
   enum OpenpilotState @0xdbe58b96d2d1ac61 {
     disabled @0;
     preEnabled @1;
@@ -798,6 +834,10 @@ struct SelfdriveState {
     mid @2;
     full @3;
   }
+
+  deprecated :group {
+    alertSound @8 :Car.CarControl.HUDControl.AudibleAlert;
+  }
 }
 
 struct ControlsState @0x97ff69c53601abf1 {
@@ -818,7 +858,7 @@ struct ControlsState @0x97ff69c53601abf1 {
     debugState @59 :LateralDebugState;
     torqueState @60 :LateralTorqueState;
 
-    curvatureStateDEPRECATED @65 :Deprecated.LateralCurvatureState;
+    curvatureState @65 :LateralCurvatureState;
     lqrStateDEPRECATED @55 :Deprecated.LateralLQRState;
     indiStateDEPRECATED @52 :Deprecated.LateralINDIState;
   }
@@ -867,6 +907,18 @@ struct ControlsState @0x97ff69c53601abf1 {
     saturated @3 :Bool;
   }
 
+  struct LateralCurvatureState @0xad9d8095c06f7c61 {
+    active @0 :Bool;
+    actualCurvature @1 :Float32;
+    desiredCurvature @2 :Float32;
+    error @3 :Float32;
+    p @4 :Float32;
+    i @5 :Float32;
+    f @6 :Float32;
+    output @7 :Float32;
+    saturated @8 :Bool;
+  }
+
   deprecated :group {
     vEgo @0 :Float32;
     vEgoRaw @32 :Float32;
@@ -906,7 +958,7 @@ struct ControlsState @0x97ff69c53601abf1 {
     alertStatus @38 :SelfdriveState.AlertStatus;
     alertSize @39 :SelfdriveState.AlertSize;
     alertType @44 :Text;
-    alertSound2 @56 :Car.CarControl.HUDControl.AudibleAlert;
+    alertSound2 @56 :SelfdriveState.AudibleAlert;
     engageable @41 :Bool;  # can OP be engaged?
     state @31 :SelfdriveState.OpenpilotState;
     enabled @19 :Bool;
@@ -989,7 +1041,6 @@ struct ModelDataV2 {
   roadEdgeStds @14 :List(Float32);
 
   # predicted lead cars
-  leads @11 :List(LeadDataV2);
   leadsV3 @18 :List(LeadDataV3);
 
   meta @12 :MetaData;
@@ -999,8 +1050,9 @@ struct ModelDataV2 {
   action @26: Action;
 
   lateralPlannerSolutionDEPRECATED @25: Deprecated.LateralPlannerSolution;
+  leadsDEPRECATED @11 :List(LeadDataV2DEPRECATED);
 
-  struct LeadDataV2 {
+  struct LeadDataV2DEPRECATED {
     prob @0 :Float32; # probability that car is your lead at time t
     t @1 :Float32;
 
@@ -1120,7 +1172,7 @@ struct EncodeIndex {
   }
 }
 
-struct AndroidLogEntry {
+struct OperatingSystemLogEntry {
   id @0 :UInt8;
   ts @1 :UInt64;
   priority @2 :UInt8;
@@ -1258,7 +1310,7 @@ struct LateralPlan @0xe1e9318e2ae8b51e {
 struct LiveLocationKalman {
 
   # More info on reference frames:
-  # https://github.com/commaai/openpilot/tree/master/common/transformations
+  # https://github.com/commaai/openpilot/tree/master/openpilot/common/transformations
 
   positionECEF @0 : Measurement;
   positionGeodetic @1 : Measurement;
@@ -1315,7 +1367,7 @@ struct LiveLocationKalman {
 
 struct LivePose {
   # More info on reference frames:
-  # https://github.com/commaai/openpilot/tree/master/common/transformations
+  # https://github.com/commaai/openpilot/tree/master/openpilot/common/transformations
   orientationNED @0 :XYZMeasurement;
   velocityDevice @1 :XYZMeasurement;
   accelerationDevice @2 :XYZMeasurement;
@@ -2108,8 +2160,10 @@ struct DriverMonitoringStateDEPRECATED @0xb83cda094a1da284 {
 
 struct DriverMonitoringState {
   lockout @0 :Bool;
-  alertCountLockoutPercent @1 :Int8;
+  lockoutRecoveryPercent @11 :Int8;
-  alertTimeLockoutPercent @2 :Int8;
+  alert3Count @12 :Int8;
+  noResponseCount @13 :Int8;
+  noResponseForceDecel @14 :Bool;
 
   alwaysOn @3 :Bool;
   alwaysOnLockout @4 :Bool;
@@ -2173,6 +2227,11 @@ struct DriverMonitoringState {
     calibratedPercent @0 :Int8;
     offset @1 :Float32;
   }
+
+  deprecated :group {
+    alertCountLockoutPercent @1 :Int8;
+    alertTimeLockoutPercent @2 :Int8;
+  }
 }
 
 struct Boot {
@@ -2247,6 +2306,7 @@ struct LiveDelayData {
   lateralDelayEstimateStd @5 :Float32;
   points @4 :List(Float32);
   calPerc @6 :Int8;
+  version @7 :Int32;
 
   enum Status {
     unestimated @0;
@@ -2504,7 +2564,7 @@ struct Event {
     rawAudioData @147 :AudioData;
 
     # systems stuff
-    androidLog @20 :AndroidLogEntry;
+    operatingSystemLog @20 :OperatingSystemLogEntry;
     managerState @78 :ManagerState;
     procLog @33 :ProcLog;
     clocks @35 :Clocks;

openpilot/cereal/messaging/__init__.py

@@ -9,8 +9,8 @@ import time
 
 from typing import Optional, List, Union, Dict
 
-from cereal import log
+from openpilot.cereal import log
-from cereal.services import SERVICE_LIST
+from openpilot.cereal.services import SERVICE_LIST
 from openpilot.common.utils import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1

openpilot/cereal/messaging/bridge.cc

@@ -1,7 +1,7 @@
 #include <cassert>
 
-#include "cereal/messaging/msgq_to_zmq.h"
+#include "openpilot/cereal/messaging/msgq_to_zmq.h"
-#include "cereal/services.h"
+#include "openpilot/cereal/services.h"
 #include "common/util.h"
 
 ExitHandler do_exit;

openpilot/cereal/messaging/bridge_zmq.cc

@@ -1,4 +1,4 @@
-#include "cereal/messaging/bridge_zmq.h"
+#include "openpilot/cereal/messaging/bridge_zmq.h"
 
 #include <cassert>
 #include <cstring>

openpilot/cereal/messaging/messaging.h

@@ -8,7 +8,7 @@
 
 #include <capnp/serialize.h>
 
-#include "cereal/gen/cpp/log.capnp.h"
+#include "openpilot/cereal/gen/cpp/log.capnp.h"
 #include "common/timing.h"
 #include "msgq/ipc.h"
 

openpilot/cereal/messaging/msgq_to_zmq.cc

@@ -1,8 +1,8 @@
-#include "cereal/messaging/msgq_to_zmq.h"
+#include "openpilot/cereal/messaging/msgq_to_zmq.h"
 
 #include <cassert>
 
-#include "cereal/services.h"
+#include "openpilot/cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;

openpilot/cereal/messaging/msgq_to_zmq.h

@@ -8,7 +8,7 @@
 #include <vector>
 
 #include "msgq/impl_msgq.h"
-#include "cereal/messaging/bridge_zmq.h"
+#include "openpilot/cereal/messaging/bridge_zmq.h"
 
 class MsgqToZmq {
 public:

openpilot/cereal/messaging/socketmaster.cc

@@ -3,8 +3,8 @@
 #include <string>
 #include <mutex>
 
-#include "cereal/services.h"
+#include "openpilot/cereal/services.h"
-#include "cereal/messaging/messaging.h"
+#include "openpilot/cereal/messaging/messaging.h"
 
 const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
 

openpilot/cereal/messaging/tests/test_messaging.py

@@ -8,9 +8,10 @@ import time
 from openpilot.common.parameterized import parameterized
 import pytest
 
-from cereal import log, car
+from openpilot.cereal import log
-import cereal.messaging as messaging
+from opendbc.car.structs import car
-from cereal.services import SERVICE_LIST
+import openpilot.cereal.messaging as messaging
+from openpilot.cereal.services import SERVICE_LIST
 
 events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
 

openpilot/cereal/messaging/tests/test_pub_sub_master.py

@@ -2,11 +2,11 @@ import random
 import time
 from typing import Sized, cast
 
-import cereal.messaging as messaging
+import openpilot.cereal.messaging as messaging
-from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
+from openpilot.cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
                                                   random_bytes, random_carstate, assert_carstate, \
                                                   zmq_sleep
-from cereal.services import SERVICE_LIST
+from openpilot.cereal.services import SERVICE_LIST
 
 
 class TestSubMaster:

openpilot/cereal/messaging/tests/test_services.py

@@ -1,10 +1,10 @@
-import os
+import subprocess
 import tempfile
-from typing import Dict
+
 from openpilot.common.parameterized import parameterized
 
-import cereal.services as services
+import openpilot.cereal.services as services
-from cereal.services import SERVICE_LIST
+from openpilot.cereal.services import SERVICE_LIST
 
 
 class TestServices:
@@ -17,5 +17,5 @@ class TestServices:
 
   def test_generated_header(self):
     with tempfile.NamedTemporaryFile(suffix=".h") as f:
-      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
+      ret = subprocess.run(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11", shell=True).returncode
       assert ret == 0, "generated services header is not valid C"

openpilot/cereal/services.py

@@ -41,7 +41,7 @@ _services: dict[str, tuple] = {
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
   "liveDelay": (True, 4., 1),
-  "androidLog": (True, 0.),
+  "operatingSystemLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),

openpilot/common

@@ -1 +0,0 @@
-../common

openpilot/common/SConscript

@@ -5,6 +5,7 @@ common_libs = [
   'swaglog.cc',
   'util.cc',
   'ratekeeper.cc',
+  'yuv.cc',
 ]
 
 _common = env.Library('common', common_libs, LIBS="json11")
@@ -12,7 +13,7 @@ Export('_common')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
-              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              ['tests/test_runner.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
               LIBS=[_common, 'json11', 'zmq', 'pthread'])
 
 # Cython bindings

openpilot/common/api/base.py

@@ -3,8 +3,8 @@ import os
 import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
-from openpilot.system.hardware.hw import Paths
+from openpilot.common.hardware.hw import Paths
-from openpilot.system.version import get_version
+from openpilot.common.version import get_version
 
 # name: jwt signature algorithm
 KEYS = {"id_rsa": "RS256",

openpilot/common/basedir.py

@@ -1,4 +1,4 @@
 import os
 
 
-BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))

openpilot/common/esim/__init__.py

@@ -0,0 +1,3 @@
+from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+
+__all__ = ["LPABase", "LPAError", "LPAProfileNotFoundError", "Profile"]

openpilot/common/esim/base.py

@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+class LPAError(RuntimeError):
+  pass
+
+
+class LPAProfileNotFoundError(LPAError):
+  pass
+
+
+@dataclass
+class Profile:
+  iccid: str
+  nickname: str
+  enabled: bool
+  provider: str
+
+  @property
+  def is_comma(self) -> bool:
+    return self.provider == 'Webbing' and self.iccid.startswith('8985235')
+
+
+class LPABase(ABC):
+  @abstractmethod
+  def list_profiles(self) -> list[Profile]:
+    pass
+
+  @abstractmethod
+  def get_active_profile(self) -> Profile | None:
+    pass
+
+  @abstractmethod
+  def delete_profile(self, iccid: str) -> None:
+    pass
+
+  @abstractmethod
+  def download_profile(self, qr: str, nickname: str | None = None) -> None:
+    pass
+
+  @abstractmethod
+  def nickname_profile(self, iccid: str, nickname: str) -> None:
+    pass
+
+  @abstractmethod
+  def switch_profile(self, iccid: str) -> None:
+    pass
+
+  def process_notifications(self) -> None:
+    pass
+
+  @abstractmethod
+  def is_euicc(self) -> bool:
+    pass

openpilot/common/esim/esim.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
 import argparse
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
-from openpilot.system.hardware.base import LPABase, Profile
+from openpilot.common.esim.base import LPABase, Profile
 
 
 def sorted_profiles(lpa: LPABase) -> list[Profile]:

openpilot/common/esim/lpa.py

@@ -6,7 +6,6 @@ import fcntl
 import hashlib
 import os
 import requests
-import serial
 import subprocess
 import sys
 import termios
@@ -19,7 +18,8 @@ from typing import Any
 from pathlib import Path
 
 from openpilot.common.time_helpers import system_time_valid
-from openpilot.system.hardware.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+from openpilot.common.serial import Serial, SerialException
 
 GSMA_CI_BUNDLE = str(Path(__file__).parent / "gsma_ci_bundle.pem")
 
@@ -133,7 +133,7 @@ class AtClient:
     self._device = device
     self._baud = baud
     self._timeout = timeout
-    self._serial: serial.Serial | None = None
+    self._serial: Serial | None = None
 
   def send_raw(self, data: bytes) -> None:
     self._ensure_serial()
@@ -185,14 +185,14 @@ class AtClient:
         pass
       self._serial = None
     if self._serial is None:
-      self._serial = serial.Serial(self._device, baudrate=self._baud, timeout=self._timeout)
+      self._serial = Serial(self._device, baudrate=self._baud, timeout=self._timeout)
 
   def query(self, cmd: str) -> list[str]:
     self._ensure_serial()
     try:
       self._send(cmd)
       return self._expect()
-    except serial.SerialException:
+    except SerialException:
       self._ensure_serial(reconnect=True)
       self._send(cmd)
       return self._expect()
@@ -208,7 +208,7 @@ class AtClient:
     if self._serial:
       try:
         self._serial.reset_input_buffer()
-      except (OSError, serial.SerialException, termios.error):
+      except (OSError, SerialException, termios.error):
         self._ensure_serial(reconnect=True)
     for line in self.query(f'AT+CCHO="{ISDR_AID}"'):
       if line.startswith("+CCHO:") and (ch := line.split(":", 1)[1].strip()):
@@ -230,7 +230,7 @@ class AtClient:
       try:
         self._open_isdr_once()
         return
-      except (RuntimeError, TimeoutError, termios.error, serial.SerialException):
+      except (RuntimeError, TimeoutError, termios.error, SerialException):
         time.sleep(OPEN_ISDR_RETRY_DELAY_S)
         if attempt == OPEN_ISDR_RESET_ATTEMPT:
           self._reset_modem()

openpilot/common/file_chunker.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import io
 import sys
 import math
 import os
@@ -21,13 +22,12 @@ def get_chunk_targets(path, file_size):
 
 def chunk_file(path, targets):
   manifest_path, *chunk_paths = targets
-  with open(path, 'rb') as f:
+  actual_num_chunks = max(1, math.ceil(os.path.getsize(path) / CHUNK_SIZE))
-    data = f.read()
-  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
   assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
-  for i, chunk_path in enumerate(chunk_paths):
+  with open(path, 'rb') as f:
-    with open(chunk_path, 'wb') as f:
+    for chunk_path in chunk_paths:
-      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
+      with open(chunk_path, 'wb') as out:
+        out.write(f.read(CHUNK_SIZE))
   Path(manifest_path).write_text(str(len(chunk_paths)))
   os.remove(path)
 
@@ -39,14 +39,40 @@ def get_existing_chunks(path):
     return _chunk_paths(path, num_chunks)
   raise FileNotFoundError(path)
 
-def read_file_chunked(path):
+class ChunkStream(io.RawIOBase):
+  def __init__(self, paths):
+    self._paths = iter(paths)
+    self._buf = memoryview(b'')
+
+  def readable(self):
+    return True
+
+  def readinto(self, b):
+    n = 0
+    while n < len(b):
+      if not self._buf:
+        p = next(self._paths, None)
+        if p is None:
+          break
+        with open(p, 'rb') as f:
+          self._buf = memoryview(f.read())
+        continue
+      take = min(len(b) - n, len(self._buf))
+      b[n:n + take] = self._buf[:take]
+      self._buf = self._buf[take:]
+      n += take
+    return n
+
+def open_file_chunked(path):
   manifest_path = get_manifest_path(path)
   if os.path.isfile(manifest_path):
     num_chunks = int(Path(manifest_path).read_text().strip())
-    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
+    paths = [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
-  if os.path.isfile(path):
+  elif os.path.isfile(path):
-    return Path(path).read_bytes()
+    paths = [path]
-  raise FileNotFoundError(path)
+  else:
+    raise FileNotFoundError(path)
+  return io.BufferedReader(ChunkStream(paths))
 
 
 if __name__ == "__main__":

openpilot/common/hardware/__init__.py

@@ -1,9 +1,9 @@
 import os
 from typing import cast
 
-from openpilot.system.hardware.base import HardwareBase
+from openpilot.common.hardware.base import HardwareBase
-from openpilot.system.hardware.tici.hardware import Tici
+from openpilot.common.hardware.tici.hardware import Tici
-from openpilot.system.hardware.pc.hardware import Pc
+from openpilot.common.hardware.pc.hardware import Pc
 
 TICI = os.path.isfile('/TICI')
 AGNOS = os.path.isfile('/AGNOS')

openpilot/common/hardware/base.h

@@ -5,15 +5,13 @@
 #include <map>
 #include <string>
 
-#include "cereal/gen/cpp/log.capnp.h"
+#include "openpilot/cereal/gen/cpp/log.capnp.h"
 
 // no-op base hw class
 class HardwareNone {
 public:
   static std::string get_name() { return ""; }
   static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; }
-  static int get_voltage() { return 0; }
-  static int get_current() { return 0; }
 
   static std::string get_serial() { return "cccccc"; }
 
@@ -24,6 +22,4 @@ public:
   static void set_ir_power(int percentage) {}
 
   static bool PC() { return false; }
-  static bool TICI() { return false; }
-  static bool AGNOS() { return false; }
 };

openpilot/common/hardware/base.py

@@ -2,28 +2,12 @@ import os
 from abc import abstractmethod, ABC
 from dataclasses import dataclass, fields
 
-from cereal import log
+from openpilot.cereal import log
+from openpilot.common.esim.base import LPABase
 
 NetworkType = log.DeviceState.NetworkType
 NetworkStrength = log.DeviceState.NetworkStrength
 
-class LPAError(RuntimeError):
-  pass
-
-class LPAProfileNotFoundError(LPAError):
-  pass
-
-@dataclass
-class Profile:
-  iccid: str
-  nickname: str
-  enabled: bool
-  provider: str
-
-  @property
-  def is_comma(self) -> bool:
-    return self.provider == 'Webbing' and self.iccid.startswith('8985235')
-
 @dataclass
 class ThermalZone:
   # a zone from /sys/class/thermal/thermal_zone*
@@ -70,38 +54,6 @@ class ThermalConfig:
           ret[f.name + "TempC"] = v.read()
     return ret
 
-class LPABase(ABC):
-  @abstractmethod
-  def list_profiles(self) -> list[Profile]:
-    pass
-
-  @abstractmethod
-  def get_active_profile(self) -> Profile | None:
-    pass
-
-  @abstractmethod
-  def delete_profile(self, iccid: str) -> None:
-    pass
-
-  @abstractmethod
-  def download_profile(self, qr: str, nickname: str | None = None) -> None:
-    pass
-
-  @abstractmethod
-  def nickname_profile(self, iccid: str, nickname: str) -> None:
-    pass
-
-  @abstractmethod
-  def switch_profile(self, iccid: str) -> None:
-    pass
-
-  def process_notifications(self) -> None:
-    pass
-
-  @abstractmethod
-  def is_euicc(self) -> bool:
-    pass
-
 class HardwareBase(ABC):
   @staticmethod
   def get_cmdline() -> dict[str, str]:
@@ -133,7 +85,7 @@ class HardwareBase(ABC):
   def get_device_type(self):
     pass
 
-  def get_imei(self, slot) -> str:
+  def get_imei(self) -> str:
     return ""
 
   def get_serial(self):
@@ -190,21 +142,12 @@ class HardwareBase(ABC):
   def get_gpu_usage_percent(self):
     return 0
 
-  def get_modem_version(self):
-    return None
-
   def get_modem_temperatures(self):
     return []
 
   def initialize_hardware(self):
     pass
 
-  def get_networks(self):
-    return None
-
-  def has_internal_panda(self) -> bool:
-    return False
-
   def reset_internal_panda(self):
     pass
 
@@ -214,11 +157,5 @@ class HardwareBase(ABC):
   def get_modem_data_usage(self):
     return -1, -1
 
-  def get_voltage(self) -> float:
-    return 0.
-
-  def get_current(self) -> float:
-    return 0.
-
   def set_ir_power(self, percent: int):
     pass

openpilot/common/hardware/hw.h

@@ -2,14 +2,14 @@
 
 #include <string>
 
-#include "system/hardware/base.h"
+#include "common/hardware/base.h"
 #include "common/util.h"
 
 #if __TICI__
-#include "system/hardware/tici/hardware.h"
+#include "common/hardware/tici/hardware.h"
 #define Hardware HardwareTici
 #else
-#include "system/hardware/pc/hardware.h"
+#include "common/hardware/pc/hardware.h"
 #define Hardware HardwarePC
 #endif
 

openpilot/common/hardware/hw.py

@@ -2,7 +2,7 @@ import os
 import platform
 from pathlib import Path
 
-from openpilot.system.hardware import PC
+from openpilot.common.hardware import PC
 
 DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache"
 

openpilot/common/hardware/pc/hardware.h

@@ -2,13 +2,11 @@
 
 #include <string>
 
-#include "system/hardware/base.h"
+#include "common/hardware/base.h"
 
 class HardwarePC : public HardwareNone {
 public:
   static std::string get_name() { return "pc"; }
   static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; }
   static bool PC() { return true; }
-  static bool TICI() { return util::getenv("TICI", 0) == 1; }
-  static bool AGNOS() { return util::getenv("TICI", 0) == 1; }
 };

openpilot/common/hardware/pc/hardware.py

@@ -0,0 +1,10 @@
+from openpilot.cereal import log
+from openpilot.common.hardware.base import HardwareBase
+
+class Pc(HardwareBase):
+  def get_device_type(self):
+    return "pc"
+
+  def get_network_type(self):
+    # some stuff is gated on wifi, so just assume for now
+    return log.DeviceState.NetworkType.wifi

openpilot/common/hardware/tici/__init__.py

@@ -0,0 +1 @@
+

openpilot/common/hardware/tici/agnos.py

@@ -10,12 +10,9 @@ from collections.abc import Generator
 
 import requests
 
-import openpilot.system.updated.casync.casync as casync
-
 SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
-CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/"
 
-AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json"
+AGNOS_MANIFEST_FILE = "openpilot/system/hardware/tici/agnos.json"
 
 
 class StreamingDecompressor:
@@ -187,50 +184,6 @@ def extract_compressed_image(target_slot_number: int, partition: dict, cloudlog)
     os.sync()
 
 
-def extract_casync_image(target_slot_number: int, partition: dict, cloudlog):
-  path = get_partition_path(target_slot_number, partition)
-  seed_path = path[:-1] + ('b' if path[-1] == 'a' else 'a')
-
-  target = casync.parse_caibx(partition['casync_caibx'])
-
-  sources: list[tuple[str, casync.ChunkReader, casync.ChunkDict]] = []
-
-  # First source is the current partition.
-  try:
-    raw_hash = get_raw_hash(seed_path, partition['size'])
-    caibx_url = f"{CAIBX_URL}{partition['name']}-{raw_hash}.caibx"
-
-    try:
-      cloudlog.info(f"casync fetching {caibx_url}")
-      sources += [('seed', casync.FileChunkReader(seed_path), casync.build_chunk_dict(casync.parse_caibx(caibx_url)))]
-    except requests.RequestException:
-      cloudlog.error(f"casync failed to load {caibx_url}")
-  except Exception:
-    cloudlog.exception("casync failed to hash seed partition")
-
-  # Second source is the target partition, this allows for resuming
-  sources += [('target', casync.FileChunkReader(path), casync.build_chunk_dict(target))]
-
-  # Finally we add the remote source to download any missing chunks
-  sources += [('remote', casync.RemoteChunkReader(partition['casync_store']), casync.build_chunk_dict(target))]
-
-  last_p = 0
-
-  def progress(cur):
-    nonlocal last_p
-    p = int(cur / partition['size'] * 100)
-    if p != last_p:
-      last_p = p
-      print(f"Installing {partition['name']}: {p}", flush=True)
-
-  stats = casync.extract(target, sources, path, progress)
-  cloudlog.error(f'casync done {json.dumps(stats)}')
-
-  os.sync()
-  if not verify_partition(target_slot_number, partition, force_full_check=True):
-    raise Exception(f"Raw hash mismatch '{partition['hash_raw'].lower()}'")
-
-
 def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalone=False):
   cloudlog.info(f"Downloading and writing {partition['name']}")
 
@@ -245,10 +198,7 @@ def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalo
 
   path = get_partition_path(target_slot_number, partition)
 
-  if ('casync_caibx' in partition) and not standalone:
+  extract_compressed_image(target_slot_number, partition, cloudlog)
-    extract_casync_image(target_slot_number, partition, cloudlog)
-  else:
-    extract_compressed_image(target_slot_number, partition, cloudlog)
 
   # Write hash after successful flash
   if not full_check:
@@ -278,7 +228,7 @@ def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, st
   cloudlog.info(f"Target slot {target_slot_number}")
 
   # set target slot as unbootable
-  os.system(f"abctl --set_unbootable {target_slot_number}")
+  subprocess.run(f"abctl --set_unbootable {target_slot_number}", shell=True)
 
   for partition in update:
     success = False

openpilot/common/hardware/tici/hardware.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <cstdlib>
 #include <cassert>
 #include <fstream>
 #include <map>
@@ -8,7 +7,7 @@
 #include <algorithm>  // for std::clamp
 
 #include "common/util.h"
-#include "system/hardware/base.h"
+#include "common/hardware/base.h"
 
 class HardwareTici : public HardwareNone {
 public:
@@ -22,7 +21,6 @@ public:
 
   static cereal::InitData::DeviceType get_device_type() {
     static const std::map<std::string, cereal::InitData::DeviceType> device_map = {
-      {"tici", cereal::InitData::DeviceType::TICI},
       {"tizi", cereal::InitData::DeviceType::TIZI},
       {"mici", cereal::InitData::DeviceType::MICI}
     };
@@ -31,9 +29,6 @@ public:
     return it->second;
   }
 
-  static int get_voltage() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/in1_input").c_str()); }
-  static int get_current() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/curr1_input").c_str()); }
-
   static std::string get_serial() {
     static std::string serial("");
     if (serial.empty()) {
@@ -54,8 +49,7 @@ public:
 
   static void set_ir_power(int percent) {
     auto device = get_device_type();
-    if (device == cereal::InitData::DeviceType::TICI ||
+    if (device == cereal::InitData::DeviceType::TIZI) {
-        device == cereal::InitData::DeviceType::TIZI) {
       return;
     }
 

openpilot/common/hardware/tici/hardware.py

@@ -7,14 +7,14 @@ import time
 from functools import cached_property, lru_cache
 from pathlib import Path
 
-from cereal import log
+from openpilot.cereal import log
 from openpilot.common.utils import sudo_read, sudo_write
 from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
-from openpilot.system.hardware.base import HardwareBase, LPABase, ThermalConfig, ThermalZone
+from openpilot.common.esim.base import LPABase
-from openpilot.system.hardware.tici import iwlist
+from openpilot.common.hardware.base import HardwareBase, ThermalConfig, ThermalZone
-from openpilot.system.hardware.tici.lpa import TiciLPA
+from openpilot.common.esim.lpa import TiciLPA
-from openpilot.system.hardware.tici.pins import GPIO
+from openpilot.common.hardware.tici.pins import GPIO
-from openpilot.system.hardware.tici.amplifier import Amplifier
+from openpilot.common.hardware.tici.amplifier import Amplifier
 
 MODEM_STATE_PATH = "/dev/shm/modem"
 
@@ -147,9 +147,7 @@ class Tici(HardwareBase):
   def get_sim_lpa(self) -> LPABase:
     return TiciLPA()
 
-  def get_imei(self, slot):
+  def get_imei(self):
-    if slot != 0:
-      return ""
     return self.get_modem_state().get('imei', '')
 
   def get_network_info(self):
@@ -233,9 +231,6 @@ class Tici(HardwareBase):
 
     return super().get_network_metered(network_type)
 
-  def get_modem_version(self):
-    return self.get_modem_state().get('modem_version') or None
-
   def get_modem_temperatures(self):
     return self.get_modem_state().get('temperatures', [])
 
@@ -246,7 +241,7 @@ class Tici(HardwareBase):
     return (self.read_param_file("/sys/class/power_supply/bms/voltage_now", int) * self.read_param_file("/sys/class/power_supply/bms/current_now", int) / 1e12)
 
   def shutdown(self):
-    os.system("sudo poweroff")
+    subprocess.run("sudo poweroff", shell=True)
 
   def get_thermal_config(self):
     intake, exhaust, gnss, bottomSoc = None, None, None, None
@@ -313,6 +308,9 @@ class Tici(HardwareBase):
         continue
       gov = 'ondemand' if powersave_enabled else 'performance'
       sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
+      if not powersave_enabled:
+        # cap max core freq to 1689 Mhz
+        sudo_write('1689600', f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_max_freq')
 
     # *** IRQ config ***
 
@@ -337,7 +335,7 @@ class Tici(HardwareBase):
       self.amplifier.initialize_configuration()
 
     # Allow hardwared to write engagement status to kmsg
-    os.system("sudo chmod a+w /dev/kmsg")
+    subprocess.run("sudo chmod a+w /dev/kmsg", shell=True)
 
     # Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
     gpio_init(GPIO.SOM_ST_IO, True)
@@ -380,43 +378,13 @@ class Tici(HardwareBase):
       pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
       subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
       subprocess.call(["sudo", "taskset", "-pc", "3", pid])
-    except subprocess.CalledProcessException as e:
+    except subprocess.CalledProcessError as e:
       print(str(e))
 
-  def get_networks(self):
-    r = {}
-
-    wlan = iwlist.scan()
-    if wlan is not None:
-      r['wlan'] = wlan
-
-    lte_info = self.get_network_info()
-    if lte_info is not None:
-      extra = lte_info['extra']
-
-      # <state>,"LTE",<is_tdd>,<mcc>,<mnc>,<cellid>,<pcid>,<earfcn>,<freq_band_ind>,
-      # <ul_bandwidth>,<dl_bandwidth>,<tac>,<rsrp>,<rsrq>,<rssi>,<sinr>,<srxlev>
-      if 'LTE' in extra:
-        extra = extra.split(',')
-        try:
-          r['lte'] = [{
-            "mcc": int(extra[3]),
-            "mnc": int(extra[4]),
-            "cid": int(extra[5], 16),
-            "nmr": [{"pci": int(extra[6]), "earfcn": int(extra[7])}],
-          }]
-        except (ValueError, IndexError):
-          pass
-
-    return r
-
   def get_modem_data_usage(self):
     ms = self.get_modem_state()
     return ms.get('tx_bytes', -1), ms.get('rx_bytes', -1)
 
-  def has_internal_panda(self):
-    return True
-
   def reset_internal_panda(self):
     gpio_init(GPIO.STM_RST_N, True)
     gpio_init(GPIO.STM_BOOT0, True)

openpilot/common/hardware/tici/modem.py

@@ -3,7 +3,6 @@ import fcntl
 import json
 import logging
 import os
-import serial
 import signal
 import subprocess
 import tempfile
@@ -13,6 +12,8 @@ from ipaddress import IPv4Address, AddressValueError
 
 from enum import Enum
 
+from openpilot.common.serial import Serial
+
 logging.basicConfig(
   level=logging.INFO,
   format="%(asctime)s.%(msecs)03d %(levelname)-7s modem: %(message)s",
@@ -96,7 +97,7 @@ class PPPSession:
   def reset_data_port():
     """Drop DTR on PPP_PORT so the modem terminates any stuck PPP session."""
     try:
-      with serial.Serial(PPP_PORT, 460800, timeout=1) as s:
+      with Serial(PPP_PORT, baudrate=460800, timeout=1) as s:
         s.dtr = False
         time.sleep(0.2)
         s.dtr = True
@@ -220,7 +221,7 @@ class Modem:
       os.close(fd)
       return []
     try:
-      with serial.Serial(AT_PORT, 9600, timeout=5) as ser:
+      with Serial(AT_PORT, baudrate=9600, timeout=5) as ser:
         ser.reset_input_buffer()
         ser.write((cmd + "\r").encode())
         lines = []
@@ -505,7 +506,11 @@ class Modem:
       except (AddressValueError, ValueError):
         pass
     if not dns_servers:
-      logging.warning(f"no cellular DNS servers reported by modem: {v!r}")
+      dns_servers = [
+        "8.8.8.8",  # Google
+        "1.1.1.1",  # Cloudflare
+      ]
+      logging.warning(f"no cellular DNS servers reported by modem: {v!r}; using fallback {dns_servers}")
     return dns_servers
 
   def _poll_byte_counters(self) -> dict:

openpilot/common/hardware/tici/tests/__init__.py

@@ -0,0 +1 @@
+

openpilot/common/hardware/tici/tests/test_amplifier.py

@@ -1,11 +1,10 @@
 import pytest
 import time
-import random
 import subprocess
 
 from panda import Panda
-from openpilot.system.hardware import TICI, HARDWARE
+from openpilot.common.hardware import TICI, HARDWARE
-from openpilot.system.hardware.tici.amplifier import Amplifier
+from openpilot.common.hardware.tici.amplifier import Amplifier
 
 
 class TestAmplifier:
@@ -57,7 +56,7 @@ class TestAmplifier:
       time.sleep(0.1)
 
       self.panda.set_siren(True)
-      time.sleep(random.randint(0, 5))
+      time.sleep(0.1)
 
       amp = Amplifier(debug=True)
       r = amp.initialize_configuration()