HKG: Car Port for G80 2022 (3.5T, HDA2)

* bump tg

* bump tg

* assign

* bump

* cpu llvm

* frame buffer updated in place, no need to return

* don't bake in stale pointers

* fix update image output indices

* lint

* bump

.clang-tidy

@@ -1,19 +0,0 @@
----
-Checks: '
-bugprone-*,
--bugprone-integer-division,
--bugprone-narrowing-conversions,
-performance-*,
-clang-analyzer-*,
-misc-*,
--misc-unused-parameters,
-modernize-*,
--modernize-avoid-c-arrays,
--modernize-deprecated-headers,
--modernize-use-auto,
--modernize-use-using,
--modernize-use-nullptr,
--modernize-use-trailing-return-type,
-'
-CheckOptions:
-...

.dockerignore

@@ -13,27 +13,6 @@
 *.o-*
 *.os
 *.os-*
-*.so
-*.a
 
 venv/
 .venv/
-
-notebooks
-phone
-massivemap
-neos
-installer
-chffr/app2
-chffr/backend/env
-selfdrive/nav
-selfdrive/baseui
-selfdrive/test/simulator2
-**/cache_data
-xx/plus
-xx/community
-xx/projects
-!xx/projects/eon_testing_master
-!xx/projects/map3d
-xx/ops
-xx/junk

.editorconfig

@@ -5,7 +5,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[{*.py, *.pyx, *.pxd}]
+[*.{py,pyx,pxd}]
 charset = utf-8
 indent_style = space
 indent_size = 2

.gitattributes

@@ -7,12 +7,12 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
+*.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
-selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text
-selfdrive/ui/qt/text_larch64 filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/car_bug_report.yml

@@ -1,53 +0,0 @@
-name: Car bug report
-description: For issues with a particular car make or model
-labels: ["car", "bug"]
-body:
-
-  - type: markdown
-    attributes:
-      value: >
-        Before creating a **bug report**, please check the following:
-          * Ensure you're running the latest openpilot release.
-          * Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
-          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
-          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
-
-        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
-
-  - type: textarea
-    attributes:
-      label: Describe the bug
-      description: Also include a description of how to reproduce the bug
-    validations:
-      required: true
-
-  - type: input
-    id: car
-    attributes:
-      label: Which car does this affect?
-      placeholder: Toyota Prius 2017
-    validations:
-      required: true
-
-  - type: input
-    id: route
-    attributes:
-      label: Provide a route where the issue occurs
-      description: Ensure the route is fully uploaded at https://useradmin.comma.ai
-      placeholder: 77611a1fac303767|2020-05-11--16-37-07
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: openpilot version
-      description: If you're not on release, provide the commit hash
-      placeholder: 0.8.10
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Additional info
-

.github/ISSUE_TEMPLATE/config.yml

@@ -1,9 +1,12 @@
 blank_issues_enabled: false
 contact_links:
+  - name: Car bug report
+    url: https://github.com/commaai/opendbc/issues/new
+    about: For issues with a particular car make or model
   - name: Join the Discord
     url: https://discord.comma.ai
     about: The community Discord is for both openpilot development and experience discussion
-  - name: Report model bugs
+  - name: Report driving behavior feedback
     url: https://discord.com/channels/469524606043160576/1254834193066623017
     about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord
   - name: Community Wiki

.github/PULL_REQUEST_TEMPLATE/bugfix.md

@@ -1,15 +0,0 @@
----
-name: Bug fix
-about: For openpilot bug fixes
-title: ''
-labels: 'bugfix'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the bug and the fix. Also link the issue if it exists. -->
-
-**Verification**
-
-<!-- Explain how you tested this bug fix. -->

.github/PULL_REQUEST_TEMPLATE/car_bugfix.md

@@ -1,19 +0,0 @@
----
-name: Car Bug fix
-about: For vehicle/brand specific bug fixes
-title: ''
-labels: 'car bug fix'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the bug and the fix. Also link the issue if it exists. -->
-
-**Verification**
-
-<!-- Explain how you tested this bug fix. -->
-
-**Route**
-
-Route: [a route with the bug fix]

.github/PULL_REQUEST_TEMPLATE/car_port.md

@@ -1,15 +0,0 @@
----
-name: Car port
-about: For new car ports
-title: ''
-labels: 'car port'
-assignees: ''
----
-
-**Checklist**
-
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
-- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
-- [ ] route with openpilot:
-- [ ] route with stock system:
-- [ ] car harness used (if comma doesn't sell it, put N/A):

.github/PULL_REQUEST_TEMPLATE/fingerprint.md

@@ -1,13 +0,0 @@
----
-name: Fingerprint
-about: For adding fingerprints to existing cars
-title: ''
-labels: 'fingerprint'
-assignees: ''
----
-
-**Car**
-Which car (make, model, year) this fingerprint is for
-
-**Route**
-A route with the fingerprint

.github/PULL_REQUEST_TEMPLATE/refactor.md

@@ -1,15 +0,0 @@
----
-name: Refactor
-about: For code refactors
-title: ''
-labels: 'refactor'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the refactor, including the goals it accomplishes. -->
-
-**Verification**
-
-<!-- Explain how you tested the refactor for regressions. -->

.github/PULL_REQUEST_TEMPLATE/tuning.md

@@ -1,31 +0,0 @@
----
-name: Tuning
-about: For openpilot tuning changes
-title: ''
-labels: 'tuning'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of what is wrong with the current tuning and how the PR addresses this. -->
-
-**Verification**
-
-<!-- To verify tuning, capture the following scenarios (broadly, not exactly), with current tune and this tune.
-Use the PlotJuggler tuning layout to compare planned versus actual behavior.
-
-Run ./juggle.py <route> --layout layouts/tuning.xml , screenshot the full tab of interest, and paste into this PR.
-
-Longitudinal:
-* Maintaining speed at 25, 40, 65mph
-* Driving up and down hills
-* Accelerating from a stop
-* Decelerating to a stop
-* Following large changes in set speed
-* Coming to a stop behind a lead car
-
-Lateral:
-* Straight driving at ~25, ~45 and ~65mph
-* Turns driving at ~25, ~45 and ~65mph
--->

.github/build.py

@@ -1,30 +0,0 @@
-import pathlib
-
-GITHUB_FOLDER = pathlib.Path(__file__).parent
-
-PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
-
-order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
-
-def create_pull_request_template():
-  with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
-    f.write("<!-- Please copy and paste the relevant template -->\n\n")
-
-    for t in order:
-      template = PULL_REQUEST_TEMPLATES / f"{t}.md"
-      text = template.read_text()
-
-      # Remove metadata for GitHub
-      start = text.find("---")
-      end = text.find("---", start+1)
-      text = text[end + 4:]
-
-      # Remove comments
-      text = text.replace("<!-- ", "").replace("-->", "")
-
-      f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
-      f.write(text)
-      f.write("\n\n")
-      f.write("-->\n\n")
-
-create_pull_request_template()

.github/labeler.yaml

@@ -1,6 +1,6 @@
 CI / testing:
   - changed-files:
-    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
+    - any-glob-to-all-files: "{.github/**,**/test_*,**/test/**,Jenkinsfile}"
 
 car:
   - changed-files:
@@ -12,7 +12,7 @@ simulation:
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: 'selfdrive/ui/**'
+    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
 
 tools:
   - changed-files:

.github/workflows/auto-cache/action.yaml

@@ -1,58 +0,0 @@
-name: 'automatically cache based on current runner'
-
-inputs:
-  path:
-    description: 'path to cache'
-    required: true
-  key:
-    description: 'key'
-    required: true
-  restore-keys:
-    description: 'restore-keys'
-    required: true
-  save:
-    description: 'whether to save the cache'
-    default: 'false'
-    required: false
-outputs:
-  cache-hit:
-    description: 'cache hit occurred'
-    value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
-               (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: setup namespace cache
-      id: ns-cache
-      if: ${{ contains(runner.name, 'nsc') }}
-      uses: namespacelabs/nscloud-cache-action@v1
-      with:
-        path: ${{ inputs.path }}
-
-    - name: setup github cache
-      id: gha-cache
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
-      uses: 'actions/cache@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    - name: setup github cache
-      id: gha-cache-ro
-      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
-      uses: 'actions/cache/restore@v4'
-      with:
-        path: ${{ inputs.path }}
-        key: ${{ inputs.key }}
-        restore-keys: ${{ inputs.restore-keys }}
-
-    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
-    - id: scons-cache-setup
-      shell: bash
-      run: |
-        mkdir -p ${{ inputs.path }}
-        sudo chmod -R 777 ${{ inputs.path }}
-        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -1,7 +1,7 @@
 name: "PR review"
 on:
   pull_request_target:
-    types: [opened, reopened, synchronize, edited, edited]
+    types: [opened, reopened, synchronize, edited]
 
 jobs:
   labeler:
@@ -11,12 +11,12 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v5.0.0
+    - uses: actions/labeler@v6
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -33,21 +33,3 @@ jobs:
         change-to: ${{ github.base_ref }}
         already-exists-action: close_this
         already-exists-comment: "Your PR should be made against the `master` branch"
-
-    # Welcome comment
-    - name: "First timers PR"
-      uses: actions/first-interaction@v1
-      if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        pr-message: |
-            <!-- _(run_id **${{ github.run_id }}**)_ -->
-            Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
-            * Convert your PR to a draft unless it's ready to review
-            * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
-            * Before marking as "ready for review", ensure:
-              * the goal is clearly stated in the description
-              * all the tests are passing
-              * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
-              * include a route or your device' dongle ID if relevant
-      

.github/workflows/badges.yaml

@@ -5,9 +5,7 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
+  PYTHONPATH: ${{ github.workspace }}
 
 jobs:
   badges:
@@ -17,13 +15,13 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: true
-    - uses: ./.github/workflows/setup-with-retry
+    - run: ./tools/op.sh setup
     - name: Push badges
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py"
+        python3 selfdrive/ui/translations/create_badges.py
 
         rm .gitattributes
 

.github/workflows/ci_weekly_report.yaml

@@ -1,101 +0,0 @@
-name: weekly CI test report
-on:
-  schedule:
-    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
-  workflow_dispatch:
-    inputs:
-      ci_runs:
-        description: 'The amount of runs to trigger in CI test report'
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
-
-jobs:
-  setup:
-    if: github.repository == 'commaai/openpilot'
-    runs-on: ubuntu-latest
-    outputs:
-      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
-    steps:
-      - id: ci_runs_setup
-        name: CI_RUNS=${{ env.CI_RUNS }}
-        run: |
-          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
-
-  ci_matrix_run:
-    needs: [ setup ]
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
-    uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
-    with:
-      run_number: ${{ matrix.run_number }}
-
-  report:
-    needs: [ci_matrix_run]
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Get job results
-        uses: actions/github-script@v7
-        id: get-job-results
-        with:
-          script: |
-            const jobs = await github
-              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
-                owner: "commaai",
-                repo: "${{ github.event.repository.name }}",
-                run_id: "${{ github.run_id }}",
-                attempt: "${{ github.run_attempt }}",
-              })
-            var report = {}
-            jobs.slice(1, jobs.length-1).forEach(job => {
-              if (job.conclusion === "skipped") return;
-              const jobName = job.name.split(" / ")[2];
-              const runRegex = /\((.*?)\)/;
-              const run = job.name.match(runRegex)[1];
-              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
-              switch (job.conclusion) {
-                case "success":
-                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
-                case "failure":
-                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
-                case "canceled":
-                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
-              }
-            });
-            return JSON.stringify({"jobs": report});
-
-      - name: Add job results to summary
-        env:
-          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
-        run: |
-          cat <<EOF >> template.html
-          <table>
-            <thead>
-              <tr>
-                <th></th>
-                <th>Job</th>
-                <th>✅ Passing</th>
-                <th>❌ Failure Details</th>
-              </tr>
-            </thead>
-            <tbody>
-              {% for key in jobs.keys() %}<tr>
-                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
-                <td>{{ key }}</td>
-                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
-                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
-                </td>
-              </tr>{% endfor %}
-          </table>
-          EOF
-
-          pip install jinja2-cli
-          echo $JOB_RESULTS | jinja2 template.html > report.html
-          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
-          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -1,17 +0,0 @@
-name: weekly CI test run
-on:
-  workflow_call:
-    inputs:
-      run_number:
-        required: true
-        type: string
-
-concurrency:
-  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  selfdrive_tests:
-    uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
-    with:
-      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -1,21 +0,0 @@
-name: 'compile openpilot'
-
-runs:
-  using: "composite"
-  steps:
-    - shell: bash
-      name: Build openpilot with all flags
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - shell: bash
-      name: Cleanup scons cache and rebuild
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-    - name: Save scons cache
-      uses: actions/cache/save@v4
-      if: github.ref == 'refs/heads/master'
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/diff_report.yaml

@@ -0,0 +1,45 @@
+name: diff report
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  comment:
+    name: comment
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+    - name: Wait for process replay
+      id: wait
+      continue-on-error: true
+      uses: lewagon/wait-on-check-action@v1.3.4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        check-name: process replay
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        allowed-conclusions: success,failure
+        wait-interval: 20
+    - name: Download diff
+      if: steps.wait.outcome == 'success'
+      uses: dawidd6/action-download-artifact@v6
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        workflow: tests.yaml
+        workflow_conclusion: ''
+        pr: ${{ github.event.number }}
+        name: diff_report_${{ github.event.number }}
+        path: .
+        allow_forks: true
+    - name: Comment on PR
+      if: steps.wait.outcome == 'success'
+      uses: thollander/actions-comment-pull-request@v2
+      with:
+        filePath: diff_report.txt
+        comment_tag: diff_report
+        pr_number: ${{ github.event.number }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -18,11 +18,11 @@ concurrency:
 jobs:
   docs:
     name: build docs
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: commaai/timeout@v1
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: true
 
@@ -34,7 +34,7 @@ jobs:
         mkdocs build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       with:
         path: openpilot-docs

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,14 +5,54 @@ on:
     types: [created, edited]
 
 jobs:
-  # TODO: gc old branches in a separate job in this workflow
+  cleanup-branches:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Delete stale Jenkins branches
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
+            const prefixes = ['tmp-jenkins', '__jenkins'];
+
+            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              per_page: 100,
+            })) {
+              for (const branch of response.data) {
+                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
+
+                const { data: commit } = await github.rest.repos.getCommit({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  ref: branch.commit.sha,
+                });
+
+                const commitDate = new Date(commit.commit.committer.date).getTime();
+                if (commitDate < cutoff) {
+                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
+                  await github.rest.git.deleteRef({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    ref: `heads/${branch.name}`,
+                  });
+                }
+              }
+            }
+
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
+    permissions:
+      contents: write
+      issues: write
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v7
+      uses: actions/github-script@v8
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -32,7 +72,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -43,3 +83,14 @@ jobs:
         git config --global user.email "github-actions[bot]@users.noreply.github.com"
         git checkout -b tmp-jenkins-${{ github.event.issue.number }}
         GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
+
+    - name: Delete trigger comment
+      if: steps.check_comment.outputs.result == 'true' && always()
+      uses: actions/github-script@v8
+      with:
+        script: |
+          await github.rest.issues.deleteComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            comment_id: context.payload.comment.id,
+          });

.github/workflows/model_review.yaml

@@ -0,0 +1,42 @@
+name: "model review"
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - 'selfdrive/modeld/models/*.onnx'
+  workflow_dispatch:
+
+jobs:
+  comment:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Checkout master
+        uses: actions/checkout@v6
+        with:
+          ref: master
+          path: base
+      - run: git lfs pull
+      - run: cd base && git lfs pull
+
+      - name: scripts/reporter.py
+        id: report
+        run: |
+          echo "content<<EOF" >> $GITHUB_OUTPUT
+          echo "## Model Review" >> $GITHUB_OUTPUT
+          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Post model report comment
+        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
+        with:
+          header: model-review
+          message: ${{ steps.report.outputs.content }}

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh prebuilt
+  BUILD: selfdrive/test/docker_build.sh
 
 jobs:
   build_prebuilt:
@@ -28,8 +28,8 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v4
+        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
+    - uses: actions/checkout@v6
       with:
         submodules: true
     - run: git lfs pull

.github/workflows/release.yaml

@@ -7,48 +7,25 @@ on:
 jobs:
   build_masterci:
     name: build master-ci
-    env:
-      TARGET_DIR: /tmp/openpilot
-      ImageOS: ubuntu20
-    container:
-      image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'commaai/openpilot'
     permissions:
       checks: read
       contents: write
     steps:
-    - name: Install wait-on-check-action dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
-      if: ${{ github.event_name != 'workflow_dispatch' }}
+      if: ${{ github.event_name == 'schedule' }}
       uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build master-ci'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt).*).)*$
+        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-    - name: Pull LFS
-      run: |
-        git config --global --add safe.directory '*'
-        git lfs pull
-    - name: Build master-ci
-      run: |
-        release/build_devel.sh
-    - name: Run tests
-      run: |
-        export PYTHONPATH=$TARGET_DIR
-        cd $TARGET_DIR
-        scons -j$(nproc)
-        pytest -n logical selfdrive/car/tests/test_car_interfaces.py
+    - run: ./tools/op.sh setup
     - name: Push master-ci
-      run: |
-        unset TARGET_DIR
-        BRANCH=master-ci release/build_devel.sh
+      run: BRANCH=__nightly release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -5,34 +5,51 @@ on:
     - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
   workflow_dispatch:
 
+env:
+  PYTHONPATH: ${{ github.workspace }}
+
 jobs:
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/commaai/openpilot-base:latest
     if: github.repository == 'commaai/openpilot'
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         submodules: true
+    - run: ./tools/op.sh setup
     - name: uv lock
+      run: uv lock --upgrade
+    - name: uv pip tree
+      id: pip_tree
       run: |
-        python3 -m ensurepip --upgrade
-        pip3 install uv
-        uv lock --upgrade
+        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
+        uv pip tree >> $GITHUB_OUTPUT
+        echo 'EOF' >> $GITHUB_OUTPUT
+    - name: venv size
+      id: venv_size
+      run: |
+        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
+        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
+        echo "" >> $GITHUB_OUTPUT
+        echo "Top 10 by size:" >> $GITHUB_OUTPUT
+        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
+          | grep -v '\.dist-info' \
+          | grep -v '__pycache__' \
+          | sort -rh \
+          | head -10 \
+          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
+        echo 'EOF' >> $GITHUB_OUTPUT
     - name: bump submodules
       run: |
-        git config --global --add safe.directory '*'
-        git -c submodule."tinygrad".update=none submodule update --remote
+        git submodule update --remote
         git add .
     - name: update car docs
       run: |
-        scons -j$(nproc) --minimal opendbc_repo
-        PYTHONPATH=. python selfdrive/car/docs.py
+        python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
+      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
       with:
         author: Vehicle Researcher <user@comma.ai>
         token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
@@ -41,5 +58,16 @@ jobs:
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: 'Automatic PR from repo-maintenance -> package_updates'
+        body: |
+          Automatic PR from repo-maintenance -> package_updates
+
+          ```
+          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
+          ${{ steps.venv_size.outputs.VENV_SIZE }}
+          ```
+
+          ```
+          $ uv pip tree
+          ${{ steps.pip_tree.outputs.PIP_TREE }}
+          ```
         labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -1,357 +0,0 @@
-name: selfdrive
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  PYTHONWARNINGS: error
-  BASE_IMAGE: openpilot-base
-  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
-
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh base
-
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
-  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
-
-jobs:
-  build_release:
-    name: build release
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Check submodules
-      if: github.repository == 'commaai/openpilot'
-      timeout-minutes: 3
-      run: release/check-submodules.sh
-    - name: Build openpilot and run checks
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "python3 system/manager/build.py"
-    - name: Run tests
-      timeout-minutes: 1
-      run: |
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "release/check-dirty.sh && \
-                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car system/manager"
-    - name: Static analysis
-      timeout-minutes: 1
-      run: |
-        cd $GITHUB_WORKSPACE
-        cp pyproject.toml $STRIPPED_DIR
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "scripts/lint/lint.sh --skip check_added_large_files"
-
-  build:
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Setup docker push
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-      run: |
-        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
-    - uses: ./.github/workflows/setup-with-retry
-    - uses: ./.github/workflows/compile-openpilot
-      timeout-minutes: 30
-
-  build_mac:
-    name: build macOS
-    runs-on: ${{ github.repository == 'commaai/openpilot' && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Homebrew cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: ~/Library/Caches/Homebrew
-    - name: Install dependencies
-      run: ./tools/mac_setup.sh
-      env:
-        # package install has DeprecationWarnings
-        PYTHONWARNINGS: default
-    - run: git lfs pull
-    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - name: Getting scons cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: /tmp/scons_cache
-    - name: Building openpilot
-      run: . .venv/bin/activate && scons -j$(nproc)
-
-  static_analysis:
-    name: static analysis
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    env:
-      PYTHONWARNINGS: default
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Static analysis
-      timeout-minutes: 1
-      run: ${{ env.RUN }} "scripts/lint/lint.sh"
-
-  unit_tests:
-    name: unit tests
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
-      run: |
-        ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \
-                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
-                        ./selfdrive/ui/tests/create_test_translations.sh && \
-                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
-                        chmod -R 777 /tmp/comma_download_cache"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  process_replay:
-    name: process replay
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v4
-      with:
-        path: .ci_cache/comma_download_cache
-        key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
-      run: |
-        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
-                        chmod -R 777 /tmp/comma_download_cache && \
-                        coverage combine && \
-                        coverage xml"
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v4
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload reference logs
-      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
-      run: |
-        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      run: |
-        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
-                        chmod -R 777 /tmp/comma_download_cache"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  test_cars:
-    name: cars
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        job: [0, 1, 2, 3]
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Cache test routes
-      id: routes-cache
-      uses: actions/cache@v4
-      with:
-        path: .ci_cache/comma_download_cache
-        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
-    - name: Build openpilot
-      run: ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Test car models
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 20 }}
-      run: |
-        ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
-                        chmod -R 777 /tmp/comma_download_cache"
-      env:
-        NUM_JOBS: 4
-        JOB_ID: ${{ matrix.job }}
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v4
-      with:
-        name: ${{ github.job }}-${{ matrix.job }}
-      env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-  car_docs_diff:
-    name: PR comments
-    runs-on: ubuntu-latest
-    #if: github.event_name == 'pull_request'
-    if: false  # TODO: run this in opendbc?
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          ref: ${{ github.event.pull_request.base.ref }}
-      - run: git lfs pull
-      - uses: ./.github/workflows/setup-with-retry
-      - name: Get base car info
-        run: |
-          ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
-          sudo chown -R $USER:$USER ${{ github.workspace }}
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          path: current
-      - run: cd current && git lfs pull
-      - name: Save car docs diff
-        id: save_diff
-        run: |
-          cd current
-          ${{ env.RUN }} "scons -j$(nproc)"
-          output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
-          output="${output//$'\n'/'%0A'}"
-          echo "::set-output name=diff::$output"
-      - name: Find comment
-        if: ${{ env.AZURE_TOKEN != '' }}
-        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          body-includes: This PR makes changes to
-      - name: Update comment
-        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
-        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body: "${{ steps.save_diff.outputs.diff }}"
-          edit-mode: replace
-      - name: Delete comment
-        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
-        uses: actions/github-script@v7
-        with:
-          script: |
-            github.rest.issues.deleteComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              comment_id: ${{ steps.fc.outputs.comment-id }}
-            })
-
-  simulator_driving:
-    name: simulator driving
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Driving test
-      timeout-minutes: 1
-      run: |
-        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
-                        source selfdrive/test/setup_vsound.sh && \
-                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
-
-  create_ui_report:
-    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
-    name: Create UI Report
-    runs-on:
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
-      - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - uses: ./.github/workflows/setup-with-retry
-      - name: caching frames
-        id: frames-cache
-        uses: actions/cache@v4
-        with:
-          path: .ci_cache/comma_download_cache
-          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
-      - name: Build openpilot
-        run: ${{ env.RUN }} "scons -j$(nproc)"
-      - name: Create Test Report
-        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }}
-        run: >
-            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-                            source selfdrive/test/setup_xvfb.sh &&
-                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
-                            chmod -R 777 /tmp/comma_download_cache"
-      - name: Upload Test Report
-        uses: actions/upload-artifact@v4
-        with:
-          name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-with-retry/action.yaml

@@ -1,37 +0,0 @@
-name: 'openpilot env setup, with retry on failure'
-
-inputs:
-  docker_hub_pat:
-    description: 'Auth token for Docker Hub, required for BuildJet jobs'
-    required: false
-    default: ''
-  sleep_time:
-    description: 'Time to sleep between retries'
-    required: false
-    default: 30
-
-runs:
-  using: "composite"
-  steps:
-    - id: setup1
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup1.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup2
-      if: steps.setup1.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      continue-on-error: true
-      with:
-        is_retried: true
-    - if: steps.setup2.outcome == 'failure'
-      shell: bash
-      run: sleep ${{ inputs.sleep_time }}
-    - id: setup3
-      if: steps.setup2.outcome == 'failure'
-      uses: ./.github/workflows/setup
-      with:
-        is_retried: true

.github/workflows/setup/action.yaml

@@ -1,56 +0,0 @@
-name: 'openpilot env setup'
-
-inputs:
-  is_retried:
-    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
-    required: false
-    default: 'false'
-
-runs:
-  using: "composite"
-  steps:
-    # assert that this action is retried using the setup-with-retry
-    - shell: bash
-      if: ${{ inputs.is_retried == 'false' }}
-      run: |
-        echo "You should not run this action directly. Use setup-with-retry instead"
-        exit 1
-
-    - shell: bash
-      name: No retries!
-      run: |
-        if [ "${{ github.run_attempt }}" -gt 1 ]; then
-          echo -e "\033[0;31m##################################################"
-          echo -e "\033[0;31m    Retries not allowed! Fix the flaky test!      "
-          echo -e "\033[0;31m##################################################\033[0m"
-          exit 1
-        fi
-
-    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
-    - shell: bash
-      run: git lfs pull
-
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
-    - id: scons-cache
-      uses: ./.github/workflows/auto-cache
-      with:
-        path: .ci_cache/scons_cache
-        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}
-    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
-    - id: normalize-file-permissions
-      shell: bash
-      name: Normalize file permissions to ensure a consistent docker build cache
-      run: |
-        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
-        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
-    # build our docker image
-    - shell: bash
-      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -5,14 +5,15 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 2
-  DAYS_BEFORE_PR_STALE: 9
+  DAYS_BEFORE_PR_CLOSE: 7
+  DAYS_BEFORE_PR_STALE: 24
+  DAYS_BEFORE_PR_STALE_DRAFT: 30
 
 jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@v10
         with:
           exempt-all-milestones: true
 
@@ -24,6 +25,28 @@ jobs:
           exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
           days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
           days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
+          exempt-draft-pr: false
+
+          # issue config
+          days-before-issue-stale: -1 # ignore issues for now
+
+  # same as above, but give draft PRs more time
+  stale_drafts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v10
+        with:
+          exempt-all-milestones: true
+
+          # pull request config
+          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
+          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
+          stale-pr-label: stale
+          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
+          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
+          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }}
+          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
+          exempt-draft-pr: true
 
           # issue config
           days-before-issue-stale: -1 # ignore issues for now

.github/workflows/tests.yaml

@@ -0,0 +1,231 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
+
+concurrency:
+  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  CI: 1
+  PYTHONPATH: ${{ github.workspace }}
+  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
+
+jobs:
+  build_release:
+    name: build release
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    env:
+      STRIPPED_DIR: /tmp/releasepilot
+      PYTHONPATH: /tmp/releasepilot
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - name: Getting LFS files
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: git lfs pull
+    - name: Build devel
+      timeout-minutes: 1
+      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
+    - run: ./tools/op.sh setup
+    - name: Build openpilot and run checks
+      timeout-minutes: 30
+      working-directory: ${{ env.STRIPPED_DIR }}
+      run: python3 system/manager/build.py
+    - name: Run tests
+      timeout-minutes: 1
+      working-directory: ${{ env.STRIPPED_DIR }}
+      run: release/check-dirty.sh
+    - name: Check submodules
+      if: github.repository == 'commaai/openpilot'
+      timeout-minutes: 3
+      run: release/check-submodules.sh
+
+  build_mac:
+    name: build macOS
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - name: Remove Homebrew from environment
+      run: |
+        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
+        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
+    - run: ./tools/op.sh setup
+    - name: Building openpilot
+      run: scons
+
+  static_analysis:
+    name: static analysis
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Static analysis
+      timeout-minutes: 1
+      run: scripts/lint/lint.sh
+
+  unit_tests:
+    name: unit tests
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Build openpilot
+      run: scons -j$(nproc)
+    - name: Run unit tests
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
+      run: |
+        source selfdrive/test/setup_xvfb.sh
+        # Pre-compile Python bytecode so each pytest worker doesn't need to
+        $PYTEST --collect-only -m 'not slow' -qq
+        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
+
+  process_replay:
+    name: process replay
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Build openpilot
+      run: scons -j$(nproc)
+    - name: Run replay
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
+      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
+      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
+    - name: Print diff
+      id: print-diff
+      if: always()
+      run: cat selfdrive/test/process_replay/diff.txt
+    - name: Print diff report
+      if: always()
+      run: cat selfdrive/test/process_replay/diff_report.txt
+    - uses: actions/upload-artifact@v6
+      if: always()
+      continue-on-error: true
+      with:
+        name: process_replay_diff.txt
+        path: selfdrive/test/process_replay/diff.txt
+    - name: Upload diff report
+      uses: actions/upload-artifact@v6
+      if: always() && github.event_name == 'pull_request'
+      continue-on-error: true
+      with:
+        name: diff_report_${{ github.event.number }}
+        path: selfdrive/test/process_replay/diff_report.txt
+    - name: Checkout ci-artifacts
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
+      uses: actions/checkout@v4
+      with:
+        repository: commaai/ci-artifacts
+        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+        path: ${{ github.workspace }}/ci-artifacts
+    - name: Push refs
+      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
+      working-directory: ${{ github.workspace }}/ci-artifacts
+      run: |
+        git config user.name "GitHub Actions Bot"
+        git config user.email "<>"
+        git fetch origin process-replay || true
+        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
+        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
+        echo "${{ github.sha }}" > ref_commit
+        git add .
+        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
+        git push origin process-replay --force
+    - name: Run regen
+      if: false
+      timeout-minutes: 4
+      env:
+        ONNXCPU: 1
+      run: $PYTEST selfdrive/test/process_replay/test_regen.py
+
+  simulator_driving:
+    name: simulator driving
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    if: false  # FIXME: Started to timeout recently
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        submodules: true
+    - run: ./tools/op.sh setup
+    - name: Build openpilot
+      run: scons -j$(nproc)
+    - name: Driving test
+      timeout-minutes: 2
+      run: |
+        source selfdrive/test/setup_xvfb.sh
+        pytest -s tools/sim/tests/test_metadrive_bridge.py
+
+  create_ui_report:
+    name: Create UI Report
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
+      - run: ./tools/op.sh setup
+      - name: Build openpilot
+        run: scons -j$(nproc)
+      - name: Create UI Report
+        run: |
+          source selfdrive/test/setup_xvfb.sh
+          python3 selfdrive/ui/tests/diff/replay.py
+          python3 selfdrive/ui/tests/diff/replay.py --big
+      - name: Upload UI Report
+        uses: actions/upload-artifact@v6
+        with:
+          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -8,14 +8,20 @@ on:
     branches:
       - 'master'
     paths:
+      - 'selfdrive/assets/**'
       - 'selfdrive/ui/**'
+      - 'system/ui/**'
   workflow_dispatch:
 
 env:
   UI_JOB_NAME: "Create UI Report"
   REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
   SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
+  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
+
+  # variant:video_prefix:master_branch
+  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
 
 jobs:
   preview:
@@ -28,8 +34,9 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - name: Waiting for ui generation to start
-        run: sleep 30
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
 
       - name: Waiting for ui generation to end
         uses: lewagon/wait-on-check-action@v1.3.4
@@ -46,98 +53,93 @@ jobs:
           echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
       - name: Getting proposed ui
-        id: download-artifact
         uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: report-1-${{ env.REPORT_NAME }}
+          name: ui-report-1-${{ env.REPORT_NAME }}
           path: ${{ github.workspace }}/pr_ui
 
-      - name: Getting master ui
-        uses: actions/checkout@v4
+      - name: Getting mici master ui
+        uses: actions/checkout@v6
         with:
           repository: commaai/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_ui
-          ref: openpilot_master_ui
+          path: ${{ github.workspace }}/master_mici
+          ref: openpilot_master_ui_mici_raylib
+
+      - name: Getting big master ui
+        uses: actions/checkout@v6
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_big
+          ref: openpilot_master_ui_big_raylib
 
       - name: Saving new master ui
         if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          git checkout --orphan=new_master_ui
-          git rm -rf *
-          git branch -D openpilot_master_ui
-          git branch -m openpilot_master_ui
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          mv ${{ github.workspace }}/pr_ui/*.png .
-          git add .
-          git commit -m "screenshots for commit ${{ env.SHA }}"
-          git push origin openpilot_master_ui --force
+          for variant in $VARIANTS; do
+            IFS=':' read -r name video branch <<< "$variant"
+            master_dir="${{ github.workspace }}/master_${name}"
+            cd "$master_dir"
+            git checkout --orphan=new_branch
+            git rm -rf *
+            git branch -D "$branch"
+            git branch -m "$branch"
+            git config user.name "GitHub Actions Bot"
+            git config user.email "<>"
+            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
+            git add .
+            git commit -m "${name} video for commit ${{ env.SHA }}"
+            git push origin "$branch" --force
+          done
 
-      - name: Finding diff
+      - name: Setup FFmpeg
+        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
+
+      - name: Finding diffs
         if: github.event_name == 'pull_request_target'
         id: find_diff
-        run: >-
-          sudo apt-get install -y imagemagick
+        run: |
+          export PYTHONPATH=${{ github.workspace }}
+          baseurl="https://github.com/commaai/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
 
-          scenes="homescreen settings_device settings_software settings_toggles settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard"
-          A=($scenes)
+          COMMENT=""
+          for variant in $VARIANTS; do
+            IFS=':' read -r name video _ <<< "$variant"
+            diff_name="${name}_diff"
 
-          DIFF=""
-          TABLE="<details><summary>All Screenshots</summary>"
-          TABLE="${TABLE}<table>"
+            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
+            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
 
-          for ((i=0; i<${#A[*]}; i=i+1));
-          do
+            diff_exit_code=0
+            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
+              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
+              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
+              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
 
-            if ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
-              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
-              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
-              convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
 
-              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
-
-              DIFF="${DIFF}<details open>"
-              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
-              DIFF="${DIFF}<table>"
-
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
-              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-              DIFF="${DIFF}</tr>"
-
-              DIFF="${DIFF}<tr>"
-              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
-              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
-              DIFF="${DIFF}</tr>"
-
-              DIFF="${DIFF}</table>"
-              DIFF="${DIFF}</details>"
+            REPORT_URL="https://commaai.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
+            if [ $diff_exit_code -eq 0 ]; then
+              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
             else
-              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
-            fi
-
-            INDEX=$(($i % 2))
-            if [[ $INDEX -eq 0 ]]; then
-              TABLE="${TABLE}<tr>"
-            fi
-            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
-            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
-              TABLE="${TABLE}</tr>"
+              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
             fi
           done
 
-          TABLE="${TABLE}</table></details>"
-
-          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
+          {
+            echo "COMMENT<<EOF"
+            echo "$COMMENT"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
 
       - name: Saving proposed ui
         if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_ui
+        working-directory: ${{ github.workspace }}/master_mici
         run: |
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
@@ -145,17 +147,29 @@ jobs:
           git rm -rf *
           mv ${{ github.workspace }}/pr_ui/* .
           git add .
-          git commit -m "screenshots for PR #${{ github.event.number }}"
+          git commit -m "ui videos for PR #${{ github.event.number }}"
           git push origin ${{ env.BRANCH_NAME }} --force
 
-      - name: Comment Screenshots on PR
+          # Append diff reports to report files branch
+          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
+          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
+          for variant in $VARIANTS; do
+            IFS=':' read -r name _ _ <<< "$variant"
+            diff_name="${name}_diff"
+            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
+            git add "${diff_name}_pr_${{ github.event.number }}.html"
+          done
+          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
+          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
+
+      - name: Comment on PR
         if: github.event_name == 'pull_request_target'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
-            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
+            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
             ## UI Preview
-            ${{ steps.find_diff.outputs.DIFF }}
-          comment_tag: run_id_screenshots
+            ${{ steps.find_diff.outputs.COMMENT }}
+          comment_tag: run_id_ui_preview
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -10,15 +10,16 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
-model2.png
 a.out
 .hypothesis
+.cache/
+bin/
 
-/docs_site/
-
+*.mp4
 *.dylib
 *.DSYM
 *.d
+*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -35,71 +36,53 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
-*.qm
+*.mo
+*_pyx.cpp
+*.stats
+*.pkl
+*.pkl*
 config.json
-clcache
 compile_commands.json
 compare_runtime*.html
 
-persist
+# build artifacts
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
 cereal/messaging/bridge
-selfdrive/logcatd/logcatd
-selfdrive/mapd/default_speeds_by_region.json
-system/proclogd/proclogd
-selfdrive/ui/translations/alerts_generated.h
 selfdrive/ui/translations/tmp
-selfdrive/test/longitudinal_maneuvers/out
 selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 
-notebooks
-hyperthneed
-provisioning
-
 .coverage*
 coverage.xml
 htmlcov
-pandaextra
-
-.mypy_cache/
-flycheck_*
-
-cppcheck_report.txt
-comma*.sh
-
-selfdrive/modeld/thneed/compile
-selfdrive/modeld/models/*.thneed
-selfdrive/modeld/models/*.pkl
 
+# openpilot log files
 *.bz2
 *.zst
+*.rlog
 
 build/
 
 !**/.gitkeep
 
-poetry.toml
-Pipfile
 
 ### VisualStudioCode ###
+*.vsix
+.history
+.ionide
 .vscode/*
+.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# Local History for Visual Studio Code
-.history/
-
-# Built Visual Studio Code Extensions
-*.vsix
-
-### VisualStudioCode Patch ###
-# Ignore all local history of files
-.history
-.ionide
+# agents
+.claude/
+.context/
+PLAN.md
+TASK.md

.gitmodules

@@ -15,4 +15,4 @@
   url = ../../commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/commaai/tinygrad.git
+  url = https://github.com/tinygrad/tinygrad.git

.python-version

@@ -0,0 +1 @@
+3.12.13

.vscode/launch.json

@@ -23,6 +23,11 @@
       "id": "args",
       "description": "Arguments to pass to the process",
       "type": "promptString"
+    },
+    {
+      "id": "replayArg",
+      "type": "promptString",
+      "description": "Enter route or segment to replay."
     }
   ],
   "configurations": [
@@ -40,7 +45,41 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Attach LLDB to Replay drive",
+      "type": "lldb",
+      "request": "attach",
+      "pid": "${command:pickMyProcess}",
+      "initCommands": [
+        "script import time; time.sleep(3)"
+      ]
+    },
+    {
+      "name": "Replay drive",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
+      "args": [
+        "${input:replayArg}"
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      },
+      "subProcess": true,
+      "stopOnEntry": false
+    }
+  ],
+  "compounds": [
+    {
+      "name": "Replay drive + Safety LLDB",
+      "configurations": [
+        "Replay drive",
+        "Attach LLDB to Replay drive"
+      ]
     }
   ]
 }

Dockerfile.openpilot

@@ -1,13 +1,38 @@
-FROM ghcr.io/commaai/openpilot-base:latest
+FROM ubuntu:24.04
 
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
-ENV OPENPILOT_PATH /home/batman/openpilot
-ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales && \
+    rm -rf /var/lib/apt/lists/*
 
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+ENV OPENPILOT_PATH=/home/$USER/openpilot
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY . ${OPENPILOT_PATH}/
+COPY --chown=$USER . ${OPENPILOT_PATH}/
 
-RUN scons --cache-readonly -j$(nproc)
+ENV UV_BIN="/home/$USER/.local/bin/"
+ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
+ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
+RUN tools/setup_dependencies.sh && \
+    sudo rm -rf /var/lib/apt/lists/*
+
+USER root
+RUN git config --global --add safe.directory '*'

Dockerfile.openpilot_base

@@ -1,81 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED 1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/* && \
-    cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
-
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX 1
-
-RUN dbus-uuidgen > /etc/machine-id
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /home/$USER
-COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /home/$USER && \
-    tools/install_python_dependencies.sh && \
-    rm -rf tools/ pyproject.toml uv.lock .cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -22,7 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-export LOGPRINT=debug
+#export LOGPRINT=debug # this has gotten too spammy...
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -31,7 +31,7 @@ export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
 export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
 # only use 1 thread for tici tests since most require HIL
-export PYTEST_ADDOPTS="-n 0"
+export PYTEST_ADDOPTS="-n0 -s"
 
 
 export GIT_SSH_COMMAND="ssh -i /data/gitkey"
@@ -103,7 +103,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
             def diffPaths = args.diffPaths ?: []
             def cmdTimeout = args.timeout ?: 9999
 
-            if (branch != "master" && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
+            if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
               println "Skipping ${name}: no changes in ${diffPaths}."
               return
             } else {
@@ -166,11 +166,11 @@ node {
   env.GIT_BRANCH = checkout(scm).GIT_BRANCH
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
-  def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'testing-closet*', 'hotfix-*']
+  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
+                         'release-tici', 'release-tizi', 'release-tizi-staging', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master') {
+  if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])
@@ -178,20 +178,20 @@ node {
 
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release3-staging", "tici-needs-can", [], [
-        step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"),
+      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh && git push -f origin release-tizi-staging:release-mici-staging"),
       ])
     }
 
-    if (env.BRANCH_NAME == 'master-ci') {
+    if (env.BRANCH_NAME == '__nightly') {
       parallel (
         'nightly': {
-          deviceStage("build nightly", "tici-needs-can", [], [
+          deviceStage("build nightly", "tizi-needs-can", [], [
             step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
-          deviceStage("build nightly-dev", "tici-needs-can", [], [
+          deviceStage("build nightly-dev", "tizi-needs-can", [], [
             step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
@@ -200,57 +200,43 @@ node {
 
     if (!env.BRANCH_NAME.matches(excludeRegex)) {
     parallel (
-      // tici tests
       'onroad tests': {
-        deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [
-          // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
-          //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
+        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
           step("check dirty", "release/check-dirty.sh"),
           step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
-          //["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
         ])
       },
       'HW + Unit Tests': {
-        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
+        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
           step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
-          step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py", [diffPaths: ["system/ubloxd/"]]),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
-      'loopback': {
-        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
-          step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+      'camerad OX03C10': {
+        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
-      'camerad': {
-        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
+      'camerad OS04C10': {
+        deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
-        ])
-        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
-          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'sensord': {
-        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
-        ])
-        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
+        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
-        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
+        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
           step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
         ])
@@ -258,9 +244,8 @@ node {
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
           step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
         ])

README.md

@@ -5,7 +5,7 @@
 <p>
   <b>openpilot is an operating system for robotics.</b>
   <br>
-  Currently, it upgrades the driver assistance system in 275+ supported cars.
+  Currently, it upgrades the driver assistance system in 300+ supported cars.
 </p>
 
 <h3>
@@ -17,13 +17,12 @@
   <span> · </span>
   <a href="https://discord.comma.ai">Community</a>
   <span> · </span>
-  <a href="https://comma.ai/shop">Try it on a comma 3X</a>
+  <a href="https://comma.ai/shop">Try it on a comma four</a>
 </h3>
 
 Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
 
-![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg)
-[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)
+[![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/tests.yaml/badge.svg)](https://github.com/commaai/openpilot/actions/workflows/tests.yaml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
 [![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
@@ -38,17 +37,30 @@ Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
   </tr>
 </table>
 
-To start using openpilot in a car
+
+Using openpilot in a car
 ------
 
 To use openpilot in a car, you need four things:
-1. **Supported Device:** a comma 3/3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x).
-2. **Software:** The setup procedure for the comma 3/3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
+1. **Supported Device:** a comma four, available at [comma.ai/shop/comma-four](https://www.comma.ai/shop/comma-four).
+2. **Software:** The setup procedure for the comma four allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
 3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
-4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3/3X to your car.
+4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma four to your car.
 
 We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
 
+
+### Branches
+
+Running `master` and other branches directly is supported, but it's recommended to run one of the following prebuilt branches:
+
+| comma four branch      | comma 3X branch        | URL                                    | description                                                                         |
+|------------------------|------------------------|----------------------------------------|-------------------------------------------------------------------------------------|
+| `release-mici`         | `release-tizi`         | openpilot.comma.ai                     | This is openpilot's release branch.                                                 |
+| `release-mici-staging` | `release-tizi-staging` | openpilot-test.comma.ai                | This is the staging branch for releases. Use it to get new releases slightly early. |
+| `nightly`              | `nightly`              | openpilot-nightly.comma.ai             | This is the bleeding edge development branch. Do not expect this to be stable.      |
+| `nightly-dev`          | `nightly-dev`          | installer.comma.ai/commaai/nightly-dev | Same as nightly, but includes experimental development features for some cars.      |
+
 To start developing openpilot
 ------
 
@@ -57,7 +69,6 @@ openpilot is developed by [comma](https://comma.ai/) and by users like you. We w
 * Join the [community Discord](https://discord.comma.ai)
 * Check out [the contributing docs](docs/CONTRIBUTING.md)
 * Check out the [openpilot tools](tools/)
-* Read about the [development workflow](docs/WORKFLOW.md)
 * Code documentation lives at https://docs.comma.ai
 * Information about running openpilot lives on the [community wiki](https://github.com/commaai/openpilot/wiki)
 
@@ -67,15 +78,15 @@ Safety and Testing
 ----
 
 * openpilot observes [ISO26262](https://en.wikipedia.org/wiki/ISO_26262) guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
-* openpilot has software-in-the-loop [tests](.github/workflows/selfdrive_tests.yaml) that run on every commit.
+* openpilot has software-in-the-loop [tests](.github/workflows/tests.yaml) that run on every commit.
 * The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
 * panda has software-in-the-loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
 * Internally, we have a hardware-in-the-loop Jenkins test suite that builds and unit tests the various processes.
 * panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
 * We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
 
-Licensing
-------
+<details>
+<summary>MIT Licensed</summary>
 
 openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
@@ -84,15 +95,17 @@ Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and i
 **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
 YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
 NO WARRANTY EXPRESSED OR IMPLIED.**
+</details>
 
-User Data and comma Account
-------
+<details>
+<summary>User Data and comma Account</summary>
 
 By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
 
 openpilot is open source software: the user is free to disable data collection if they wish to do so.
 
 openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
+The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
 
 By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
+</details>

RELEASES.md

@@ -1,16 +1,88 @@
-Version 0.9.8 (2024-XX-XX)
+Version 0.11.1 (2026-04-22)
+========================
+* New driver monitoring model
+* Improved image processing pipeline for driver camera
+* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
+
+Version 0.11.0 (2026-03-17)
+========================
+* New driving model #36798
+  * Fully trained using a learned simulator
+  * Improved longitudinal performance in Experimental mode
+* Reduce comma four standby power usage by 77% to 52 mW
+* Kia K7 2017 support thanks to royjr!
+* Lexus LS 2018 support thanks to Hacheoy!
+
+Version 0.10.3 (2025-12-17)
+========================
+* New driving model #36249
+  * New temporal policy architecture
+  * New on-policy training physics noise model
+* New driver monitoring model #36409
+  * Trained on a new dataset, including comma four data
+* Improved inter-process communication memory efficiency
+
+Version 0.10.2 (2025-11-19)
+========================
+* comma four support
+
+Version 0.10.1 (2025-09-08)
+========================
+* New driving model #36276
+  * World Model: removed global localization inputs
+  * World Model: 2x the number of parameters
+  * World Model: trained on 4x the number of segments
+  * VAE Compression Model: new architecture and training objective
+  * Driving Vision Model: trained on 4x the number of segments
+* New Driver Monitoring model #36198
+* Acura TLX 2021 support thanks to MVL!
+* Honda City 2023 support thanks to vanillagorillaa and drFritz!
+* Honda N-Box 2018 support thanks to miettal!
+* Honda Odyssey 2021-25 support thanks to csouers and MVL!
+* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
+
+Version 0.10.0 (2025-08-05)
 ========================
 * New driving model
-  * Trained in brand new ML simulator
-  * Model now gates applying positive accel in Chill mode
-* New driving monitoring model
+  * New training architecture
+     * Described in our CVPR paper: "Learning to Drive from a World Model"
+     * Longitudinal MPC replaced by E2E planning from World Model in Experimental Mode
+     * Action from lateral MPC as training objective replaced by E2E planning from World Model
+  * Low-speed lead car ground-truth fixes
+* Enable live-learned steering actuation delay
+* Opt-in audio recording for dashcam video
+* Acura MDX 2025 support thanks to vanillagorillaa and MVL!
+* Honda Accord 2023-25 support thanks to vanillagorillaa and MVL!
+* Honda CR-V 2023-25 support thanks to vanillagorillaa and MVL!
+* Honda Pilot 2023-25 support thanks to vanillagorillaa and MVL!
+
+Version 0.9.9 (2025-05-23)
+========================
+* New driving model
+  * New training architecture using parts from MLSIM
+* Steering actuation delay is now learned online
+* Ford Escape 2023-24 support thanks to incognitojam!
+* Ford Kuga 2024 support thanks to incognitojam!
+* Hyundai Nexo 2021 support thanks to sunnyhaibin!
+* Tesla Model 3 and Y support thanks to lukasloetkolben!
+* Lexus RC 2023 support thanks to nelsonjchen!
+
+Version 0.9.8 (2025-02-28)
+========================
+* New driving model
+  * Model now gates applying positive acceleration in Chill mode
+* New driver monitoring model
   * Reduced false positives related to passengers
 * Image processing pipeline moved to the ISP
-  * More GPU time for driving models
-  * Power draw reduced 0.5W, which means your device runs cooler 
+  * More GPU time for bigger driving models
+  * Power draw reduced 0.5W, which means your device runs cooler
 * Added toggle to enable driver monitoring even when openpilot is not engaged
+* Localizer rewritten to remove GPS dependency at runtime
+* Firehose Mode for maximizing your training data uploads
 * Enable openpilot longitudinal control for Ford Q3 vehicles
 * New Toyota TSS2 longitudinal tune
+* Rivian R1S and R1T support thanks to lukasloetkolben!
+* Ford F-150, F-150 Hybrid, Mach-E, and Ranger support
 
 Version 0.9.7 (2024-06-13)
 ========================

SConstruct

@@ -3,358 +3,227 @@ import subprocess
 import sys
 import sysconfig
 import platform
+import shlex
+import importlib
 import numpy as np
 
 import SCons.Errors
+from SCons.Defaults import _stripixes
 
 SCons.Warnings.warningAsException(True)
 
-# pending upstream fix - https://github.com/SCons/scons/issues/4461
-#SetOption('warn', 'all')
-
-TICI = os.path.isfile('/TICI')
-AGNOS = TICI
-
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', int(os.cpu_count()/2))
-
-AddOption('--kaitai',
-          action='store_true',
-          help='Regenerate kaitai struct parsers')
-
-AddOption('--asan',
-          action='store_true',
-          help='turn on ASAN')
-
-AddOption('--ubsan',
-          action='store_true',
-          help='turn on UBSan')
-
-AddOption('--coverage',
-          action='store_true',
-          help='build with test coverage options')
-
-AddOption('--clazy',
-          action='store_true',
-          help='build with clazy')
-
-AddOption('--compile_db',
-          action='store_true',
-          help='build clang compilation database')
-
-AddOption('--ccflags',
-          action='store',
-          type='string',
-          default='',
-          help='pass arbitrary flags over the command line')
-
-AddOption('--snpe',
-          action='store_true',
-          help='use SNPE on PC')
-
-AddOption('--external-sconscript',
-          action='store',
-          metavar='FILE',
-          dest='external_sconscript',
-          help='add an external SConscript to the build')
-
-AddOption('--pc-thneed',
-          action='store_true',
-          dest='pc_thneed',
-          help='use thneed on pc')
-
-AddOption('--mutation',
-          action='store_true',
-          help='generate mutation-ready code')
+SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
 
+AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
+AddOption('--verbose', action='store_true', default=False, help='show full build commands')
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
+          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-## Architecture name breakdown (arch)
-## - larch64: linux tici aarch64
-## - aarch64: linux pc aarch64
-## - x86_64:  linux pc x64
-## - Darwin:  mac x64 or arm64
-real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+# Detect platform
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-elif arch == "aarch64" and AGNOS:
+elif arch == "aarch64" and os.path.isfile('/TICI'):
   arch = "larch64"
-assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
+assert arch in [
+  "larch64",  # linux tici arm64
+  "aarch64",  # linux pc arm64
+  "x86_64",   # linux pc x64
+  "Darwin",   # macOS arm64 (x86 not supported)
+]
 
-lenv = {
-  "PATH": os.environ['PATH'],
-  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
-  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+pkg_names = ['bzip2', 'capnproto', 'eigen', 'ffmpeg', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
+pkgs = [importlib.import_module(name) for name in pkg_names]
 
-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+
+# ***** enforce a whitelist of system libraries *****
+# this prevents silently relying on a 3rd party package,
+# e.g. apt-installed libusb. all libraries should either
+# be distributed with all Linux distros and macOS, or
+# vendored in commaai/dependencies.
+allowed_system_libs = {
+  "EGL", "GLESv2", "GL",
+  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
+  "dl", "drm", "gbm", "m", "pthread",
 }
 
-rpath = lenv["LD_LIBRARY_PATH"].copy()
+def _resolve_lib(env, name):
+  for d in env.Flatten(env.get('LIBPATH', [])):
+    p = Dir(str(d)).abspath
+    for ext in ('.a', '.so', '.dylib'):
+      f = File(os.path.join(p, f'lib{name}{ext}'))
+      if f.exists() or f.has_builder():
+        return name
+  if name in allowed_system_libs:
+    return name
+  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
 
-if arch == "larch64":
-  cpppath = [
-    "#third_party/opencl/include",
-  ]
-
-  libpath = [
-    "/usr/local/lib",
-    "/system/vendor/lib64",
-    f"#third_party/acados/{arch}/lib",
-  ]
-
-  libpath += [
-    "#third_party/snpe/larch64",
-    "#third_party/libyuv/larch64/lib",
-    "/usr/lib/aarch64-linux-gnu"
-  ]
-  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  rpath += ["/usr/local/lib"]
-else:
-  cflags = []
-  cxxflags = []
-  cpppath = []
-  rpath += []
-
-  # MacOS
-  if arch == "Darwin":
-    libpath = [
-      f"#third_party/libyuv/{arch}/lib",
-      f"#third_party/acados/{arch}/lib",
-      f"{brew_prefix}/lib",
-      f"{brew_prefix}/opt/openssl@3.0/lib",
-      "/System/Library/Frameworks/OpenGL.framework/Libraries",
-    ]
-
-    cflags += ["-DGL_SILENCE_DEPRECATION"]
-    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
-    cpppath += [
-      f"{brew_prefix}/include",
-      f"{brew_prefix}/opt/openssl@3.0/include",
-    ]
-    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
-  # Linux
-  else:
-    libpath = [
-      f"#third_party/acados/{arch}/lib",
-      f"#third_party/libyuv/{arch}/lib",
-      "/usr/lib",
-      "/usr/local/lib",
-    ]
-
-    if arch == "x86_64":
-      libpath += [
-        f"#third_party/snpe/{arch}"
-      ]
-      rpath += [
-        Dir(f"#third_party/snpe/{arch}").abspath,
-      ]
-
-if GetOption('asan'):
-  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
-  ldflags = ["-fsanitize=address"]
-elif GetOption('ubsan'):
-  ccflags = ["-fsanitize=undefined"]
-  ldflags = ["-fsanitize=undefined"]
-else:
-  ccflags = []
-  ldflags = []
-
-# no --as-needed on mac linker
-if arch != "Darwin":
-  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
-
-ccflags_option = GetOption('ccflags')
-if ccflags_option:
-  ccflags += ccflags_option.split(' ')
+def _libflags(target, source, env, for_signature):
+  libs = []
+  lp = env.subst('$LIBLITERALPREFIX')
+  for lib in env.Flatten(env.get('LIBS', [])):
+    if isinstance(lib, str):
+      if os.sep in lib or lib.startswith('#'):
+        libs.append(File(lib))
+      elif lib.startswith('-') or (lp and lib.startswith(lp)):
+        libs.append(lib)
+      else:
+        libs.append(_resolve_lib(env, lib))
+    else:
+      libs.append(lib)
+  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
+                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
 
 env = Environment(
-  ENV=lenv,
+  ENV={
+    "PATH": os.environ['PATH'],
+    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+  },
   CCFLAGS=[
     "-g",
     "-fPIC",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow",
+    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
     "-Wno-vla-cxx-extension",
-  ] + cflags + ccflags,
-
-  CPPPATH=cpppath + [
+  ],
+  CFLAGS=["-std=gnu11"],
+  CXXFLAGS=["-std=c++1z"],
+  CPPPATH=[
     "#",
+    "#msgq",
+    "#third_party",
+    "#third_party/json11",
+    "#third_party/linux/include",
     "#third_party/acados/include",
     "#third_party/acados/include/blasfeo/include",
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
-    "#third_party/libyuv/include",
-    "#third_party/json11",
-    "#third_party/linux/include",
-    "#third_party/snpe/include",
-    "#third_party",
-    "#msgq",
+    [x.INCLUDE_DIR for x in pkgs],
   ],
-
-  CC='clang',
-  CXX='clang++',
-  LINKFLAGS=ldflags,
-
-  RPATH=rpath,
-
-  CFLAGS=["-std=gnu11"] + cflags,
-  CXXFLAGS=["-std=c++1z"] + cxxflags,
-  LIBPATH=libpath + [
+  LIBPATH=[
+    "#common",
     "#msgq_repo",
     "#third_party",
     "#selfdrive/pandad",
-    "#common",
     "#rednose/helpers",
+    f"#third_party/acados/{arch}/lib",
+    [x.LIB_DIR for x in pkgs],
   ],
+  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
+if arch != "larch64":
+  env['_LIBFLAGS'] = _libflags
 
-if arch == "Darwin":
-  # RPATH is not supported on macOS, instead use the linker flags
-  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
-  env["LINKFLAGS"] += darwin_rpath_link_flags
+# Arch-specific flags and paths
+if arch == "larch64":
+  env["CC"] = "clang"
+  env["CXX"] = "clang++"
+  env.Append(LIBPATH=[
+    "/usr/lib/aarch64-linux-gnu",
+  ])
+  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57"]
+  env.Append(CCFLAGS=arch_flags)
+  env.Append(CXXFLAGS=arch_flags)
+elif arch == "Darwin":
+  env.Append(LIBPATH=[
+    "/System/Library/Frameworks/OpenGL.framework/Libraries",
+  ])
+  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
 
-if GetOption('compile_db'):
-  env.CompilationDatabase('compile_commands.json')
+_extra_cc = shlex.split(GetOption('ccflags') or '')
+if _extra_cc:
+  env.Append(CCFLAGS=_extra_cc)
 
-# Setup cache dir
-cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+# no --as-needed on mac linker
+if arch != "Darwin":
+  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
 
+# Shorter build output: show brief descriptions instead of full commands.
+# Full command lines are still printed on failure by scons.
+if not GetOption('verbose'):
+  for action, short in (
+    ("CC",     "CC"),
+    ("CXX",    "CXX"),
+    ("LINK",   "LINK"),
+    ("SHCC",   "CC"),
+    ("SHCXX",  "CXX"),
+    ("SHLINK", "LINK"),
+    ("AR",     "AR"),
+    ("RANLIB", "RANLIB"),
+    ("AS",     "AS"),
+  ):
+    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
+
+# progress output
 node_interval = 5
 node_count = 0
 def progress_function(node):
   global node_count
   node_count += node_interval
   sys.stderr.write("progress: %d\n" % node_count)
-
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-# Cython build environment
-py_include = sysconfig.get_paths()['include']
+# ********** Cython build environment **********
 envCython = env.Clone()
-envCython["CPPPATH"] += [py_include, np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
+  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-# Qt build environment
-qt_env = env.Clone()
-qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
+Export('env', 'arch')
 
-qt_libs = []
-if arch == "Darwin":
-  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
-  qt_dirs = [
-    os.path.join(qt_env['QTDIR'], "include"),
-  ]
-  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
-  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
-  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
-  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
-else:
-  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
-  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
+# Setup cache dir
+cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
 
-  qt_env['QTDIR'] = qt_install_prefix
-  qt_dirs = [
-    f"{qt_install_headers}",
-  ]
-
-  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
-  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
-  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
-  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
-
-  qt_libs = [f"Qt5{m}" for m in qt_modules]
-  if arch == "larch64":
-    qt_libs += ["GLESv2", "wayland-client"]
-    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
-  elif arch != "Darwin":
-    qt_libs += ["GL"]
-qt_env['QT3DIR'] = qt_env['QTDIR']
-
-# compatibility for older SCons versions
-try:
-  qt_env.Tool('qt3')
-except SCons.Errors.UserError:
-  qt_env.Tool('qt')
-
-qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
-qt_flags = [
-  "-D_REENTRANT",
-  "-DQT_NO_DEBUG",
-  "-DQT_WIDGETS_LIB",
-  "-DQT_GUI_LIB",
-  "-DQT_CORE_LIB",
-  "-DQT_MESSAGELOGCONTEXT",
-]
-qt_env['CXXFLAGS'] += qt_flags
-qt_env['LIBPATH'] += ['#selfdrive/ui', ]
-qt_env['LIBS'] = qt_libs
-
-if GetOption("clazy"):
-  checks = [
-    "level0",
-    "level1",
-    "no-range-loop",
-    "no-non-pod-global-static",
-  ]
-  qt_env['CXX'] = 'clazy'
-  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
-  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
-
-Export('env', 'qt_env', 'arch', 'real_arch')
+# ********** start building stuff **********
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common', '_gpucommon')
-
+Import('_common')
 common = [_common, 'json11', 'zmq']
-gpucommon = [_gpucommon]
-
-Export('common', 'gpucommon')
+Export('common')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
 # Enable swaglog include in submodules
 env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
-SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
 
 SConscript(['cereal/SConscript'])
 
 Import('socketmaster', 'msgq')
-messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
+messaging = [socketmaster, msgq, 'capnp', 'kj',]
 Export('messaging')
 
 
@@ -366,16 +235,8 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
-  'system/ui/SConscript',
-  'system/proclogd/SConscript',
-  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
-if arch != "Darwin":
-  SConscript([
-    'system/sensord/SConscript',
-    'system/logcatd/SConscript',
-  ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
@@ -383,13 +244,23 @@ if arch == "larch64":
 # Build openpilot
 SConscript(['third_party/SConscript'])
 
-SConscript(['selfdrive/SConscript'])
+# Build selfdrive
+SConscript([
+  'selfdrive/pandad/SConscript',
+  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
+  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
+  'selfdrive/locationd/SConscript',
+  'selfdrive/modeld/SConscript',
+  'selfdrive/ui/SConscript',
+])
 
-if Dir('#tools/cabana/').exists() and GetOption('extras'):
-  SConscript(['tools/replay/SConscript'])
-  if arch != "larch64":
-    SConscript(['tools/cabana/SConscript'])
+# Build tools
+if arch != "larch64":
+  SConscript([
+    'tools/replay/SConscript',
+    'tools/cabana/SConscript',
+    'tools/jotpluggler/SConscript',
+  ])
 
-external_sconscript = GetOption('external_sconscript')
-if external_sconscript:
-  SConscript([external_sconscript])
+
+env.CompilationDatabase('compile_commands.json')

cereal/README.md

@@ -29,6 +29,50 @@ then means breaking backwards-compatibility with all old logs of your fork. So w
 [custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
+An example of compatible changes:
+```diff
+diff --git a/cereal/custom.capnp b/cereal/custom.capnp
+index 3348e859e..3365c7b98 100644
+--- a/cereal/custom.capnp
++++ b/cereal/custom.capnp
+@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
+ # DO rename the structs
+ # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
+-struct CustomReserved0 @0x81c2f05a394cf4af {
++struct SteeringInfo @0x81c2f05a394cf4af {
++  active @0 :Bool;
++  steeringAngleDeg @1 :Float32;
++  steeringRateDeg @2 :Float32;
++  steeringAccelDeg @3 :Float32;
+ }
+
+ struct CustomReserved1 @0xaedffd8f31e7b55d {
+diff --git a/cereal/log.capnp b/cereal/log.capnp
+index 1209f3fd9..b189f58b6 100644
+--- a/cereal/log.capnp
++++ b/cereal/log.capnp
+@@ -2558,14 +2558,14 @@ struct Event {
+
+     # DO change the name of the field
+     # DON'T change anything after the "@"
+-    customReservedRawData0 @124 :Data;
++    rawCanData @124 :Data;
+     customReservedRawData1 @125 :Data;
+     customReservedRawData2 @126 :Data;
+
+     # DO change the name of the field and struct
+     # DON'T change the ID (e.g. @107)
+     # DON'T change which struct it points to
+-    customReserved0 @107 :Custom.CustomReserved0;
++    steeringInfo @107 :Custom.SteeringInfo;
+     customReserved1 @108 :Custom.CustomReserved1;
+     customReserved2 @109 :Custom.CustomReserved2;
+     customReserved3 @110 :Custom.CustomReserved3;
+```
+
+---
+
 Example
 ---
 ```python

cereal/SConscript

@@ -13,7 +13,7 @@ cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
 socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 

cereal/__init__.py

@@ -1,9 +1,11 @@
 import os
 import capnp
+from importlib.resources import as_file, files
 
-CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
 capnp.remove_import_hook()
 
-log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
-car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
-custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))
+with as_file(files("cereal")) as fspath:
+  CEREAL_PATH = fspath.as_posix()
+  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/custom.capnp

@@ -7,7 +7,9 @@ $Cxx.namespace("cereal");
 # These structs are guaranteed to remain reserved and empty in mainline
 # cereal, so use these if you want custom events in your fork.
 
-# you can rename the struct, but don't change the identifier
+# DO rename the structs
+# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
 struct CustomReserved0 @0x81c2f05a394cf4af {
 }
 
@@ -37,3 +39,33 @@ struct CustomReserved8 @0xf416ec09499d9d19 {
 
 struct CustomReserved9 @0xa1680744031fdb2d {
 }
+
+struct CustomReserved10 @0xcb9fd56c7057593a {
+}
+
+struct CustomReserved11 @0xc2243c65e0340384 {
+}
+
+struct CustomReserved12 @0x9ccdc8676701b412 {
+}
+
+struct CustomReserved13 @0xcd96dafb67a082d0 {
+}
+
+struct CustomReserved14 @0xb057204d7deadf3f {
+}
+
+struct CustomReserved15 @0xbd443b539493bc68 {
+}
+
+struct CustomReserved16 @0xfc6241ed8877b611 {
+}
+
+struct CustomReserved17 @0xa30662f84033036c {
+}
+
+struct CustomReserved18 @0xc86a3d38d13eb3ef {
+}
+
+struct CustomReserved19 @0xa4f1eb3323f5f582 {
+}

cereal/log.capnp

@@ -48,6 +48,7 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     preEnableStandstill @12;  # added during pre-enable state with brake
     gasPressedOverride @13;  # added when user is pressing gas with no disengage on gas
     steerOverride @14;
+    steerDisengage @94;  # exits active state
     cruiseDisabled @15;
     speedTooLow @16;
     outOfSpace @17;
@@ -59,6 +60,7 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     pcmEnable @23;
     pcmDisable @24;
     radarFault @25;
+    radarTempUnavailable @93;
     brakeHold @26;
     parkBrake @27;
     manualRestart @28;
@@ -85,6 +87,8 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     laneChange @50;
     lowMemory @51;
     stockAeb @52;
+    stockLkas @98;
+    lateralManeuver @99;
     ldw @53;
     carUnrecognized @54;
     invalidLkasSetting @55;
@@ -125,6 +129,9 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     espActive @90;
     personalityChanged @91;
     aeb @92;
+    userBookmark @95;
+    excessiveActuation @96;
+    audioFeedback @97;
 
     soundsUnavailableDEPRECATED @47;
   }
@@ -151,6 +158,10 @@ struct InitData {
   gitBranch @11 :Text;
   gitRemote @13 :Text;
 
+  # this is source commit for prebuilt branches
+  gitSrcCommit @23 :Text;
+  gitSrcCommitDate @24 :Text;
+
   androidProperties @16 :Map(Text, Text);
 
   pandaInfo @8 :PandaInfo;
@@ -409,6 +420,7 @@ struct GpsLocationData {
   speedAccuracy @12 :Float32;
 
   hasFix @13 :Bool;
+  satelliteCount @14 :Int8;
 
   enum SensorSource {
     android @0;
@@ -482,13 +494,14 @@ struct DeviceState @0xa4d8b5af2aa492eb {
   # device thermals
   cpuTempC @26 :List(Float32);
   gpuTempC @27 :List(Float32);
+  dspTempC @49 :Float32;
   memoryTempC @28 :Float32;
-  nvmeTempC @35 :List(Float32);
   modemTempC @36 :List(Float32);
   pmicTempC @39 :List(Float32);
   intakeTempC @46 :Float32;
   exhaustTempC @47 :Float32;
-  caseTempC @48 :Float32;
+  gnssTempC @48 :Float32;
+  bottomSocTempC @50 :Float32;
   maxTempC @44 :Float32;  # max of other temps, used to control fan
   thermalZones @38 :List(ThermalZone);
   thermalStatus @14 :ThermalStatus;
@@ -559,6 +572,7 @@ struct DeviceState @0xa4d8b5af2aa492eb {
   chargingDisabledDEPRECATED @18 :Bool;
   usbOnlineDEPRECATED @12 :Bool;
   ambientTempCDEPRECATED @30 :Float32;
+  nvmeTempCDEPRECATED @35 :List(Float32);
 }
 
 struct PandaState @0xa7649e2575e4591e {
@@ -574,13 +588,13 @@ struct PandaState @0xa7649e2575e4591e {
   heartbeatLost @22 :Bool;
   interruptLoad @25 :Float32;
   fanPower @28 :UInt8;
-  fanStallCount @34 :UInt8;
 
-  spiChecksumErrorCount @33 :UInt16;
+  spiErrorCount @33 :UInt16;
 
   harnessStatus @21 :HarnessStatus;
   sbu1Voltage @35 :Float32;
   sbu2Voltage @36 :Float32;
+  soundOutputLevel @37 :UInt16;
 
   # can health
   canState0 @29 :PandaCanState;
@@ -703,6 +717,7 @@ struct PandaState @0xa7649e2575e4591e {
   usbPowerModeDEPRECATED @12 :PeripheralState.UsbPowerModeDEPRECATED;
   safetyParamDEPRECATED @20 :Int16;
   safetyParam2DEPRECATED @26 :UInt32;
+  fanStallCountDEPRECATED @34 :UInt8;
 }
 
 struct PeripheralState {
@@ -723,7 +738,7 @@ struct PeripheralState {
 struct RadarState @0x9a185389d6fdd05f {
   mdMonoTime @6 :UInt64;
   carStateMonoTime @11 :UInt64;
-  radarErrors @12 :List(Car.RadarData.Error);
+  radarErrors @13 :Car.RadarData.Error;
 
   leadOne @3 :LeadData;
   leadTwo @4 :LeadData;
@@ -757,6 +772,7 @@ struct RadarState @0x9a185389d6fdd05f {
   calPercDEPRECATED @9 :Int8;
   canMonoTimesDEPRECATED @10 :List(UInt64);
   cumLagMsDEPRECATED @5 :Float32;
+  radarErrorsDEPRECATED @12 :List(Car.RadarData.ErrorDEPRECATED);
 }
 
 struct LiveCalibrationData {
@@ -906,6 +922,8 @@ struct ControlsState @0x97ff69c53601abf1 {
     saturated @7 :Bool;
     actualLateralAccel @9 :Float32;
     desiredLateralAccel @10 :Float32;
+    desiredLateralJerk @11 :Float32;
+    version @12 :Int32;
    }
 
   struct LateralLQRState {
@@ -1073,7 +1091,7 @@ struct ModelDataV2 {
   confidence @23: ConfidenceClass;
 
   # Model perceived motion
-  temporalPose @21 :Pose;
+  temporalPoseDEPRECATED @21 :Pose;
 
   # e2e lateral planner
   action @26: Action;
@@ -1167,6 +1185,8 @@ struct ModelDataV2 {
 
   struct Action {
     desiredCurvature @0 :Float32;
+    desiredAcceleration @1 :Float32;
+    shouldStop @2 :Bool;
   }
 }
 
@@ -1222,6 +1242,10 @@ struct DriverAssistance {
   # FCW, AEB, etc. will go here
 }
 
+struct LateralManeuverPlan {
+  desiredCurvature @0 :Float32;  # 1/m
+}
+
 struct LongitudinalPlan @0xe00b5b3eba12876c {
   modelMonoTime @9 :UInt64;
   hasLead @7 :Bool;
@@ -1407,6 +1431,8 @@ struct LivePose {
   posenetOK @5 :Bool = false;
   sensorsOK @6 :Bool = false;
 
+  timestamp @8 :UInt64;
+
   debugFilterState @7 :FilterState;
 
   struct XYZMeasurement {
@@ -1461,6 +1487,11 @@ struct ProcLog {
 
     cmdline @15 :List(Text);
     exe @16 :Text;
+
+    # from /proc/<pid>/smaps_rollup (proportional/private memory)
+    memPss @17 :UInt64;        # Pss — shared pages split by mapper count
+    memPssAnon @18 :UInt64;    # Pss_Anon — private anonymous (heap, stack)
+    memPssShmem @19 :UInt64;   # Pss_Shmem — proportional MSGQ/tmpfs share
   }
 
   struct CPUTimes {
@@ -1579,6 +1610,10 @@ struct UbloxGnss {
       svId @0 :UInt8;
       gnssId @1 :UInt8;
       flagsBitfield @2 :UInt32;
+      cno @3 :UInt8;
+      elevationDeg @4 :Int8;
+      azimuthDeg @5 :Int16;
+      pseudorangeResidual @6 :Float32;
     }
   }
 
@@ -2128,13 +2163,10 @@ struct Joystick {
 struct DriverStateV2 {
   frameId @0 :UInt32;
   modelExecutionTime @1 :Float32;
-  dspExecutionTimeDEPRECATED @2 :Float32;
   gpuExecutionTime @8 :Float32;
   rawPredictions @3 :Data;
 
-  poorVisionProb @4 :Float32;
   wheelOnRightProb @5 :Float32;
-
   leftDriverData @6 :DriverData;
   rightDriverData @7 :DriverData;
 
@@ -2144,15 +2176,21 @@ struct DriverStateV2 {
     facePosition @2 :List(Float32);
     facePositionStd @3 :List(Float32);
     faceProb @4 :Float32;
-    leftEyeProb @5 :Float32;
-    rightEyeProb @6 :Float32;
-    leftBlinkProb @7 :Float32;
-    rightBlinkProb @8 :Float32;
-    sunglassesProb @9 :Float32;
-    occludedProb @10 :Float32;
-    readyProb @11 :List(Float32);
-    notReadyProb @12 :List(Float32);
+    eyesVisibleProb @14 :Float32;
+    eyesClosedProb @15 :Float32;
+    phoneProb @13 :Float32;
+    leftEyeProbDEPRECATED @5 :Float32;
+    rightEyeProbDEPRECATED @6 :Float32;
+    leftBlinkProbDEPRECATED @7 :Float32;
+    rightBlinkProbDEPRECATED @8 :Float32;
+    sunglassesProbDEPRECATED @9 :Float32;
+    notReadyProbDEPRECATED @12 :List(Float32);
+    occludedProbDEPRECATED @10 :Float32;
+    readyProbDEPRECATED @11 :List(Float32);
   }
+
+  dspExecutionTimeDEPRECATED @2 :Float32;
+  poorVisionProbDEPRECATED @4 :Float32;
 }
 
 struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
@@ -2204,7 +2242,10 @@ struct DriverMonitoringState @0xb83cda094a1da284 {
   hiStdCount @14 :UInt32;
   isActiveMode @16 :Bool;
   isRHD @4 :Bool;
+  uncertainCount @19 :UInt32;
 
+  phoneProbOffsetDEPRECATED @20 :Float32;
+  phoneProbValidCountDEPRECATED @21 :UInt32;
   isPreviewDEPRECATED @15 :Bool;
   rhdCheckedDEPRECATED @5 :Bool;
   eventsDEPRECATED @0 :List(Car.OnroadEventDEPRECATED);
@@ -2237,6 +2278,11 @@ struct LiveParametersData {
   roll @14 :Float32;
   debugFilterState @16 :FilterState;
 
+  angleOffsetValid @17 :Bool = true;
+  angleOffsetAverageValid @18 :Bool = true;
+  steerRatioValid @19 :Bool = true;
+  stiffnessFactorValid @20 :Bool = true;
+
   yawRateDEPRECATED @7 :Float32;
   filterStateDEPRECATED @15 :LiveLocationKalman.Measurement;
 
@@ -2260,6 +2306,24 @@ struct LiveTorqueParametersData {
   points @10 :List(List(Float32));
   version @11 :Int32;
   useParams @12 :Bool;
+  calPerc @13 :Int8;
+}
+
+struct LiveDelayData {
+  lateralDelay @0 :Float32;
+  validBlocks @1 :Int32;
+  status @2 :Status;
+
+  lateralDelayEstimate @3 :Float32;
+  lateralDelayEstimateStd @5 :Float32;
+  points @4 :List(Float32);
+  calPerc @6 :Int8;
+
+  enum Status {
+    unestimated @0;
+    estimated @1;
+    invalid @2;
+  }
 }
 
 struct LiveMapDataDEPRECATED {
@@ -2428,16 +2492,35 @@ struct DebugAlert {
   alertText2 @1 :Text;
 }
 
-struct UserFlag {
+struct UserBookmark @0xfe346a9de48d9b50 {
 }
 
-struct Microphone {
+struct SoundPressure @0xdc24138990726023 {
   soundPressure @0 :Float32;
 
   # uncalibrated, A-weighted
   soundPressureWeighted @3 :Float32;
   soundPressureWeightedDb @1 :Float32;
-  filteredSoundPressureWeightedDb @2 :Float32;
+
+  filteredSoundPressureWeightedDbDEPRECATED @2 :Float32;
+}
+
+struct AudioData {
+  data @0 :Data;
+  sampleRate @1 :UInt32;
+}
+
+struct AudioFeedback {
+  audio @0 :AudioData;
+  blockNum @1 :UInt16;
+}
+
+struct Touch {
+  sec @0 :Int64;
+  usec @1 :Int64;
+  type @2 :UInt8;
+  code @3 :Int32;
+  value @4 :Int32;
 }
 
 struct Event {
@@ -2458,13 +2541,10 @@ struct Event {
     controlsState @7 :ControlsState;
     selfdriveState @130 :SelfdriveState;
     gyroscope @99 :SensorEventData;
-    gyroscope2 @100 :SensorEventData;
     accelerometer @98 :SensorEventData;
-    accelerometer2 @101 :SensorEventData;
     magnetometer @95 :SensorEventData;
     lightSensor @96 :SensorEventData;
     temperatureSensor @97 :SensorEventData;
-    temperatureSensor2 @123 :SensorEventData;
     pandaStates @81 :List(PandaState);
     peripheralState @80 :PeripheralState;
     radarState @13 :RadarState;
@@ -2484,6 +2564,7 @@ struct Event {
     gnssMeasurements @91 :GnssMeasurements;
     liveParameters @61 :LiveParametersData;
     liveTorqueParameters @94 :LiveTorqueParametersData;
+    liveDelay @146 : LiveDelayData;
     cameraOdometry @63 :CameraOdometry;
     thumbnail @66: Thumbnail;
     onroadEvents @134: List(OnroadEvent);
@@ -2508,7 +2589,8 @@ struct Event {
     livestreamDriverEncodeIdx @119 :EncodeIndex;
 
     # microphone data
-    microphone @103 :Microphone;
+    soundPressure @103 :SoundPressure;
+    rawAudioData @147 :AudioData;
 
     # systems stuff
     androidLog @20 :AndroidLogEntry;
@@ -2520,6 +2602,9 @@ struct Event {
     logMessage @18 :Text;
     errorLogMessage @85 :Text;
 
+    # touch frame
+    touch @135 :List(Touch);
+
     # navigation
     navInstruction @82 :NavInstruction;
     navRoute @83 :NavRoute;
@@ -2527,9 +2612,15 @@ struct Event {
     mapRenderState @105: MapRenderState;
 
     # UI services
-    userFlag @93 :UserFlag;
     uiDebug @102 :UIDebug;
 
+    # driving feedback
+    userBookmark @93 :UserBookmark;
+    bookmarkButton @148 :UserBookmark;
+    audioFeedback @149 :AudioFeedback;
+
+    lateralManeuverPlan @150 :LateralManeuverPlan;
+
     # *********** debug ***********
     testJoystick @52 :Joystick;
     roadEncodeData @86 :EncodeData;
@@ -2542,11 +2633,17 @@ struct Event {
     livestreamWideRoadEncodeData @121 :EncodeData;
     livestreamDriverEncodeData @122 :EncodeData;
 
+    # *********** Custom: reserved for forks ***********
+
+    # DO change the name of the field
+    # DON'T change anything after the "@"
     customReservedRawData0 @124 :Data;
     customReservedRawData1 @125 :Data;
     customReservedRawData2 @126 :Data;
 
-    # *********** Custom: reserved for forks ***********
+    # DO change the name of the field and struct
+    # DON'T change the ID (e.g. @107)
+    # DON'T change which struct it points to
     customReserved0 @107 :Custom.CustomReserved0;
     customReserved1 @108 :Custom.CustomReserved1;
     customReserved2 @109 :Custom.CustomReserved2;
@@ -2557,6 +2654,16 @@ struct Event {
     customReserved7 @114 :Custom.CustomReserved7;
     customReserved8 @115 :Custom.CustomReserved8;
     customReserved9 @116 :Custom.CustomReserved9;
+    customReserved10 @136 :Custom.CustomReserved10;
+    customReserved11 @137 :Custom.CustomReserved11;
+    customReserved12 @138 :Custom.CustomReserved12;
+    customReserved13 @139 :Custom.CustomReserved13;
+    customReserved14 @140 :Custom.CustomReserved14;
+    customReserved15 @141 :Custom.CustomReserved15;
+    customReserved16 @142 :Custom.CustomReserved16;
+    customReserved17 @143 :Custom.CustomReserved17;
+    customReserved18 @144 :Custom.CustomReserved18;
+    customReserved19 @145 :Custom.CustomReserved19;
 
     # *********** legacy + deprecated ***********
     model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated
@@ -2602,5 +2709,8 @@ struct Event {
     liveLocationKalmanDEPRECATED @72 :LiveLocationKalman;
     liveTracksDEPRECATED @16 :List(LiveTracksDEPRECATED);
     onroadEventsDEPRECATED @68: List(Car.OnroadEventDEPRECATED);
+    gyroscope2DEPRECATED @100 :SensorEventData;
+    accelerometer2DEPRECATED @101 :SensorEventData;
+    temperatureSensor2DEPRECATED @123 :SensorEventData;
   }
 }

cereal/maptile.capnp

@@ -1,49 +0,0 @@
-using Cxx = import "./include/c++.capnp";
-$Cxx.namespace("cereal");
-
-@0xa086df597ef5d7a0;
-
-# Geometry
-struct Point {
-  x @0: Float64;
-  y @1: Float64;
-  z @2: Float64;
-}
-
-struct PolyLine {
-  points @0: List(Point);
-}
-
-# Map features
-struct Lane {
-  id @0 :Text;
-
-  leftBoundary @1 :LaneBoundary;
-  rightBoundary @2 :LaneBoundary;
-
-  leftAdjacentId @3 :Text;
-  rightAdjacentId @4 :Text;
-
-  inboundIds @5 :List(Text);
-  outboundIds @6 :List(Text);
-
-  struct LaneBoundary {
-    polyLine @0 :PolyLine;
-    startHeading @1 :Float32; # WRT north
-  }
-}
-
-# Map tiles
-struct TileSummary {
-  version @0 :Text;
-  updatedAt @1 :UInt64; # Millis since epoch
-
-  level @2 :UInt8;
-  x @3 :UInt16;
-  y @4 :UInt16;
-}
-
-struct MapTile {
-  summary @0 :TileSummary;
-  lanes @1 :List(Lane);
-}

cereal/messaging/__init__.py

@@ -1,23 +1,35 @@
 # must be built with scons
-from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-from msgq.ipc_pyx import MultiplePublishersError, IpcError
-from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
+from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
+                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
 import msgq
-
 import os
 import capnp
 import time
 
-from typing import Optional, List, Union, Dict, Deque
-from collections import deque
+from typing import Optional, List, Union, Dict
 
 from cereal import log
 from cereal.services import SERVICE_LIST
+from openpilot.common.utils import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
+def pub_sock(endpoint: str) -> PubSocket:
+  service = SERVICE_LIST.get(endpoint)
+  segment_size = service.queue_size if service else 0
+  return msgq.pub_sock(endpoint, segment_size)
+
+
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
+  service = SERVICE_LIST.get(endpoint)
+  segment_size = service.queue_size if service else 0
+  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
+                       timeout=timeout, segment_size=segment_size)
+
+
 def reset_context():
   msgq.context = Context()
 
@@ -108,10 +120,8 @@ class FrequencyTracker:
 
     self.min_freq = min_freq * 0.8
     self.max_freq = max_freq * 1.2
-    self.recv_dts: Deque[float] = deque(maxlen=int(10 * freq))
-    self.recv_dts_sum = 0.0
-    self.recent_recv_dts: Deque[float] = deque(maxlen=int(freq))
-    self.recent_recv_dts_sum = 0.0
+    self.avg_dt = MovingAverage(int(10 * freq))
+    self.recent_avg_dt = MovingAverage(int(freq))
     self.prev_time = 0.0
 
   def record_recv_time(self, cur_time: float) -> None:
@@ -119,28 +129,21 @@ class FrequencyTracker:
     if self.prev_time > 1e-5:
       dt = cur_time - self.prev_time
 
-      if len(self.recv_dts) == self.recv_dts.maxlen:
-        self.recv_dts_sum -= self.recv_dts[0]
-      self.recv_dts.append(dt)
-      self.recv_dts_sum += dt
-
-      if len(self.recent_recv_dts) == self.recent_recv_dts.maxlen:
-        self.recent_recv_dts_sum -= self.recent_recv_dts[0]
-      self.recent_recv_dts.append(dt)
-      self.recent_recv_dts_sum += dt
+      self.avg_dt.add_value(dt)
+      self.recent_avg_dt.add_value(dt)
 
     self.prev_time = cur_time
 
   @property
   def valid(self) -> bool:
-    if not self.recv_dts:
+    if self.avg_dt.count == 0:
       return False
 
-    avg_freq = len(self.recv_dts) / self.recv_dts_sum
+    avg_freq = 1.0 / self.avg_dt.get_average()
     if self.min_freq <= avg_freq <= self.max_freq:
       return True
 
-    avg_freq_recent = len(self.recent_recv_dts) / self.recent_recv_dts_sum
+    avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
     return self.min_freq <= avg_freq_recent <= self.max_freq
 
 
@@ -154,12 +157,16 @@ class SubMaster:
     self.updated = {s: False for s in services}
     self.recv_time = {s: 0. for s in services}
     self.recv_frame = {s: 0 for s in services}
-    self.alive = {s: False for s in services}
-    self.freq_ok = {s: False for s in services}
     self.sock = {}
     self.data = {}
-    self.valid = {}
-    self.logMonoTime = {}
+    self.logMonoTime = {s: 0 for s in services}
+
+    # zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
+    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
+    self.static_freq_services = set(s for s in services if not on_demand[s])
+    self.alive = {s: on_demand[s] for s in services}
+    self.freq_ok = {s: on_demand[s] for s in services}
+    self.valid = {s: on_demand[s] for s in services}
 
     self.freq_tracker: Dict[str, FrequencyTracker] = {}
     self.poller = Poller()
@@ -186,8 +193,6 @@ class SubMaster:
         data = new_message(s, 0) # lists
 
       self.data[s] = getattr(data.as_reader(), s)
-      self.logMonoTime[s] = 0
-      self.valid[s] = False
       self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
 
   def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
@@ -224,14 +229,10 @@ class SubMaster:
       self.logMonoTime[s] = msg.logMonoTime
       self.valid[s] = msg.valid
 
-    for s in self.services:
-      if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation:
-        # alive if delay is within 10x the expected frequency
-        self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency)
-        self.freq_ok[s] = self.freq_tracker[s].valid
-      else:
-        self.freq_ok[s] = True
-        self.alive[s] = self.seen[s] if self.simulation else True
+    for s in self.static_freq_services:
+      # alive if delay is within 10x the expected frequency; checks relaxed in simulator
+      self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
+      self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
 
   def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)

cereal/messaging/bridge.cc

@@ -6,12 +6,12 @@
 
 ExitHandler do_exit;
 
-static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
+static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
   std::vector<std::string> service_list;
   for (const auto& it : services) {
     std::string name = it.second.name;
     bool in_whitelist = whitelist_str.find(name) != std::string::npos;
-    if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
+    if (zmq_to_msgq && !in_whitelist) {
       continue;
     }
     service_list.push_back(name);
@@ -25,15 +25,16 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<ZMQPoller>();
-  auto pub_context = std::make_unique<MSGQContext>();
-  auto sub_context = std::make_unique<ZMQContext>();
-  std::map<SubSocket *, PubSocket *> sub2pub;
+  auto poller = std::make_unique<BridgeZmqPoller>();
+  auto pub_context = std::make_unique<Context>();
+  auto sub_context = std::make_unique<BridgeZmqContext>();
+  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new MSGQPubSocket();
-    auto sub_sock = new ZMQSubSocket();
-    pub_sock->connect(pub_context.get(), endpoint);
+    auto pub_sock = new PubSocket();
+    auto sub_sock = new BridgeZmqSubSocket();
+    size_t queue_size = services.at(endpoint).queue_size;
+    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/bridge_zmq.cc

@@ -0,0 +1,170 @@
+#include "cereal/messaging/bridge_zmq.h"
+
+#include <cassert>
+#include <cstring>
+#include <unistd.h>
+
+static size_t fnv1a_hash(const std::string &str) {
+  const size_t fnv_prime = 0x100000001b3;
+  size_t hash_value = 0xcbf29ce484222325;
+  for (char c : str) {
+    hash_value ^= (unsigned char)c;
+    hash_value *= fnv_prime;
+  }
+  return hash_value;
+}
+
+// FIXME: This is a hack to get the port number from the socket name, might have collisions.
+static int get_port(std::string endpoint) {
+  size_t hash_value = fnv1a_hash(endpoint);
+  int start_port = 8023;
+  int max_port = 65535;
+  return start_port + (hash_value % (max_port - start_port));
+}
+
+BridgeZmqContext::BridgeZmqContext() {
+  context = zmq_ctx_new();
+}
+
+BridgeZmqContext::~BridgeZmqContext() {
+  if (context != nullptr) {
+    zmq_ctx_term(context);
+  }
+}
+
+void BridgeZmqMessage::init(size_t sz) {
+  size = sz;
+  data = new char[size];
+}
+
+void BridgeZmqMessage::init(char *d, size_t sz) {
+  size = sz;
+  data = new char[size];
+  memcpy(data, d, size);
+}
+
+void BridgeZmqMessage::close() {
+  if (size > 0) {
+    delete[] data;
+  }
+  data = nullptr;
+  size = 0;
+}
+
+BridgeZmqMessage::~BridgeZmqMessage() {
+  close();
+}
+
+int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
+  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
+  if (sock == nullptr) {
+    return -1;
+  }
+
+  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
+
+  if (conflate) {
+    int arg = 1;
+    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
+  }
+
+  int reconnect_ivl = 500;
+  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
+
+  full_endpoint = "tcp://" + address + ":";
+  if (check_endpoint) {
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
+
+  return zmq_connect(sock, full_endpoint.c_str());
+}
+
+void BridgeZmqSubSocket::setTimeout(int timeout) {
+  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
+}
+
+Message *BridgeZmqSubSocket::receive(bool non_blocking) {
+  zmq_msg_t msg;
+  assert(zmq_msg_init(&msg) == 0);
+
+  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
+  int rc = zmq_msg_recv(&msg, sock, flags);
+
+  Message *ret = nullptr;
+  if (rc >= 0) {
+    ret = new BridgeZmqMessage;
+    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
+  }
+
+  zmq_msg_close(&msg);
+  return ret;
+}
+
+BridgeZmqSubSocket::~BridgeZmqSubSocket() {
+  if (sock != nullptr) {
+    zmq_close(sock);
+  }
+}
+
+int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
+  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
+  if (sock == nullptr) {
+    return -1;
+  }
+
+  full_endpoint = "tcp://*:";
+  if (check_endpoint) {
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
+
+  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
+  pid = getpid();
+
+  return zmq_bind(sock, full_endpoint.c_str());
+}
+
+int BridgeZmqPubSocket::sendMessage(Message *message) {
+  assert(pid == getpid());
+  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
+}
+
+int BridgeZmqPubSocket::send(char *data, size_t size) {
+  assert(pid == getpid());
+  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
+}
+
+BridgeZmqPubSocket::~BridgeZmqPubSocket() {
+  if (sock != nullptr) {
+    zmq_close(sock);
+  }
+}
+
+void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
+  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
+  polls[num_polls].socket = socket->getRawSocket();
+  polls[num_polls].events = ZMQ_POLLIN;
+
+  sockets.push_back(socket);
+  num_polls++;
+}
+
+std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
+  std::vector<BridgeZmqSubSocket *> ret;
+
+  int rc = zmq_poll(polls, num_polls, timeout);
+  if (rc < 0) {
+    return ret;
+  }
+
+  for (size_t i = 0; i < num_polls; i++) {
+    if (polls[i].revents) {
+      ret.push_back(sockets[i]);
+    }
+  }
+
+  return ret;
+}

cereal/messaging/bridge_zmq.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include <cstddef>
+#include <string>
+#include <vector>
+
+#include <zmq.h>
+
+#include "msgq/ipc.h"
+
+class BridgeZmqContext {
+public:
+  BridgeZmqContext();
+  void *getRawContext() { return context; }
+  ~BridgeZmqContext();
+
+private:
+  void *context = nullptr;
+};
+
+class BridgeZmqMessage : public Message {
+public:
+  void init(size_t size);
+  void init(char *data, size_t size);
+  void close();
+  size_t getSize() { return size; }
+  char *getData() { return data; }
+  ~BridgeZmqMessage();
+
+private:
+  char *data = nullptr;
+  size_t size = 0;
+};
+
+class BridgeZmqSubSocket {
+public:
+  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
+  void setTimeout(int timeout);
+  Message *receive(bool non_blocking = false);
+  void *getRawSocket() { return sock; }
+  ~BridgeZmqSubSocket();
+
+private:
+  void *sock = nullptr;
+  std::string full_endpoint;
+};
+
+class BridgeZmqPubSocket {
+public:
+  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
+  int sendMessage(Message *message);
+  int send(char *data, size_t size);
+  void *getRawSocket() { return sock; }
+  ~BridgeZmqPubSocket();
+
+private:
+  void *sock = nullptr;
+  std::string full_endpoint;
+  int pid = -1;
+};
+
+class BridgeZmqPoller {
+public:
+  void registerSocket(BridgeZmqSubSocket *socket);
+  std::vector<BridgeZmqSubSocket *> poll(int timeout);
+
+private:
+  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
+  std::vector<BridgeZmqSubSocket *> sockets;
+  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
+  size_t num_polls = 0;
+};

cereal/messaging/msgq_to_zmq.cc

@@ -2,6 +2,7 @@
 
 #include <cassert>
 
+#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -21,14 +22,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<ZMQContext>();
-  msgq_context = std::make_unique<MSGQContext>();
+  zmq_context = std::make_unique<BridgeZmqContext>();
+  msgq_context = std::make_unique<Context>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
+    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -48,7 +49,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
+        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -71,7 +72,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -108,7 +109,8 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
+            size_t queue_size = services.at(pair.endpoint).queue_size;
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -128,7 +130,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,9 +7,8 @@
 #include <string>
 #include <vector>
 
-#define private public
 #include "msgq/impl_msgq.h"
-#include "msgq/impl_zmq.h"
+#include "cereal/messaging/bridge_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -22,16 +21,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<ZMQPubSocket> pub_sock;
+    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<MSGQContext> msgq_context;
-  std::unique_ptr<ZMQContext> zmq_context;
+  std::unique_ptr<Context> msgq_context;
+  std::unique_ptr<BridgeZmqContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
+  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -33,7 +33,7 @@ MessageContext message_context;
 struct SubMaster::SubMessage {
   std::string name;
   SubSocket *socket = nullptr;
-  int freq = 0;
+  float freq = 0.0f;
   bool updated = false, alive = false, valid = false, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,7 +187,8 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    PubSocket *socket = PubSocket::create(message_context.context(), name);
+    service serv = services.at(std::string(name));
+    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/messaging/tests/test_messaging.py

@@ -5,7 +5,7 @@ import numbers
 import random
 import threading
 import time
-from parameterized import parameterized
+from openpilot.common.parameterized import parameterized
 import pytest
 
 from cereal import log, car
@@ -30,7 +30,7 @@ def zmq_sleep(t=1):
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:
@@ -177,8 +177,8 @@ class TestMessaging:
 
     # wait 5 socket timeouts before sending
     msg = random_carstate()
-    delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
     start_time = time.monotonic()
+    delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
     recvd = messaging.recv_one_retry(sub_sock)
     assert (time.monotonic() - start_time) >= sock_timeout*5
     assert isinstance(recvd, capnp._DynamicStructReader)

cereal/messaging/tests/test_pub_sub_master.py

@@ -6,6 +6,7 @@ import cereal.messaging as messaging
 from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
                                                   random_bytes, random_carstate, assert_carstate, \
                                                   zmq_sleep
+from cereal.services import SERVICE_LIST
 
 
 class TestSubMaster:
@@ -26,7 +27,9 @@ class TestSubMaster:
     sm = messaging.SubMaster(socks)
     assert sm.frame == -1
     assert not any(sm.updated.values())
-    assert not any(sm.alive.values())
+    assert not any(sm.seen.values())
+    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
+    assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
     assert all(t == 0. for t in sm.recv_time.values())
     assert all(f == 0 for f in sm.recv_frame.values())
     assert all(t == 0 for t in sm.logMonoTime.values())
@@ -83,6 +86,7 @@ class TestSubMaster:
         "cameraOdometry": (20, 10),
         "liveCalibration": (4, 4),
         "carParams": (None, None),
+        "userBookmark": (None, None),
       }
 
       for service, (max_freq, min_freq) in checks.items():

cereal/messaging/tests/test_services.py

@@ -1,7 +1,7 @@
 import os
 import tempfile
 from typing import Dict
-from parameterized import parameterized
+from openpilot.common.parameterized import parameterized
 
 import cereal.services as services
 from cereal.services import SERVICE_LIST

cereal/services.py

@@ -1,47 +1,57 @@
 #!/usr/bin/env python3
+from enum import IntEnum
 from typing import Optional
 
 
+# TODO: this should be automatically determined using the capnp schema
+class QueueSize(IntEnum):
+  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
+  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
+  SMALL = 250 * 1024          # 250KB - most services
+
+
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
+               queue_size: QueueSize = QueueSize.SMALL):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
+    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
-  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
-  "accelerometer2": (True, 100., 100),
   "magnetometer": (True, 25.),
   "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
-  "temperatureSensor2": (True, 2., 200),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10),
+  "touch": (True, 20., 1),
+  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
   "selfdriveState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139),
+  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
   "logMessage": (True, 0.),
   "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
+  "liveDelay": (True, 4., 1),
   "androidLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
+  "lateralManeuverPlan": (True, 20.),
   "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15),
+  "procLog": (True, 0.5, 15, QueueSize.BIG),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
@@ -52,7 +62,7 @@ _services: dict[str, tuple] = {
   "livePose": (True, 20., 4),
   "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 10),
-  "thumbnail": (True, 0.2, 1),
+  "thumbnail": (True, 1 / 60., 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
   "roadCameraState": (True, 20., 20),
@@ -63,30 +73,33 @@ _services: dict[str, tuple] = {
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20.),
+  "modelV2": (True, 20., None, QueueSize.BIG),
   "managerState": (True, 2., 1),
   "uploaderState": (True, 0., 1),
   "navInstruction": (True, 1., 10),
   "navRoute": (True, 0.),
   "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
-  "userFlag": (True, 0., 1),
-  "microphone": (True, 10., 10),
+  "userBookmark": (True, 0., 1),
+  "soundPressure": (True, 10., 10),
+  "rawAudioData": (False, 20.),
+  "bookmarkButton": (True, 0., 1),
+  "audioFeedback": (True, 0., 1),
+  "roadEncodeData": (False, 20., None, QueueSize.BIG),
+  "driverEncodeData": (False, 20., None, QueueSize.BIG),
+  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
-  "roadEncodeData": (False, 20.),
-  "driverEncodeData": (False, 20.),
-  "wideRoadEncodeData": (False, 20.),
-  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20.),
-  "livestreamRoadEncodeData": (False, 20.),
-  "livestreamDriverEncodeData": (False, 20.),
+  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
   "customReservedRawData0": (True, 0.),
   "customReservedRawData1": (True, 0.),
   "customReservedRawData2": (True, 0.),
@@ -104,13 +117,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
+  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation)
+    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation, v.queue_size)
   h += "};\n"
 
   h += "#endif\n"

codecov.yml

@@ -1,13 +0,0 @@
-comment: false
-coverage:
-  status:
-    project:
-      default:
-        informational: true
-    patch: off
-
-ignore:
-  - "**/test_*.py"
-  - "selfdrive/test/**"
-  - "system/version.py" # codecov changes depending on if we are in a branch or not
-  - "tools"

common/.gitignore

@@ -1 +0,0 @@
-*.cpp

common/SConscript

@@ -1,25 +1,14 @@
-Import('env', 'envCython', 'arch')
+Import('env', 'envCython')
 
 common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
-  'i2c.cc',
-  'watchdog.cc',
-  'ratekeeper.cc'
+  'ratekeeper.cc',
 ]
 
-if arch != "Darwin":
-  common_libs.append('gpio.cc')
-
 _common = env.Library('common', common_libs, LIBS="json11")
-
-files = [
-  'clutil.cc',
-]
-
-_gpucommon = env.Library('gpucommon', files)
-Export('_common', '_gpucommon')
+Export('_common')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
@@ -29,11 +18,6 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-SConscript([
-  'transformations/SConscript',
-])
-
-Import('transformations_python')
-common_python = [params_python, transformations_python]
+common_python = [params_python]
 
 Export('common_python')

common/api.py

@@ -7,11 +7,15 @@ from openpilot.system.version import get_version
 
 API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 
+# name: jwt signature algorithm
+KEYS = {"id_rsa": "RS256",
+        "id_ecdsa": "ES256"}
+
+
 class Api:
   def __init__(self, dongle_id):
     self.dongle_id = dongle_id
-    with open(Paths.persist_root()+'/comma/id_rsa') as f:
-      self.private_key = f.read()
+    self.jwt_algorithm, self.private_key, _ = get_key_pair()
 
   def get(self, *args, **kwargs):
     return self.request('GET', *args, **kwargs)
@@ -22,7 +26,7 @@ class Api:
   def request(self, method, endpoint, timeout=None, access_token=None, **params):
     return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
 
-  def get_token(self, expiry_hours=1):
+  def get_token(self, payload_extra=None, expiry_hours=1):
     now = datetime.now(UTC).replace(tzinfo=None)
     payload = {
       'identity': self.dongle_id,
@@ -30,17 +34,29 @@ class Api:
       'iat': now,
       'exp': now + timedelta(hours=expiry_hours)
     }
-    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if payload_extra is not None:
+      payload.update(payload_extra)
+    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
     if isinstance(token, bytes):
       token = token.decode('utf8')
     return token
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
   headers = {}
   if access_token is not None:
     headers['Authorization'] = "JWT " + access_token
 
   headers['User-Agent'] = "openpilot-" + get_version()
 
-  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)
+  # TODO: add session to Api
+  req = requests if session is None else session
+  return req.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)
+
+
+def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+  for key in KEYS:
+    if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
+      with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
+        return KEYS[key], private.read(), public.read()
+  return None, None, None

common/clutil.cc

@@ -1,200 +0,0 @@
-#include "common/clutil.h"
-
-#include <cassert>
-#include <iostream>
-#include <memory>
-
-#include "common/util.h"
-#include "common/swaglog.h"
-
-namespace {  // helper functions
-
-template <typename Func, typename Id, typename Name>
-std::string get_info(Func get_info_func, Id id, Name param_name) {
-  size_t size = 0;
-  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
-  std::string info(size, '\0');
-  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
-  return info;
-}
-inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
-inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
-
-void cl_print_info(cl_platform_id platform, cl_device_id device) {
-  size_t work_group_size = 0;
-  cl_device_type device_type = 0;
-  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
-  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
-  const char *type_str = "Other...";
-  switch (device_type) {
-    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
-    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
-    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
-  }
-
-  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
-  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
-  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
-  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
-  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
-  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
-  LOGD("max work group size: %zu", work_group_size);
-  LOGD("type = %d, %s", (int)device_type, type_str);
-}
-
-void cl_print_build_errors(cl_program program, cl_device_id device) {
-  cl_build_status status;
-  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
-  size_t log_size;
-  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
-  std::string log(log_size, '\0');
-  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
-
-  LOGE("build failed; status=%d, log: %s", status, log.c_str());
-}
-
-}  // namespace
-
-cl_device_id cl_get_device_id(cl_device_type device_type) {
-  cl_uint num_platforms = 0;
-  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
-  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
-  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
-
-  for (size_t i = 0; i < num_platforms; ++i) {
-    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
-
-    // Get first device
-    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
-      cl_print_info(platform_ids[i], device_id);
-      return device_id;
-    }
-  }
-  LOGE("No valid openCL platform found");
-  assert(0);
-  return nullptr;
-}
-
-cl_context cl_create_context(cl_device_id device_id) {
-  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
-}
-
-cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
-  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
-}
-
-cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
-  const char *csrc = src.c_str();
-  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
-  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
-    cl_print_build_errors(prg, device_id);
-    assert(0);
-  }
-  return prg;
-}
-
-cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args) {
-  cl_program prg = CL_CHECK_ERR(clCreateProgramWithBinary(ctx, 1, &device_id, &length, &binary, NULL, &err));
-  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
-    cl_print_build_errors(prg, device_id);
-    assert(0);
-  }
-  return prg;
-}
-
-// Given a cl code and return a string representation
-#define CL_ERR_TO_STR(err) case err: return #err
-const char* cl_get_error_string(int err) {
-  switch (err) {
-    CL_ERR_TO_STR(CL_SUCCESS);
-    CL_ERR_TO_STR(CL_DEVICE_NOT_FOUND);
-    CL_ERR_TO_STR(CL_DEVICE_NOT_AVAILABLE);
-    CL_ERR_TO_STR(CL_COMPILER_NOT_AVAILABLE);
-    CL_ERR_TO_STR(CL_MEM_OBJECT_ALLOCATION_FAILURE);
-    CL_ERR_TO_STR(CL_OUT_OF_RESOURCES);
-    CL_ERR_TO_STR(CL_OUT_OF_HOST_MEMORY);
-    CL_ERR_TO_STR(CL_PROFILING_INFO_NOT_AVAILABLE);
-    CL_ERR_TO_STR(CL_MEM_COPY_OVERLAP);
-    CL_ERR_TO_STR(CL_IMAGE_FORMAT_MISMATCH);
-    CL_ERR_TO_STR(CL_IMAGE_FORMAT_NOT_SUPPORTED);
-    CL_ERR_TO_STR(CL_MAP_FAILURE);
-    CL_ERR_TO_STR(CL_MISALIGNED_SUB_BUFFER_OFFSET);
-    CL_ERR_TO_STR(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST);
-    CL_ERR_TO_STR(CL_COMPILE_PROGRAM_FAILURE);
-    CL_ERR_TO_STR(CL_LINKER_NOT_AVAILABLE);
-    CL_ERR_TO_STR(CL_LINK_PROGRAM_FAILURE);
-    CL_ERR_TO_STR(CL_DEVICE_PARTITION_FAILED);
-    CL_ERR_TO_STR(CL_KERNEL_ARG_INFO_NOT_AVAILABLE);
-    CL_ERR_TO_STR(CL_INVALID_VALUE);
-    CL_ERR_TO_STR(CL_INVALID_DEVICE_TYPE);
-    CL_ERR_TO_STR(CL_INVALID_PLATFORM);
-    CL_ERR_TO_STR(CL_INVALID_DEVICE);
-    CL_ERR_TO_STR(CL_INVALID_CONTEXT);
-    CL_ERR_TO_STR(CL_INVALID_QUEUE_PROPERTIES);
-    CL_ERR_TO_STR(CL_INVALID_COMMAND_QUEUE);
-    CL_ERR_TO_STR(CL_INVALID_HOST_PTR);
-    CL_ERR_TO_STR(CL_INVALID_MEM_OBJECT);
-    CL_ERR_TO_STR(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR);
-    CL_ERR_TO_STR(CL_INVALID_IMAGE_SIZE);
-    CL_ERR_TO_STR(CL_INVALID_SAMPLER);
-    CL_ERR_TO_STR(CL_INVALID_BINARY);
-    CL_ERR_TO_STR(CL_INVALID_BUILD_OPTIONS);
-    CL_ERR_TO_STR(CL_INVALID_PROGRAM);
-    CL_ERR_TO_STR(CL_INVALID_PROGRAM_EXECUTABLE);
-    CL_ERR_TO_STR(CL_INVALID_KERNEL_NAME);
-    CL_ERR_TO_STR(CL_INVALID_KERNEL_DEFINITION);
-    CL_ERR_TO_STR(CL_INVALID_KERNEL);
-    CL_ERR_TO_STR(CL_INVALID_ARG_INDEX);
-    CL_ERR_TO_STR(CL_INVALID_ARG_VALUE);
-    CL_ERR_TO_STR(CL_INVALID_ARG_SIZE);
-    CL_ERR_TO_STR(CL_INVALID_KERNEL_ARGS);
-    CL_ERR_TO_STR(CL_INVALID_WORK_DIMENSION);
-    CL_ERR_TO_STR(CL_INVALID_WORK_GROUP_SIZE);
-    CL_ERR_TO_STR(CL_INVALID_WORK_ITEM_SIZE);
-    CL_ERR_TO_STR(CL_INVALID_GLOBAL_OFFSET);
-    CL_ERR_TO_STR(CL_INVALID_EVENT_WAIT_LIST);
-    CL_ERR_TO_STR(CL_INVALID_EVENT);
-    CL_ERR_TO_STR(CL_INVALID_OPERATION);
-    CL_ERR_TO_STR(CL_INVALID_GL_OBJECT);
-    CL_ERR_TO_STR(CL_INVALID_BUFFER_SIZE);
-    CL_ERR_TO_STR(CL_INVALID_MIP_LEVEL);
-    CL_ERR_TO_STR(CL_INVALID_GLOBAL_WORK_SIZE);
-    CL_ERR_TO_STR(CL_INVALID_PROPERTY);
-    CL_ERR_TO_STR(CL_INVALID_IMAGE_DESCRIPTOR);
-    CL_ERR_TO_STR(CL_INVALID_COMPILER_OPTIONS);
-    CL_ERR_TO_STR(CL_INVALID_LINKER_OPTIONS);
-    CL_ERR_TO_STR(CL_INVALID_DEVICE_PARTITION_COUNT);
-    case -69: return "CL_INVALID_PIPE_SIZE";
-    case -70: return "CL_INVALID_DEVICE_QUEUE";
-    case -71: return "CL_INVALID_SPEC_ID";
-    case -72: return "CL_MAX_SIZE_RESTRICTION_EXCEEDED";
-    case -1002: return "CL_INVALID_D3D10_DEVICE_KHR";
-    case -1003: return "CL_INVALID_D3D10_RESOURCE_KHR";
-    case -1004: return "CL_D3D10_RESOURCE_ALREADY_ACQUIRED_KHR";
-    case -1005: return "CL_D3D10_RESOURCE_NOT_ACQUIRED_KHR";
-    case -1006: return "CL_INVALID_D3D11_DEVICE_KHR";
-    case -1007: return "CL_INVALID_D3D11_RESOURCE_KHR";
-    case -1008: return "CL_D3D11_RESOURCE_ALREADY_ACQUIRED_KHR";
-    case -1009: return "CL_D3D11_RESOURCE_NOT_ACQUIRED_KHR";
-    case -1010: return "CL_INVALID_DX9_MEDIA_ADAPTER_KHR";
-    case -1011: return "CL_INVALID_DX9_MEDIA_SURFACE_KHR";
-    case -1012: return "CL_DX9_MEDIA_SURFACE_ALREADY_ACQUIRED_KHR";
-    case -1013: return "CL_DX9_MEDIA_SURFACE_NOT_ACQUIRED_KHR";
-    case -1093: return "CL_INVALID_EGL_OBJECT_KHR";
-    case -1092: return "CL_EGL_RESOURCE_NOT_ACQUIRED_KHR";
-    case -1001: return "CL_PLATFORM_NOT_FOUND_KHR";
-    case -1057: return "CL_DEVICE_PARTITION_FAILED_EXT";
-    case -1058: return "CL_INVALID_PARTITION_COUNT_EXT";
-    case -1059: return "CL_INVALID_PARTITION_NAME_EXT";
-    case -1094: return "CL_INVALID_ACCELERATOR_INTEL";
-    case -1095: return "CL_INVALID_ACCELERATOR_TYPE_INTEL";
-    case -1096: return "CL_INVALID_ACCELERATOR_DESCRIPTOR_INTEL";
-    case -1097: return "CL_ACCELERATOR_TYPE_NOT_SUPPORTED_INTEL";
-    case -1000: return "CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR";
-    case -1098: return "CL_INVALID_VA_API_MEDIA_ADAPTER_INTEL";
-    case -1099: return "CL_INVALID_VA_API_MEDIA_SURFACE_INTEL";
-    case -1100: return "CL_VA_API_MEDIA_SURFACE_ALREADY_ACQUIRED_INTEL";
-    case -1101: return "CL_VA_API_MEDIA_SURFACE_NOT_ACQUIRED_INTEL";
-    default: return "CL_UNKNOWN_ERROR";
-  }
-}

common/clutil.h

@@ -1,29 +0,0 @@
-#pragma once
-
-#ifdef __APPLE__
-#include <OpenCL/cl.h>
-#else
-#include <CL/cl.h>
-#endif
-
-#include <string>
-
-#define CL_CHECK(_expr)          \
-  do {                           \
-    assert(CL_SUCCESS == (_expr)); \
-  } while (0)
-
-#define CL_CHECK_ERR(_expr)           \
-  ({                                  \
-    cl_int err = CL_INVALID_VALUE;    \
-    __typeof__(_expr) _ret = _expr;   \
-    assert(_ret&& err == CL_SUCCESS); \
-    _ret;                             \
-  })
-
-cl_device_id cl_get_device_id(cl_device_type device_type);
-cl_context cl_create_context(cl_device_id device_id);
-cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
-cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
-cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);
-const char* cl_get_error_string(int err);

common/constants.py

@@ -1,6 +1,7 @@
 import numpy as np
 
-class Conversions:
+# conversions
+class CV:
   # Speed
   MPH_TO_KPH = 1.609344
   KPH_TO_MPH = 1. / MPH_TO_KPH
@@ -17,3 +18,6 @@ class Conversions:
 
   # Mass
   LB_TO_KG = 0.453592
+
+
+ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

common/dict_helpers.py

@@ -1,9 +0,0 @@
-# remove all keys that end in DEPRECATED
-def strip_deprecated_keys(d):
-  for k in list(d.keys()):
-    if isinstance(k, str):
-      if k.endswith('DEPRECATED'):
-        d.pop(k)
-      elif isinstance(d[k], dict):
-        strip_deprecated_keys(d[k])
-  return d

common/ffi_wrapper.py

@@ -1,8 +0,0 @@
-import platform
-
-
-def suffix():
-  if platform.system() == "Darwin":
-    return ".dylib"
-  else:
-    return ".so"

common/file_chunker.py

@@ -0,0 +1,37 @@
+import math
+import os
+from pathlib import Path
+
+CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
+
+def get_chunk_name(name, idx, num_chunks):
+  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
+
+def get_manifest_path(name):
+  return f"{name}.chunkmanifest"
+
+def get_chunk_paths(path, file_size):
+  num_chunks = math.ceil(file_size / CHUNK_SIZE)
+  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
+
+def chunk_file(path, targets):
+  manifest_path, *chunk_paths = targets
+  with open(path, 'rb') as f:
+    data = f.read()
+  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
+  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
+  for i, chunk_path in enumerate(chunk_paths):
+    with open(chunk_path, 'wb') as f:
+      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
+  Path(manifest_path).write_text(str(len(chunk_paths)))
+  os.remove(path)
+
+
+def read_file_chunked(path):
+  manifest_path = get_manifest_path(path)
+  if os.path.isfile(manifest_path):
+    num_chunks = int(Path(manifest_path).read_text().strip())
+    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
+  if os.path.isfile(path):
+    return Path(path).read_bytes()
+  raise FileNotFoundError(path)

common/file_helpers.py

@@ -1,37 +0,0 @@
-import os
-import tempfile
-import contextlib
-
-
-class CallbackReader:
-  """Wraps a file, but overrides the read method to also
-  call a callback function with the number of bytes read so far."""
-  def __init__(self, f, callback, *args):
-    self.f = f
-    self.callback = callback
-    self.cb_args = args
-    self.total_read = 0
-
-  def __getattr__(self, attr):
-    return getattr(self.f, attr)
-
-  def read(self, *args, **kwargs):
-    chunk = self.f.read(*args, **kwargs)
-    self.total_read += len(chunk)
-    self.callback(*self.cb_args, self.total_read)
-    return chunk
-
-
-@contextlib.contextmanager
-def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
-                        overwrite: bool = False):
-  """Write to a file atomically using a temporary file in the same directory as the destination file."""
-  dir_name = os.path.dirname(path)
-
-  if not overwrite and os.path.exists(path):
-    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
-
-  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
-    yield tmp_file
-    tmp_file_name = tmp_file.name
-  os.replace(tmp_file_name, path)

common/filter_simple.py

@@ -1,5 +1,4 @@
 class FirstOrderFilter:
-  # first order filter
   def __init__(self, x0, rc, dt, initialized=True):
     self.x = x0
     self.dt = dt
@@ -16,3 +15,20 @@ class FirstOrderFilter:
       self.initialized = True
       self.x = x
     return self.x
+
+
+class BounceFilter(FirstOrderFilter):
+  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
+    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
+    self.bounce = bounce
+    super().__init__(x0, rc, dt, initialized)
+
+  def update(self, x):
+    super().update(x)
+    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
+    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
+    self.velocity.update(0.0)
+    if abs(self.velocity.x) < 1e-3:
+      self.velocity.x = 0.0
+    self.x += self.velocity.x
+    return self.x

common/git.py

@@ -1,30 +1,30 @@
 from functools import cache
 import subprocess
-from openpilot.common.run import run_cmd, run_cmd_default
+from openpilot.common.utils import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str = None) -> str:
+def get_short_branch(cwd: str | None = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str = None) -> str:
+def get_branch(cwd: str | None = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str = None) -> str:
+def get_origin(cwd: str | None = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
@@ -34,7 +34,7 @@ def get_origin(cwd: str = None) -> str:
 
 
 @cache
-def get_normalized_origin(cwd: str = None) -> str:
+def get_normalized_origin(cwd: str | None = None) -> str:
   return get_origin(cwd) \
     .replace("git@", "", 1) \
     .replace(".git", "", 1) \

common/gpio.cc

@@ -1,84 +0,0 @@
-#include "common/gpio.h"
-
-#include <string>
-
-#ifdef __APPLE__
-int gpio_init(int pin_nr, bool output) {
-  return 0;
-}
-
-int gpio_set(int pin_nr, bool high) {
-  return 0;
-}
-
-int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
-  return 0;
-}
-
-#else
-
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <cstring>
-#include <linux/gpio.h>
-#include <sys/ioctl.h>
-
-#include "common/util.h"
-#include "common/swaglog.h"
-
-int gpio_init(int pin_nr, bool output) {
-  char pin_dir_path[50];
-  int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
-                           "/sys/class/gpio/gpio%d/direction", pin_nr);
-  if (pin_dir_path_len <= 0) {
-    return -1;
-  }
-  const char *value = output ? "out" : "in";
-  return util::write_file(pin_dir_path, (void*)value, strlen(value));
-}
-
-int gpio_set(int pin_nr, bool high) {
-  char pin_val_path[50];
-  int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
-                           "/sys/class/gpio/gpio%d/value", pin_nr);
-  if (pin_val_path_len <= 0) {
-    return -1;
-  }
-  return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);
-}
-
-int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
-
-  // Assumed that all interrupt pins are unexported and rights are given to
-  // read from gpiochip0.
-  std::string gpiochip_path = "/dev/gpiochip" + std::to_string(gpiochiop_id);
-  int fd = open(gpiochip_path.c_str(), O_RDONLY);
-  if (fd < 0) {
-    LOGE("Error opening gpiochip0 fd");
-    return -1;
-  }
-
-  // Setup event
-  struct gpioevent_request rq;
-  rq.lineoffset = pin_nr;
-  rq.handleflags = GPIOHANDLE_REQUEST_INPUT;
-
-  /* Requesting both edges as the data ready pulse from the lsm6ds sensor is
-     very short(75us) and is mostly detected as falling edge instead of rising.
-     So if it is detected as rising the following falling edge is skipped. */
-  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
-
-  strncpy(rq.consumer_label, consumer_label, std::size(rq.consumer_label) - 1);
-  int ret = util::safe_ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
-  if (ret == -1) {
-    LOGE("Unable to get line event from ioctl : %s", strerror(errno));
-    close(fd);
-    return -1;
-  }
-
-  close(fd);
-  return rq.fd;
-}
-
-#endif

common/gpio.h

@@ -1,33 +0,0 @@
-#pragma once
-
-// Pin definitions
-#ifdef QCOM2
-  #define GPIO_HUB_RST_N        30
-  #define GPIO_UBLOX_RST_N      32
-  #define GPIO_UBLOX_SAFEBOOT_N 33
-  #define GPIO_GNSS_PWR_EN      34 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
-  #define GPIO_STM_RST_N        124
-  #define GPIO_STM_BOOT0        134
-  #define GPIO_BMX_ACCEL_INT    21
-  #define GPIO_BMX_GYRO_INT     23
-  #define GPIO_BMX_MAGN_INT     87
-  #define GPIO_LSM_INT          84
-  #define GPIOCHIP_INT          0
-#else
-  #define GPIO_HUB_RST_N        0
-  #define GPIO_UBLOX_RST_N      0
-  #define GPIO_UBLOX_SAFEBOOT_N 0
-  #define GPIO_GNSS_PWR_EN      0 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
-  #define GPIO_STM_RST_N        0
-  #define GPIO_STM_BOOT0        0
-  #define GPIO_BMX_ACCEL_INT    0
-  #define GPIO_BMX_GYRO_INT     0
-  #define GPIO_BMX_MAGN_INT     0
-  #define GPIO_LSM_INT          0
-  #define GPIOCHIP_INT          0
-#endif
-
-int gpio_init(int pin_nr, bool output);
-int gpio_set(int pin_nr, bool high);
-
-int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr);

common/gpio.py

@@ -1,4 +1,6 @@
 import os
+import fcntl
+import ctypes
 from functools import cache
 
 def gpio_init(pin: int, output: bool) -> None:
@@ -52,3 +54,36 @@ def get_irqs_for_action(action: str) -> list[str]:
       if irq.isdigit() and action in get_irq_action(irq):
         ret.append(irq)
   return ret
+
+# *** gpiochip ***
+
+class gpioevent_data(ctypes.Structure):
+  _fields_ = [
+    ("timestamp", ctypes.c_uint64),
+    ("id", ctypes.c_uint32),
+  ]
+
+class gpioevent_request(ctypes.Structure):
+  _fields_ = [
+    ("lineoffset", ctypes.c_uint32),
+    ("handleflags", ctypes.c_uint32),
+    ("eventflags", ctypes.c_uint32),
+    ("label", ctypes.c_char * 32),
+    ("fd", ctypes.c_int)
+  ]
+
+def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
+  GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
+  GPIOHANDLE_REQUEST_INPUT = 0x1
+  GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
+
+  rq = gpioevent_request()
+  rq.lineoffset = pin
+  rq.handleflags = GPIOHANDLE_REQUEST_INPUT
+  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
+  rq.label = label.encode('utf-8')[:31] + b'\0'
+
+  fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
+  fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
+  os.close(fd)
+  return int(rq.fd)

common/i2c.cc

@@ -1,92 +0,0 @@
-#include "common/i2c.h"
-
-#include <fcntl.h>
-#include <sys/ioctl.h>
-#include <unistd.h>
-
-#include <cassert>
-#include <cstdio>
-#include <stdexcept>
-
-#include "common/swaglog.h"
-#include "common/util.h"
-
-#define UNUSED(x) (void)(x)
-
-#ifdef QCOM2
-// TODO: decide if we want to install libi2c-dev everywhere
-extern "C" {
-  #include <linux/i2c-dev.h>
-  #include <i2c/smbus.h>
-}
-
-I2CBus::I2CBus(uint8_t bus_id) {
-  char bus_name[20];
-  snprintf(bus_name, 20, "/dev/i2c-%d", bus_id);
-
-  i2c_fd = HANDLE_EINTR(open(bus_name, O_RDWR));
-  if (i2c_fd < 0) {
-    throw std::runtime_error("Failed to open I2C bus");
-  }
-}
-
-I2CBus::~I2CBus() {
-  if (i2c_fd >= 0) {
-    close(i2c_fd);
-  }
-}
-
-int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
-  std::lock_guard lk(m);
-
-  int ret = 0;
-
-  ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
-  if (ret < 0) { goto fail; }
-
-  ret = i2c_smbus_read_i2c_block_data(i2c_fd, register_address, len, buffer);
-  if ((ret < 0) || (ret != len)) { goto fail; }
-
-fail:
-  return ret;
-}
-
-int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
-  std::lock_guard lk(m);
-
-  int ret = 0;
-
-  ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
-  if (ret < 0) { goto fail; }
-
-  ret = i2c_smbus_write_byte_data(i2c_fd, register_address, data);
-  if (ret < 0) { goto fail; }
-
-fail:
-  return ret;
-}
-
-#else
-
-I2CBus::I2CBus(uint8_t bus_id) {
-  UNUSED(bus_id);
-  i2c_fd = -1;
-}
-
-I2CBus::~I2CBus() {}
-
-int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
-  UNUSED(device_address);
-  UNUSED(register_address);
-  UNUSED(buffer);
-  UNUSED(len);
-  return -1;
-}
-
-int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
-  UNUSED(device_address);
-  UNUSED(register_address);
-  UNUSED(data);
-  return -1;
-}
-#endif

common/i2c.h

@@ -1,19 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include <mutex>
-
-#include <sys/types.h>
-
-class I2CBus {
-  private:
-    int i2c_fd;
-    std::mutex m;
-
-  public:
-    I2CBus(uint8_t bus_id);
-    ~I2CBus();
-
-    int read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len);
-    int set_register(uint8_t device_address, uint register_address, uint8_t data);
-};

common/i2c.py

@@ -0,0 +1,81 @@
+import os
+import fcntl
+import ctypes
+
+# I2C constants from /usr/include/linux/i2c-dev.h
+I2C_SLAVE = 0x0703
+I2C_SLAVE_FORCE = 0x0706
+I2C_SMBUS = 0x0720
+
+# SMBus transfer types
+I2C_SMBUS_READ = 1
+I2C_SMBUS_WRITE = 0
+I2C_SMBUS_BYTE_DATA = 2
+I2C_SMBUS_I2C_BLOCK_DATA = 8
+
+I2C_SMBUS_BLOCK_MAX = 32
+
+
+class _I2cSmbusData(ctypes.Union):
+  _fields_ = [
+    ("byte", ctypes.c_uint8),
+    ("word", ctypes.c_uint16),
+    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
+  ]
+
+
+class _I2cSmbusIoctlData(ctypes.Structure):
+  _fields_ = [
+    ("read_write", ctypes.c_uint8),
+    ("command", ctypes.c_uint8),
+    ("size", ctypes.c_uint32),
+    ("data", ctypes.POINTER(_I2cSmbusData)),
+  ]
+
+
+class SMBus:
+  def __init__(self, bus: int):
+    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
+
+  def __enter__(self) -> 'SMBus':
+    return self
+
+  def __exit__(self, *args) -> None:
+    self.close()
+
+  def close(self) -> None:
+    if hasattr(self, '_fd') and self._fd >= 0:
+      os.close(self._fd)
+      self._fd = -1
+
+  def _set_address(self, addr: int, force: bool = False) -> None:
+    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
+    fcntl.ioctl(self._fd, ioctl_arg, addr)
+
+  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
+    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
+    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
+
+  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
+    return int(data.byte)
+
+  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    data.byte = value & 0xFF
+    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
+
+  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
+    self._set_address(addr, force)
+    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
+      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
+
+    data = _I2cSmbusData()
+    data.block[0] = length
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
+    read_len = int(data.block[0]) or length
+    read_len = min(read_len, length)
+    return [int(b) for b in data.block[1 : read_len + 1]]

common/logging_extra.py

@@ -8,6 +8,7 @@ import uuid
 import socket
 import logging
 import traceback
+import numpy as np
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
@@ -15,6 +16,8 @@ from contextlib import contextmanager
 LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
 
 def json_handler(obj):
+  if isinstance(obj, np.bool_):
+    return bool(obj)
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
   return repr(obj)
@@ -196,7 +199,6 @@ class SwagLogger(logging.Logger):
       co = f.f_code
       filename = os.path.normcase(co.co_filename)
 
-      # TODO: is this pylint exception correct?
       if filename == _srcfile:
         f = f.f_back
         continue

common/mat.h

@@ -1,85 +0,0 @@
-#pragma once
-
-typedef struct vec3 {
-  float v[3];
-} vec3;
-
-typedef struct vec4 {
-  float v[4];
-} vec4;
-
-typedef struct mat3 {
-  float v[3*3];
-} mat3;
-
-typedef struct mat4 {
-  float v[4*4];
-} mat4;
-
-static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
-  mat3 ret = {{0.0}};
-  for (int r=0; r<3; r++) {
-    for (int c=0; c<3; c++) {
-      float v = 0.0;
-      for (int k=0; k<3; k++) {
-        v += a.v[r*3+k] * b.v[k*3+c];
-      }
-      ret.v[r*3+c] = v;
-    }
-  }
-  return ret;
-}
-
-static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
-  vec3 ret = {{0.0}};
-  for (int r=0; r<3; r++) {
-    for (int c=0; c<3; c++) {
-      ret.v[r] += a.v[r*3+c] * b.v[c];
-    }
-  }
-  return ret;
-}
-
-static inline mat4 matmul(const mat4 &a, const mat4 &b) {
-  mat4 ret = {{0.0}};
-  for (int r=0; r<4; r++) {
-    for (int c=0; c<4; c++) {
-      float v = 0.0;
-      for (int k=0; k<4; k++) {
-        v += a.v[r*4+k] * b.v[k*4+c];
-      }
-      ret.v[r*4+c] = v;
-    }
-  }
-  return ret;
-}
-
-static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
-  vec4 ret = {{0.0}};
-  for (int r=0; r<4; r++) {
-    for (int c=0; c<4; c++) {
-      ret.v[r] += a.v[r*4+c] * b.v[c];
-    }
-  }
-  return ret;
-}
-
-// scales the input and output space of a transformation matrix
-// that assumes pixel-center origin.
-static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
-  // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
-
-  mat3 transform_out = {{
-    1.0f/s, 0.0f, 0.5f,
-    0.0f, 1.0f/s, 0.5f,
-    0.0f, 0.0f, 1.0f,
-  }};
-
-  mat3 transform_in = {{
-    s,  0.0f, -0.5f*s,
-    0.0f, s, -0.5f*s,
-    0.0f, 0.0f, 1.0f,
-  }};
-
-  return matmul3(transform_in, matmul3(in, transform_out));
-}

common/numpy_fast.py

@@ -1,19 +0,0 @@
-def clip(x, lo, hi):
-  return max(lo, min(hi, x))
-
-def interp(x, xp, fp):
-  N = len(xp)
-
-  def get_interp(xv):
-    hi = 0
-    while hi < N and xv > xp[hi]:
-      hi += 1
-    low = hi - 1
-    return fp[-1] if hi == N and xv > xp[low] else (
-      fp[0] if hi == 0 else
-      (xv - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]) + fp[low])
-
-  return [get_interp(v) for v in x] if hasattr(x, '__iter__') else get_interp(x)
-
-def mean(x):
-  return sum(x) / len(x)

common/parameterized.py

@@ -0,0 +1,47 @@
+import sys
+import pytest
+import inspect
+
+
+class parameterized:
+  @staticmethod
+  def expand(cases):
+    cases = list(cases)
+
+    if not cases:
+      return lambda func: pytest.mark.skip("no parameterized cases")(func)
+
+    def decorator(func):
+      params = [p for p in inspect.signature(func).parameters if p != 'self']
+      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
+      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
+      expand_params = params[: len(normalized[0])]
+      if len(expand_params) == 1:
+        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
+      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
+
+    return decorator
+
+
+def parameterized_class(attrs, input_list=None):
+  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
+    params_list = attrs
+  else:
+    assert input_list is not None
+    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
+    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
+
+  def decorator(cls):
+    globs = sys._getframe(1).f_globals
+    for i, params in enumerate(params_list):
+      name = f"{cls.__name__}_{i}"
+      new_cls = type(name, (cls,), dict(params))
+      new_cls.__module__ = cls.__module__
+      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
+      globs[name] = new_cls
+    # Don't collect the un-parametrised base, but return it so outer decorators
+    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
+    cls.__test__ = False
+    return cls
+
+  return decorator

common/params.cc

@@ -8,6 +8,7 @@
 #include <csignal>
 #include <unordered_map>
 
+#include "common/params_keys.h"
 #include "common/queue.h"
 #include "common/swaglog.h"
 #include "common/util.h"
@@ -87,121 +88,6 @@ private:
   int fd_ = -1;
 };
 
-std::unordered_map<std::string, uint32_t> keys = {
-    {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
-    {"AlwaysOnDM", PERSISTENT},
-    {"ApiCache_Device", PERSISTENT},
-    {"AssistNowToken", PERSISTENT},
-    {"AthenadPid", PERSISTENT},
-    {"AthenadUploadQueue", PERSISTENT},
-    {"AthenadRecentlyViewedRoutes", PERSISTENT},
-    {"BootCount", PERSISTENT},
-    {"CalibrationParams", PERSISTENT},
-    {"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
-    {"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
-    {"CarBatteryCapacity", PERSISTENT},
-    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"CarParamsCache", CLEAR_ON_MANAGER_START},
-    {"CarParamsPersistent", PERSISTENT},
-    {"CarParamsPrevRoute", PERSISTENT},
-    {"CompletedTrainingVersion", PERSISTENT},
-    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"CurrentBootlog", PERSISTENT},
-    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"DisablePowerDown", PERSISTENT},
-    {"DisableUpdates", PERSISTENT},
-    {"DisengageOnAccelerator", PERSISTENT},
-    {"DongleId", PERSISTENT},
-    {"DoReboot", CLEAR_ON_MANAGER_START},
-    {"DoShutdown", CLEAR_ON_MANAGER_START},
-    {"DoUninstall", CLEAR_ON_MANAGER_START},
-    {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
-    {"ExperimentalMode", PERSISTENT},
-    {"ExperimentalModeConfirmed", PERSISTENT},
-    {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"ForcePowerDown", PERSISTENT},
-    {"GitBranch", PERSISTENT},
-    {"GitCommit", PERSISTENT},
-    {"GitCommitDate", PERSISTENT},
-    {"GitDiff", PERSISTENT},
-    {"GithubSshKeys", PERSISTENT},
-    {"GithubUsername", PERSISTENT},
-    {"GitRemote", PERSISTENT},
-    {"GsmApn", PERSISTENT},
-    {"GsmMetered", PERSISTENT},
-    {"GsmRoaming", PERSISTENT},
-    {"HardwareSerial", PERSISTENT},
-    {"HasAcceptedTerms", PERSISTENT},
-    {"IMEI", PERSISTENT},
-    {"InstallDate", PERSISTENT},
-    {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
-    {"IsEngaged", PERSISTENT},
-    {"IsLdwEnabled", PERSISTENT},
-    {"IsMetric", PERSISTENT},
-    {"IsOffroad", CLEAR_ON_MANAGER_START},
-    {"IsOnroad", PERSISTENT},
-    {"IsRhdDetected", PERSISTENT},
-    {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
-    {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
-    {"IsTestedBranch", CLEAR_ON_MANAGER_START},
-    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"LanguageSetting", PERSISTENT},
-    {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
-    {"LastGPSPosition", PERSISTENT},
-    {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
-    {"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
-    {"LastUpdateException", CLEAR_ON_MANAGER_START},
-    {"LastUpdateTime", PERSISTENT},
-    {"LiveParameters", PERSISTENT},
-    {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
-    {"LocationFilterInitialState", PERSISTENT},
-    {"LongitudinalManeuverMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"LongitudinalPersonality", PERSISTENT},
-    {"NetworkMetered", PERSISTENT},
-    {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
-    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
-    {"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
-    {"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
-    {"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
-    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
-    {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
-    {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
-    {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
-    {"OpenpilotEnabledToggle", PERSISTENT},
-    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"PandaSignatures", CLEAR_ON_MANAGER_START},
-    {"PrimeType", PERSISTENT},
-    {"RecordFront", PERSISTENT},
-    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
-    {"SecOCKey", PERSISTENT | DONT_LOG},
-    {"RouteCount", PERSISTENT},
-    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"SshEnabled", PERSISTENT},
-    {"TermsVersion", PERSISTENT},
-    {"TrainingVersion", PERSISTENT},
-    {"UbloxAvailable", PERSISTENT},
-    {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
-    {"UpdaterAvailableBranches", PERSISTENT},
-    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
-    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
-    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
-    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
-    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
-    {"UpdaterState", CLEAR_ON_MANAGER_START},
-    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
-    {"UpdaterLastFetchTime", PERSISTENT},
-    {"Version", PERSISTENT},
-};
-
 } // namespace
 
 
@@ -229,8 +115,16 @@ bool Params::checkKey(const std::string &key) {
   return keys.find(key) != keys.end();
 }
 
+ParamKeyFlag Params::getKeyFlag(const std::string &key) {
+  return static_cast<ParamKeyFlag>(keys[key].flags);
+}
+
 ParamKeyType Params::getKeyType(const std::string &key) {
-  return static_cast<ParamKeyType>(keys[key]);
+  return keys[key].type;
+}
+
+std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
+  return keys[key].default_value;
 }
 
 int Params::put(const char* key, const char* value, size_t value_size) {
@@ -254,7 +148,7 @@ int Params::put(const char* key, const char* value, size_t value_size) {
     }
 
     // fsync to force persist the changes.
-    if ((result = fsync(tmp_fd)) < 0) break;
+    if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break;
 
     FileLock file_lock(params_path + "/.lock");
 
@@ -309,17 +203,17 @@ std::map<std::string, std::string> Params::readAll() {
   return util::read_files_in_dir(getParamPath());
 }
 
-void Params::clearAll(ParamKeyType key_type) {
+void Params::clearAll(ParamKeyFlag key_flag) {
   FileLock file_lock(params_path + "/.lock");
 
-  // 1) delete params of key_type
+  // 1) delete params of key_flag
   // 2) delete files that are not defined in the keys.
   if (DIR *d = opendir(getParamPath().c_str())) {
     struct dirent *de = NULL;
     while ((de = readdir(d))) {
       if (de->d_type != DT_DIR) {
         auto it = keys.find(de->d_name);
-        if (it == keys.end() || (it->second & key_type)) {
+        if (it == keys.end() || (it->second.flags & key_flag)) {
           unlink(getParamPath(de->d_name).c_str());
         }
       }

common/params.h

@@ -2,6 +2,7 @@
 
 #include <future>
 #include <map>
+#include <optional>
 #include <string>
 #include <tuple>
 #include <utility>
@@ -9,16 +10,33 @@
 
 #include "common/queue.h"
 
-enum ParamKeyType {
+enum ParamKeyFlag {
   PERSISTENT = 0x02,
   CLEAR_ON_MANAGER_START = 0x04,
   CLEAR_ON_ONROAD_TRANSITION = 0x08,
   CLEAR_ON_OFFROAD_TRANSITION = 0x10,
   DONT_LOG = 0x20,
   DEVELOPMENT_ONLY = 0x40,
+  CLEAR_ON_IGNITION_ON = 0x80,
   ALL = 0xFFFFFFFF
 };
 
+enum ParamKeyType {
+  STRING = 0, // must be utf-8 decodable
+  BOOL = 1,
+  INT = 2,
+  FLOAT = 3,
+  TIME = 4, // ISO 8601
+  JSON = 5,
+  BYTES = 6
+};
+
+struct ParamKeyAttributes {
+  uint32_t flags;
+  ParamKeyType type;
+  std::optional<std::string> default_value = std::nullopt;
+};
+
 class Params {
 public:
   explicit Params(const std::string &path = {});
@@ -29,14 +47,16 @@ public:
 
   std::vector<std::string> allKeys() const;
   bool checkKey(const std::string &key);
+  ParamKeyFlag getKeyFlag(const std::string &key);
   ParamKeyType getKeyType(const std::string &key);
+  std::optional<std::string> getKeyDefaultValue(const std::string &key);
   inline std::string getParamPath(const std::string &key = {}) {
     return params_path + params_prefix + (key.empty() ? "" : "/" + key);
   }
 
   // Delete a value
   int remove(const std::string &key);
-  void clearAll(ParamKeyType type);
+  void clearAll(ParamKeyFlag flag);
 
   // helpers for reading values
   std::string get(const std::string &key, bool block = false);

common/params.py

@@ -1,5 +1,6 @@
-from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
+from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
 assert Params
+assert ParamKeyFlag
 assert ParamKeyType
 assert UnknownKeyName
 

common/params_keys.h

@@ -0,0 +1,133 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include "cereal/gen/cpp/log.capnp.h"
+
+inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
+    {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
+    {"AdbEnabled", {PERSISTENT, BOOL}},
+    {"AlwaysOnDM", {PERSISTENT, BOOL}},
+    {"ApiCache_Device", {PERSISTENT, STRING}},
+    {"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
+    {"AssistNowToken", {PERSISTENT, STRING}},
+    {"AthenadPid", {PERSISTENT, INT}},
+    {"AthenadUploadQueue", {PERSISTENT, JSON}},
+    {"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
+    {"BootCount", {PERSISTENT, INT}},
+    {"CalibrationParams", {PERSISTENT, BYTES}},
+    {"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
+    {"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
+    {"CarBatteryCapacity", {PERSISTENT, INT}},
+    {"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
+    {"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"CarParamsPersistent", {PERSISTENT, BYTES}},
+    {"CarParamsPrevRoute", {PERSISTENT, BYTES}},
+    {"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
+    {"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"CurrentBootlog", {PERSISTENT, STRING}},
+    {"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"DisablePowerDown", {PERSISTENT, BOOL}},
+    {"DisableUpdates", {PERSISTENT, BOOL}},
+    {"DisengageOnAccelerator", {PERSISTENT, BOOL, "0"}},
+    {"DongleId", {PERSISTENT, STRING}},
+    {"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY, BOOL}},
+    {"ExperimentalMode", {PERSISTENT, BOOL}},
+    {"ExperimentalModeConfirmed", {PERSISTENT, BOOL}},
+    {"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"ForcePowerDown", {PERSISTENT, BOOL}},
+    {"GitBranch", {PERSISTENT, STRING}},
+    {"GitCommit", {PERSISTENT, STRING}},
+    {"GitCommitDate", {PERSISTENT, STRING}},
+    {"GitDiff", {PERSISTENT, STRING}},
+    {"GithubSshKeys", {PERSISTENT, STRING}},
+    {"GithubUsername", {PERSISTENT, STRING}},
+    {"GitRemote", {PERSISTENT, STRING}},
+    {"GsmApn", {PERSISTENT, STRING}},
+    {"GsmMetered", {PERSISTENT, BOOL, "1"}},
+    {"GsmRoaming", {PERSISTENT, BOOL}},
+    {"HardwareSerial", {PERSISTENT, STRING}},
+    {"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
+    {"InstallDate", {PERSISTENT, TIME}},
+    {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsEngaged", {PERSISTENT, BOOL}},
+    {"IsLdwEnabled", {PERSISTENT, BOOL}},
+    {"IsMetric", {PERSISTENT, BOOL}},
+    {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsOnroad", {PERSISTENT, BOOL}},
+    {"IsRhdDetected", {PERSISTENT, BOOL}},
+    {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LanguageSetting", {PERSISTENT, STRING, "en"}},
+    {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
+    {"LastGPSPosition", {PERSISTENT, STRING}},
+    {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
+    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
+    {"LastUpdateTime", {PERSISTENT, TIME}},
+    {"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"LiveDelay", {PERSISTENT, BYTES}},
+    {"LiveParameters", {PERSISTENT, JSON}},
+    {"LiveParametersV2", {PERSISTENT, BYTES}},
+    {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
+    {"LocationFilterInitialState", {PERSISTENT, BYTES}},
+    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LongitudinalPersonality", {PERSISTENT, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
+    {"NetworkMetered", {PERSISTENT, BOOL}},
+    {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
+    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"OpenpilotEnabledToggle", {PERSISTENT, BOOL, "1"}},
+    {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"PrimeType", {PERSISTENT, INT}},
+    {"RecordAudio", {PERSISTENT, BOOL}},
+    {"RecordAudioFeedback", {PERSISTENT, BOOL, "0"}},
+    {"RecordFront", {PERSISTENT, BOOL}},
+    {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
+    {"SecOCKey", {PERSISTENT | DONT_LOG, STRING}},
+    {"ShowDebugInfo", {PERSISTENT, BOOL}},
+    {"RouteCount", {PERSISTENT, INT, "0"}},
+    {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"SshEnabled", {PERSISTENT, BOOL}},
+    {"UbloxAvailable", {PERSISTENT, BOOL}},
+    {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
+    {"UpdaterAvailableBranches", {PERSISTENT, STRING}},
+    {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
+    {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"Version", {PERSISTENT, STRING}},
+};

common/params_pyx.pyx

@@ -1,18 +1,34 @@
 # distutils: language = c++
 # cython: language_level = 3
+import builtins
+import datetime
+import json
 from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
+from libcpp.optional cimport optional
+
+from openpilot.common.swaglog import cloudlog
 
 cdef extern from "common/params.h":
-  cpdef enum ParamKeyType:
+  cpdef enum ParamKeyFlag:
     PERSISTENT
     CLEAR_ON_MANAGER_START
     CLEAR_ON_ONROAD_TRANSITION
     CLEAR_ON_OFFROAD_TRANSITION
     DEVELOPMENT_ONLY
+    CLEAR_ON_IGNITION_ON
     ALL
 
+  cpdef enum ParamKeyType:
+    STRING
+    BOOL
+    INT
+    FLOAT
+    TIME
+    JSON
+    BYTES
+
   cdef cppclass c_Params "Params":
     c_Params(string) except + nogil
     string get(string, bool) nogil
@@ -23,10 +39,31 @@ cdef extern from "common/params.h":
     void putBoolNonBlocking(string, bool) nogil
     int putBool(string, bool) nogil
     bool checkKey(string) nogil
+    ParamKeyType getKeyType(string) nogil
+    optional[string] getKeyDefaultValue(string) nogil
     string getParamPath(string) nogil
-    void clearAll(ParamKeyType)
+    void clearAll(ParamKeyFlag)
     vector[string] allKeys()
 
+PYTHON_2_CPP = {
+  (str, STRING): lambda v: v,
+  (builtins.bool, BOOL): lambda v: "1" if v else "0",
+  (int, INT): str,
+  (float, FLOAT): str,
+  (datetime.datetime, TIME): lambda v: v.isoformat(),
+  (dict, JSON): json.dumps,
+  (list, JSON): json.dumps,
+  (bytes, BYTES): lambda v: v,
+}
+CPP_2_PYTHON = {
+  STRING: lambda v: v.decode("utf-8"),
+  BOOL: lambda v: v == b"1",
+  INT: int,
+  FLOAT: float,
+  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
+  JSON: json.loads,
+  BYTES: lambda v: v,
+}
 
 def ensure_bytes(v):
   return v.encode() if isinstance(v, str) else v
@@ -50,8 +87,8 @@ cdef class Params:
   def __dealloc__(self):
     del self.p
 
-  def clear_all(self, tx_type=ParamKeyType.ALL):
-    self.p.clearAll(tx_type)
+  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
+    self.p.clearAll(tx_flag)
 
   def check_key(self, key):
     key = ensure_bytes(key)
@@ -59,21 +96,38 @@ cdef class Params:
       raise UnknownKeyName(key)
     return key
 
-  def get(self, key, bool block=False, encoding=None):
+  def python2cpp(self, proposed_type, expected_type, value, key):
+    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
+    if cast:
+      return cast(value)
+    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
+
+  def _cpp2python(self, t, value, default, key):
+    if value is None:
+      return None
+    try:
+      return CPP_2_PYTHON[t](value)
+    except (KeyError, TypeError, ValueError):
+      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
+      return self._cpp2python(t, default, None, key)
+
+  def get(self, key, bool block=False, bool return_default=False):
     cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
     cdef string val
     with nogil:
       val = self.p.get(k, block)
 
+    default_val = (default.value() if default.has_value() else None) if return_default else None
     if val == b"":
       if block:
         # If we got no value while running in blocked mode
         # it means we got an interrupt while waiting
         raise KeyboardInterrupt
       else:
-        return None
-
-    return val if encoding is None else val.decode(encoding)
+        return self._cpp2python(t, default_val, None, key)
+    return self._cpp2python(t, val, default_val, key)
 
   def get_bool(self, key, bool block=False):
     cdef string k = self.check_key(key)
@@ -82,6 +136,11 @@ cdef class Params:
       r = self.p.getBool(k, block)
     return r
 
+  def _put_cast(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
+
   def put(self, key, dat):
     """
     Warning: This function blocks until the param is written to disk!
@@ -90,7 +149,7 @@ cdef class Params:
     in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = ensure_bytes(dat)
+    cdef string dat_bytes = self._put_cast(key, dat)
     with nogil:
       self.p.put(k, dat_bytes)
 
@@ -101,7 +160,7 @@ cdef class Params:
 
   def put_nonblocking(self, key, dat):
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = ensure_bytes(dat)
+    cdef string dat_bytes = self._put_cast(key, dat)
     with nogil:
       self.p.putNonBlocking(k, dat_bytes)
 
@@ -119,5 +178,19 @@ cdef class Params:
     cdef string key_bytes = ensure_bytes(key)
     return self.p.getParamPath(key_bytes).decode("utf-8")
 
+  def get_type(self, key):
+    return self.p.getKeyType(self.check_key(key))
+
   def all_keys(self):
     return self.p.allKeys()
+
+  def get_default_value(self, key):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
+
+  def cpp2python(self, key, value):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return self._cpp2python(t, value, None, key)

common/pid.py

@@ -1,46 +1,30 @@
 import numpy as np
 from numbers import Number
 
-from openpilot.common.numpy_fast import clip, interp
-
-
 class PIDController:
-  def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
-    self._k_p = k_p
-    self._k_i = k_i
-    self._k_d = k_d
-    self.k_f = k_f   # feedforward gain
-    if isinstance(self._k_p, Number):
-      self._k_p = [[0], [self._k_p]]
-    if isinstance(self._k_i, Number):
-      self._k_i = [[0], [self._k_i]]
-    if isinstance(self._k_d, Number):
-      self._k_d = [[0], [self._k_d]]
+  def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
+    self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
+    self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
 
-    self.pos_limit = pos_limit
-    self.neg_limit = neg_limit
+    self.set_limits(pos_limit, neg_limit)
 
-    self.i_unwind_rate = 0.3 / rate
-    self.i_rate = 1.0 / rate
+    self.i_dt = 1.0 / rate
     self.speed = 0.0
 
     self.reset()
 
   @property
   def k_p(self):
-    return interp(self.speed, self._k_p[0], self._k_p[1])
+    return np.interp(self.speed, self._k_p[0], self._k_p[1])
 
   @property
   def k_i(self):
-    return interp(self.speed, self._k_i[0], self._k_i[1])
+    return np.interp(self.speed, self._k_i[0], self._k_i[1])
 
   @property
   def k_d(self):
-    return interp(self.speed, self._k_d[0], self._k_d[1])
-
-  @property
-  def error_integral(self):
-    return self.i/self.k_i
+    return np.interp(self.speed, self._k_d[0], self._k_d[1])
 
   def reset(self):
     self.p = 0.0
@@ -49,25 +33,25 @@ class PIDController:
     self.f = 0.0
     self.control = 0
 
-  def update(self, error, error_rate=0.0, speed=0.0, override=False, feedforward=0., freeze_integrator=False):
+  def set_limits(self, pos_limit, neg_limit):
+    self.pos_limit = pos_limit
+    self.neg_limit = neg_limit
+
+  def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
     self.speed = speed
+    self.p = self.k_p * float(error)
+    self.d = self.k_d * error_rate
+    self.f = feedforward
 
-    self.p = float(error) * self.k_p
-    self.f = feedforward * self.k_f
-    self.d = error_rate * self.k_d
+    if not freeze_integrator:
+      i = self.i + self.k_i * self.i_dt * error
 
-    if override:
-      self.i -= self.i_unwind_rate * float(np.sign(self.i))
-    else:
-      if not freeze_integrator:
-        self.i = self.i + error * self.k_i * self.i_rate
-
-        # Clip i to prevent exceeding control limits
-        control_no_i = self.p + self.d + self.f
-        control_no_i = clip(control_no_i, self.neg_limit, self.pos_limit)
-        self.i = clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
+      # Don't allow windup if already clipping
+      test_control = self.p + i + self.d + self.f
+      i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
+      i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
+      self.i = np.clip(i, i_lowerbound, i_upperbound)
 
     control = self.p + self.i + self.d + self.f
-
-    self.control = clip(control, self.neg_limit, self.pos_limit)
+    self.control = np.clip(control, self.neg_limit, self.pos_limit)
     return self.control

common/prefix.h

@@ -13,7 +13,11 @@ public:
     if (prefix.empty()) {
       prefix = util::random_string(15);
     }
-    msgq_path = Path::shm_path() + "/" + prefix;
+#ifdef __APPLE__
+    msgq_path = "/tmp/msgq_" + prefix;
+#else
+    msgq_path = "/dev/shm/msgq_" + prefix;
+#endif
     bool ret = util::create_directories(msgq_path, 0777);
     assert(ret);
     setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
@@ -23,14 +27,14 @@ public:
     auto param_path = Params().getParamPath();
     if (util::file_exists(param_path)) {
       std::string real_path = util::readlink(param_path);
-      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
+      util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
       unlink(param_path.c_str());
     }
     if (getenv("COMMA_CACHE") == nullptr) {
-      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
+      util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
     }
-    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
-    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
+    util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
+    util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
     unsetenv("OPENPILOT_PREFIX");
   }
 

common/prefix.py

@@ -1,4 +1,5 @@
 import os
+import platform
 import shutil
 import uuid
 
@@ -9,20 +10,20 @@ from openpilot.system.hardware.hw import Paths
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
-  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
     self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
+    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
+    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
+    self.create_dirs_on_enter = create_dirs_on_enter
     self.clean_dirs_on_exit = clean_dirs_on_exit
     self.shared_download_cache = shared_download_cache
 
   def __enter__(self):
     self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
     os.environ['OPENPILOT_PREFIX'] = self.prefix
-    try:
-      os.mkdir(self.msgq_path)
-    except FileExistsError:
-      pass
-    os.makedirs(Paths.log_root(), exist_ok=True)
+
+    if self.create_dirs_on_enter:
+      self.create_dirs()
 
     if self.shared_download_cache:
       os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
@@ -40,6 +41,13 @@ class OpenpilotPrefix:
       pass
     return False
 
+  def create_dirs(self):
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
+
   def clean_dirs(self):
     symlink_path = Params().get_param_path()
     if os.path.exists(symlink_path):

common/ratekeeper.cc

@@ -6,9 +6,9 @@
 #include "common/timing.h"
 #include "common/util.h"
 
-RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
-    : name(name),
-      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
+RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
+    : name(name_),
+      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
   interval = 1 / rate;
   last_monitor_time = seconds_since_boot();
   next_frame_time = last_monitor_time + interval;

common/realtime.py

@@ -1,11 +1,12 @@
 """Utilities for reading real time clocks and keeping soft real time constraints."""
 import gc
 import os
+import sys
 import time
-from collections import deque
 
 from setproctitle import getproctitle
 
+from openpilot.common.utils import MovingAverage
 from openpilot.system.hardware import PC
 
 
@@ -27,19 +28,15 @@ class Priority:
   CTRL_HIGH = 53
 
 
-def set_realtime_priority(level: int) -> None:
-  if not PC:
-    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
-
-
 def set_core_affinity(cores: list[int]) -> None:
-  if not PC:
+  if sys.platform == 'linux' and not PC:
     os.sched_setaffinity(0, cores)
 
 
 def config_realtime_process(cores: int | list[int], priority: int) -> None:
   gc.disable()
-  set_realtime_priority(priority)
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(priority))
   c = cores if isinstance(cores, list) else [cores, ]
   set_core_affinity(c)
 
@@ -52,10 +49,12 @@ class Ratekeeper:
     self._frame = 0
     self._remaining = 0.0
     self._process_name = getproctitle()
-    self._dts = deque([self._interval], maxlen=100)
     self._last_monitor_time = -1.
     self._next_frame_time = -1.
 
+    self.avg_dt = MovingAverage(100)
+    self.avg_dt.add_value(self._interval)
+
   @property
   def frame(self) -> int:
     return self._frame
@@ -66,9 +65,8 @@ class Ratekeeper:
 
   @property
   def lagging(self) -> bool:
-    avg_dt = sum(self._dts) / len(self._dts)
     expected_dt = self._interval * (1 / 0.9)
-    return avg_dt > expected_dt
+    return self.avg_dt.get_average() > expected_dt
 
   # Maintain loop rate by calling this at the end of each loop
   def keep_time(self) -> bool:
@@ -85,7 +83,7 @@ class Ratekeeper:
 
     prev = self._last_monitor_time
     self._last_monitor_time = time.monotonic()
-    self._dts.append(self._last_monitor_time - prev)
+    self.avg_dt.add_value(self._last_monitor_time - prev)
 
     lagged = False
     remaining = self._next_frame_time - time.monotonic()

common/retry.py

@@ -1,30 +0,0 @@
-import time
-import functools
-
-from openpilot.common.swaglog import cloudlog
-
-
-def retry(attempts=3, delay=1.0, ignore_failure=False):
-  def decorator(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-      for _ in range(attempts):
-        try:
-          return func(*args, **kwargs)
-        except Exception:
-          cloudlog.exception(f"{func.__name__} failed, trying again")
-          time.sleep(delay)
-
-      if ignore_failure:
-        cloudlog.error(f"{func.__name__} failed after retry")
-      else:
-        raise Exception(f"{func.__name__} failed after retry")
-    return wrapper
-  return decorator
-
-
-if __name__ == "__main__":
-  @retry(attempts=10)
-  def abc():
-    raise ValueError("abc failed :(")
-  abc()

common/run.py

@@ -1,13 +0,0 @@
-import subprocess
-
-
-def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
-  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
-
-
-def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
-  try:
-    return run_cmd(cmd, cwd=cwd, env=env)
-  except subprocess.CalledProcessError:
-    return default
-

common/spinner.py

@@ -6,9 +6,9 @@ from openpilot.common.basedir import BASEDIR
 class Spinner:
   def __init__(self):
     try:
-      self.spinner_proc = subprocess.Popen(["./spinner"],
+      self.spinner_proc = subprocess.Popen(["./spinner.py"],
                                            stdin=subprocess.PIPE,
-                                           cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
+                                           cwd=os.path.join(BASEDIR, "system", "ui"),
                                            close_fds=True)
     except OSError:
       self.spinner_proc = None

common/swaglog.cc

@@ -26,6 +26,9 @@ public:
     zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
     zmq_connect(sock, Path::swaglog_ipc().c_str());
 
+    // workaround for https://github.com/dropbox/json11/issues/38
+    setlocale(LC_NUMERIC, "C");
+
     print_level = CLOUDLOG_WARNING;
     if (const char* print_lvl = getenv("LOGPRINT")) {
       if (strcmp(print_lvl, "debug") == 0) {

common/tests/test_file_helpers.py

@@ -1,7 +1,7 @@
 import os
 from uuid import uuid4
 
-from openpilot.common.file_helpers import atomic_write_in_dir
+from openpilot.common.utils import atomic_write
 
 
 class TestFileHelpers:
@@ -15,5 +15,5 @@ class TestFileHelpers:
       assert f.read() == "test"
     os.remove(path)
 
-  def test_atomic_write_in_dir(self):
-    self.run_atomic_write_func(atomic_write_in_dir)
+  def test_atomic_write(self):
+    self.run_atomic_write_func(atomic_write)

common/tests/test_numpy_fast.py

@@ -1,21 +0,0 @@
-import numpy as np
-
-from openpilot.common.numpy_fast import interp
-
-
-class TestInterp:
-  def test_correctness_controls(self):
-    _A_CRUISE_MIN_BP = np.asarray([0., 5., 10., 20., 40.])
-    _A_CRUISE_MIN_V = np.asarray([-1.0, -.8, -.67, -.5, -.30])
-    v_ego_arr = [-1, -1e-12, 0, 4, 5, 6, 7, 10, 11, 15.2, 20, 21, 39,
-                 39.999999, 40, 41]
-
-    expected = np.interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-    actual = interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-
-    np.testing.assert_equal(actual, expected)
-
-    for v_ego in v_ego_arr:
-      expected = np.interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-      actual = interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-      np.testing.assert_equal(actual, expected)

common/tests/test_params.py

@@ -1,10 +1,11 @@
 import pytest
+import datetime
 import os
 import threading
 import time
 import uuid
 
-from openpilot.common.params import Params, ParamKeyType, UnknownKeyName
+from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
 
 class TestParams:
   def setup_method(self):
@@ -12,7 +13,7 @@ class TestParams:
 
   def test_params_put_and_get(self):
     self.params.put("DongleId", "cb38263377b873ee")
-    assert self.params.get("DongleId") == b"cb38263377b873ee"
+    assert self.params.get("DongleId") == "cb38263377b873ee"
 
   def test_params_non_ascii(self):
     st = b"\xe1\x90\xff"
@@ -20,7 +21,7 @@ class TestParams:
     assert self.params.get("CarParams") == st
 
   def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", "test")
+    self.params.put("CarParams", b"test")
     self.params.put("DongleId", "cb38263377b873ee")
     assert self.params.get("CarParams") == b"test"
 
@@ -29,24 +30,24 @@ class TestParams:
       f.write("test")
     assert os.path.isfile(undefined_param)
 
-    self.params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
+    self.params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
     assert self.params.get("CarParams") is None
     assert self.params.get("DongleId") is not None
     assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
     self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", "123")
-    assert self.params.get("DongleId") == b"bob"
-    assert self.params.get("AthenadPid") == b"123"
+    self.params.put("AthenadPid", 123)
+    assert self.params.get("DongleId") == "bob"
+    assert self.params.get("AthenadPid") == 123
 
   def test_params_get_block(self):
     def _delayed_writer():
       time.sleep(0.1)
-      self.params.put("CarParams", "test")
+      self.params.put("CarParams", b"test")
     threading.Thread(target=_delayed_writer).start()
     assert self.params.get("CarParams") is None
-    assert self.params.get("CarParams", True) == b"test"
+    assert self.params.get("CarParams", block=True) == b"test"
 
   def test_params_unknown_key_fails(self):
     with pytest.raises(UnknownKeyName):
@@ -76,17 +77,17 @@ class TestParams:
     self.params.put_bool("IsMetric", False)
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", "1")
+    self.params.put("IsMetric", True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", "0")
+    self.params.put("IsMetric", False)
     assert not self.params.get_bool("IsMetric")
 
   def test_put_non_blocking_with_get_block(self):
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_nonblocking("CarParams", "test")
+      Params().put_nonblocking("CarParams", b"test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
@@ -107,3 +108,34 @@ class TestParams:
     assert len(keys) > 20
     assert len(keys) == len(set(keys))
     assert b"CarParams" in keys
+
+  def test_params_default_value(self):
+    self.params.remove("LanguageSetting")
+    self.params.remove("LongitudinalPersonality")
+    self.params.remove("LiveParameters")
+
+    assert self.params.get("LanguageSetting") is None
+    assert self.params.get("LanguageSetting", return_default=False) is None
+    assert isinstance(self.params.get("LanguageSetting", return_default=True), str)
+    assert isinstance(self.params.get("LongitudinalPersonality", return_default=True), int)
+    assert self.params.get("LiveParameters") is None
+    assert self.params.get("LiveParameters", return_default=True) is None
+
+  def test_params_get_type(self):
+    # json
+    self.params.put("ApiCache_FirehoseStats", {"a": 0})
+    assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
+
+    # int
+    self.params.put("BootCount", 1441)
+    assert self.params.get("BootCount") == 1441
+
+    # bool
+    self.params.put("AdbEnabled", True)
+    assert self.params.get("AdbEnabled")
+    assert isinstance(self.params.get("AdbEnabled"), bool)
+
+    # time
+    now = datetime.datetime.now(datetime.UTC)
+    self.params.put("InstallDate", now)
+    assert self.params.get("InstallDate") == now