Compile FrogPilot

Turn the screen off after your screen times out when onroad, but wake it back up when engagement state changes or important alerts are triggered.

.dockerignore

@@ -1,18 +0,0 @@
-**/.git
-.DS_Store
-*.dylib
-*.DSYM
-*.d
-*.pyc
-*.pyo
-.*.swp
-.*.swo
-.*.un~
-*.tmp
-*.o
-*.o-*
-*.os
-*.os-*
-
-venv/
-.venv/

.editorconfig

@@ -1,11 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-
-[*.{py,pyx,pxd}]
-charset = utf-8
-indent_style = space
-indent_size = 2

.gitattributes

@@ -1,23 +0,0 @@
-* text=auto
-
-# to move existing files into LFS:
-# git add --renormalize .
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.svg filter=lfs diff=lfs merge=lfs -text
-*.png filter=lfs diff=lfs merge=lfs -text
-*.gif filter=lfs diff=lfs merge=lfs -text
-*.ttf filter=lfs diff=lfs merge=lfs -text
-*.otf filter=lfs diff=lfs merge=lfs -text
-*.wav filter=lfs diff=lfs merge=lfs -text
-
-selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
-third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
-third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
-third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
-third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
-third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,47 +0,0 @@
-name: Bug report
-description: For issues with running openpilot on your comma device
-labels: ["bug"]
-body:
-
-  - type: markdown
-    attributes:
-      value: >
-        Before creating a **bug report**, please check the following:
-          * If the issue likely only affects your car model or make, go back and open a **car bug report** instead.
-          * If the issue is related to the driving or driver monitoring models, you should open a [discussion](https://github.com/commaai/openpilot/discussions/categories/model-feedback) instead.
-          * Ensure you're running the latest openpilot release.
-          * Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
-          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
-          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
-
-        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
-
-  - type: textarea
-    attributes:
-      label: Describe the bug
-      description: Also include a description of how to reproduce the bug
-    validations:
-      required: true
-
-  - type: input
-    id: route
-    attributes:
-      label: Provide a route where the issue occurs
-      description: Ensure the route is fully uploaded at https://useradmin.comma.ai. We cannot look into issues without routes, or at least a Dongle ID.
-      placeholder: 77611a1fac303767|2020-05-11--16-37-07
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: openpilot version
-      description: If you're not on release, provide the commit hash
-      placeholder: 0.8.10
-    validations:
-      required: true
-
-  - type: textarea
-    attributes:
-      label: Additional info
-

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Car bug report
-    url: https://github.com/commaai/opendbc/issues/new
-    about: For issues with a particular car make or model
-  - name: Join the Discord
-    url: https://discord.comma.ai
-    about: The community Discord is for both openpilot development and experience discussion
-  - name: Report driving behavior feedback
-    url: https://discord.com/channels/469524606043160576/1254834193066623017
-    about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord
-  - name: Community Wiki
-    url: https://github.com/commaai/openpilot/wiki
-    about: Check out our community wiki

.github/ISSUE_TEMPLATE/enhancement.md

@@ -1,8 +0,0 @@
----
-name: Enhancement
-about: For openpilot enhancement suggestions
-title: ''
-labels: 'enhancement'
-assignees: ''
----
-

.github/ISSUE_TEMPLATE/pc_bug_report.yml

@@ -1,42 +0,0 @@
-name: PC bug report
-description: For issues with running openpilot on PC
-labels: ["PC"]
-body:
-
-  - type: markdown
-    attributes:
-      value: >
-        Before creating a **bug report**, please check the following:
-          * Ensure you're running the latest openpilot release.
-          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
-          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
-
-        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
-
-  - type: textarea
-    attributes:
-      label: Describe the bug
-      description: Also include a description of how to reproduce the bug
-    validations:
-      required: true
-
-  - type: input
-    id: os-version
-    attributes:
-      label: OS Version
-      placeholder: Ubuntu 24.04
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: openpilot version or commit
-      placeholder: bd36f2ec8d3559909678eff2690c10a520938367
-    validations:
-      required: false
-
-  - type: textarea
-    attributes:
-      label: Additional info
-

.github/labeler.yaml

@@ -1,27 +0,0 @@
-CI / testing:
-  - changed-files:
-    - any-glob-to-all-files: "{.github/**,**/test_*,**/test/**,Jenkinsfile}"
-
-car:
-  - changed-files:
-    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
-
-simulation:
-  - changed-files:
-    - any-glob-to-all-files: 'tools/sim/**'
-
-ui:
-  - changed-files:
-    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
-
-tools:
-  - changed-files:
-    - any-glob-to-all-files: 'tools/**'
-
-multilanguage:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
-
-autonomy:
-  - changed-files:
-    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"

.github/pull_request_template.md

@@ -1,68 +0,0 @@
-<!-- Please copy and paste the relevant template -->
-
-<!--- ***** Template: Fingerprint *****
-
-**Car**
-Which car (make, model, year) this fingerprint is for
-
-**Route**
-A route with the fingerprint
-
--->
-
-<!--- ***** Template: Car Bugfix *****
-
-**Description**
-
-A description of the bug and the fix. Also link the issue if it exists. 
-
-**Verification**
-
-Explain how you tested this bug fix. 
-
-**Route**
-
-Route: [a route with the bug fix]
-
-
--->
-
-<!--- ***** Template: Bugfix *****
-
-**Description**
-
-A description of the bug and the fix. Also link the issue if it exists. 
-
-**Verification**
-
-Explain how you tested this bug fix. 
-
-
--->
-
-<!--- ***** Template: Car Port *****
-
-**Checklist**
-
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
-- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
-- [ ] route with openpilot:
-- [ ] route with stock system:
-- [ ] car harness used (if comma doesn't sell it, put N/A):
-
-
--->
-
-<!--- ***** Template: Refactor *****
-
-**Description**
-
-A description of the refactor, including the goals it accomplishes. 
-
-**Verification**
-
-Explain how you tested the refactor for regressions. 
-
-
--->
-

.github/workflows/auto_pr_review.yaml

@@ -1,35 +0,0 @@
-name: "PR review"
-on:
-  pull_request_target:
-    types: [opened, reopened, synchronize, edited]
-
-jobs:
-  labeler:
-    name: review
-    permissions:
-      contents: read
-      pull-requests: write
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: false
-
-    # Label PRs
-    - uses: actions/labeler@v6
-      with:
-        dot: true
-        configuration-path: .github/labeler.yaml
-
-    # Check PR target branch
-    - name: check branch
-      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
-      if: github.repository == 'commaai/openpilot'
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        target: /^(?!master$).*/
-        exclude: /commaai:.*/
-        change-to: ${{ github.base_ref }}
-        already-exists-action: close_this
-        already-exists-comment: "Your PR should be made against the `master` branch"

.github/workflows/badges.yaml

@@ -1,35 +0,0 @@
-name: badges
-on:
-  schedule:
-    - cron: '0 * * * *'
-  workflow_dispatch:
-
-env:
-  PYTHONPATH: ${{ github.workspace }}
-
-jobs:
-  badges:
-    name: create badges
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    permissions:
-      contents: write
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Push badges
-      run: |
-        python3 selfdrive/ui/translations/create_badges.py
-
-        rm .gitattributes
-
-        git checkout --orphan badges
-        git rm -rf --cached .
-        git config user.email "badge-researcher@comma.ai"
-        git config user.name "Badge Researcher"
-
-        git add translation_badge.svg
-        git commit -m "Add/Update badges"
-        git push -f origin HEAD

.github/workflows/diff_report.yaml

@@ -1,45 +0,0 @@
-name: diff report
-
-on:
-  pull_request_target:
-    types: [opened, synchronize, reopened]
-
-jobs:
-  comment:
-    name: comment
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-    - name: Wait for process replay
-      id: wait
-      continue-on-error: true
-      uses: lewagon/wait-on-check-action@v1.3.4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        check-name: process replay
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        allowed-conclusions: success,failure
-        wait-interval: 20
-    - name: Download diff
-      if: steps.wait.outcome == 'success'
-      uses: dawidd6/action-download-artifact@v6
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: tests.yaml
-        workflow_conclusion: ''
-        pr: ${{ github.event.number }}
-        name: diff_report_${{ github.event.number }}
-        path: .
-        allow_forks: true
-    - name: Comment on PR
-      if: steps.wait.outcome == 'success'
-      uses: thollander/actions-comment-pull-request@v2
-      with:
-        filePath: diff_report.txt
-        comment_tag: diff_report
-        pr_number: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -1,65 +0,0 @@
-name: docs
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-concurrency:
-  group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-jobs:
-  docs:
-    name: build docs
-    runs-on: ubuntu-24.04
-    steps:
-    - uses: commaai/timeout@v1
-
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-
-    # Build
-    - name: Build docs
-      run: |
-        # TODO: can we install just the "docs" dependency group without the normal deps?
-        pip install mkdocs
-        mkdocs build
-
-    # Push to docs.comma.ai
-    - uses: actions/checkout@v6
-      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
-      with:
-        path: openpilot-docs
-        ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
-        repository: commaai/openpilot-docs
-    - name: Push
-      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
-      run: |
-        set -x
-
-        source release/identity.sh
-
-        cd openpilot-docs
-        git checkout --orphan tmp
-        git rm -rf .
-
-        # copy over docs
-        cp -r ../docs_site/ docs/
-
-        # GitHub pages config
-        touch docs/.nojekyll
-        echo -n docs.comma.ai > docs/CNAME
-
-        git add -f .
-        git commit -m "build docs"
-
-        # docs live in different repo to not bloat openpilot's full clone size
-        git push -f origin tmp:gh-pages

.github/workflows/jenkins-pr-trigger.yaml

@@ -1,96 +0,0 @@
-name: jenkins scan
-
-on:
-  issue_comment:
-    types: [created, edited]
-
-jobs:
-  cleanup-branches:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Delete stale Jenkins branches
-        uses: actions/github-script@v8
-        with:
-          script: |
-            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
-            const prefixes = ['tmp-jenkins', '__jenkins'];
-
-            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              per_page: 100,
-            })) {
-              for (const branch of response.data) {
-                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
-
-                const { data: commit } = await github.rest.repos.getCommit({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: branch.commit.sha,
-                });
-
-                const commitDate = new Date(commit.commit.committer.date).getTime();
-                if (commitDate < cutoff) {
-                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
-                  await github.rest.git.deleteRef({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                    ref: `heads/${branch.name}`,
-                  });
-                }
-              }
-            }
-
-  scan-comments:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.issue.pull_request }}
-    permissions:
-      contents: write
-      issues: write
-    steps:
-    - name: Check for trigger phrase
-      id: check_comment
-      uses: actions/github-script@v8
-      with:
-        script: |
-          const triggerPhrase = "trigger-jenkins";
-          const comment = context.payload.comment.body;
-          const commenter = context.payload.comment.user.login;
-
-          const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            username: commenter
-          });
-
-          const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin';
-
-          return (hasWriteAccess && comment.includes(triggerPhrase));
-        result-encoding: json
-
-    - name: Checkout repository
-      if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v6
-      with:
-        ref: refs/pull/${{ github.event.issue.number }}/head
-
-    - name: Push to tmp-jenkins branch
-      if: steps.check_comment.outputs.result == 'true'
-      run: |
-        git config --global user.name "github-actions[bot]"
-        git config --global user.email "github-actions[bot]@users.noreply.github.com"
-        git checkout -b tmp-jenkins-${{ github.event.issue.number }}
-        GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
-
-    - name: Delete trigger comment
-      if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v8
-      with:
-        script: |
-          await github.rest.issues.deleteComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            comment_id: context.payload.comment.id,
-          });

.github/workflows/model_review.yaml

@@ -1,42 +0,0 @@
-name: "model review"
-
-on:
-  pull_request:
-    types: [opened, reopened, synchronize]
-    paths:
-      - 'selfdrive/modeld/models/*.onnx'
-  workflow_dispatch:
-
-jobs:
-  comment:
-    permissions:
-      contents: read
-      pull-requests: write
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          submodules: true
-      - name: Checkout master
-        uses: actions/checkout@v6
-        with:
-          ref: master
-          path: base
-      - run: git lfs pull
-      - run: cd base && git lfs pull
-
-      - name: scripts/reporter.py
-        id: report
-        run: |
-          echo "content<<EOF" >> $GITHUB_OUTPUT
-          echo "## Model Review" >> $GITHUB_OUTPUT
-          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-
-      - name: Post model report comment
-        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
-        with:
-          header: model-review
-          message: ${{ steps.report.outputs.content }}

.github/workflows/prebuilt.yaml

@@ -1,39 +0,0 @@
-name: prebuilt
-on:
-  schedule:
-    - cron: '0 * * * *'
-  workflow_dispatch:
-
-env:
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh
-
-jobs:
-  build_prebuilt:
-    name: build prebuilt
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    env:
-      PUSH_IMAGE: true
-    permissions:
-      checks: read
-      contents: read
-      packages: write
-    steps:
-    - name: Wait for green check mark
-      if: ${{ github.event_name != 'workflow_dispatch' }}
-      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
-      with:
-        ref: master
-        wait-interval: 30
-        running-workflow-name: 'build prebuilt'
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: git lfs pull
-    - name: Build and Push docker image
-      run: |
-        $DOCKER_LOGIN
-        eval "$BUILD"

.github/workflows/release.yaml

@@ -1,31 +0,0 @@
-name: release
-on:
-  schedule:
-    - cron: '0 9 * * *'
-  workflow_dispatch:
-
-jobs:
-  build_masterci:
-    name: build master-ci
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    permissions:
-      checks: read
-      contents: write
-    steps:
-    - name: Wait for green check mark
-      if: ${{ github.event_name == 'schedule' }}
-      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
-      with:
-        ref: master
-        wait-interval: 30
-        running-workflow-name: 'build master-ci'
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-        fetch-depth: 0
-    - run: ./tools/op.sh setup
-    - name: Push master-ci
-      run: BRANCH=__nightly release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -1,73 +0,0 @@
-name: repo maintenance
-
-on:
-  schedule:
-    - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
-  workflow_dispatch:
-
-env:
-  PYTHONPATH: ${{ github.workspace }}
-
-jobs:
-  package_updates:
-    name: package_updates
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: uv lock
-      run: uv lock --upgrade
-    - name: uv pip tree
-      id: pip_tree
-      run: |
-        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
-        uv pip tree >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
-    - name: venv size
-      id: venv_size
-      run: |
-        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
-        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
-        echo "" >> $GITHUB_OUTPUT
-        echo "Top 10 by size:" >> $GITHUB_OUTPUT
-        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
-          | grep -v '\.dist-info' \
-          | grep -v '__pycache__' \
-          | sort -rh \
-          | head -10 \
-          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
-    - name: bump submodules
-      run: |
-        git submodule update --remote
-        git add .
-    - name: update car docs
-      run: |
-        python selfdrive/car/docs.py
-        git add docs/CARS.md
-    - name: Create Pull Request
-      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
-      with:
-        author: Vehicle Researcher <user@comma.ai>
-        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
-        commit-message: Update Python packages
-        title: '[bot] Update Python packages'
-        branch: auto-package-updates
-        base: master
-        delete-branch: true
-        body: |
-          Automatic PR from repo-maintenance -> package_updates
-
-          ```
-          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
-          ${{ steps.venv_size.outputs.VENV_SIZE }}
-          ```
-
-          ```
-          $ uv pip tree
-          ${{ steps.pip_tree.outputs.PIP_TREE }}
-          ```
-        labels: bot

.github/workflows/stale.yaml

@@ -1,52 +0,0 @@
-name: stale
-on:
-  schedule:
-    - cron: '30 1 * * *'
-  workflow_dispatch:
-
-env:
-  DAYS_BEFORE_PR_CLOSE: 7
-  DAYS_BEFORE_PR_STALE: 24
-  DAYS_BEFORE_PR_STALE_DRAFT: 30
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v10
-        with:
-          exempt-all-milestones: true
-
-          # pull request config
-          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
-          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
-          stale-pr-label: stale
-          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
-          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
-          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
-          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-          exempt-draft-pr: false
-
-          # issue config
-          days-before-issue-stale: -1 # ignore issues for now
-
-  # same as above, but give draft PRs more time
-  stale_drafts:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v10
-        with:
-          exempt-all-milestones: true
-
-          # pull request config
-          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
-          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
-          stale-pr-label: stale
-          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
-          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
-          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }}
-          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-          exempt-draft-pr: true
-
-          # issue config
-          days-before-issue-stale: -1 # ignore issues for now

.github/workflows/tests.yaml

@@ -1,231 +0,0 @@
-name: tests
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  CI: 1
-  PYTHONPATH: ${{ github.workspace }}
-  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
-
-jobs:
-  build_release:
-    name: build release
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-      PYTHONPATH: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - run: ./tools/op.sh setup
-    - name: Build openpilot and run checks
-      timeout-minutes: 30
-      working-directory: ${{ env.STRIPPED_DIR }}
-      run: python3 system/manager/build.py
-    - name: Run tests
-      timeout-minutes: 1
-      working-directory: ${{ env.STRIPPED_DIR }}
-      run: release/check-dirty.sh
-    - name: Check submodules
-      if: github.repository == 'commaai/openpilot'
-      timeout-minutes: 3
-      run: release/check-submodules.sh
-
-  build_mac:
-    name: build macOS
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Remove Homebrew from environment
-      run: |
-        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
-        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
-    - run: ./tools/op.sh setup
-    - name: Building openpilot
-      run: scons
-
-  static_analysis:
-    name: static analysis
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Static analysis
-      timeout-minutes: 1
-      run: scripts/lint/lint.sh
-
-  unit_tests:
-    name: unit tests
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
-      run: |
-        source selfdrive/test/setup_xvfb.sh
-        # Pre-compile Python bytecode so each pytest worker doesn't need to
-        $PYTEST --collect-only -m 'not slow' -qq
-        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
-
-  process_replay:
-    name: process replay
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
-      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
-      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - name: Print diff report
-      if: always()
-      run: cat selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v6
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload diff report
-      uses: actions/upload-artifact@v6
-      if: always() && github.event_name == 'pull_request'
-      continue-on-error: true
-      with:
-        name: diff_report_${{ github.event.number }}
-        path: selfdrive/test/process_replay/diff_report.txt
-    - name: Checkout ci-artifacts
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v4
-      with:
-        repository: commaai/ci-artifacts
-        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-        path: ${{ github.workspace }}/ci-artifacts
-    - name: Push refs
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      working-directory: ${{ github.workspace }}/ci-artifacts
-      run: |
-        git config user.name "GitHub Actions Bot"
-        git config user.email "<>"
-        git fetch origin process-replay || true
-        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
-        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
-        echo "${{ github.sha }}" > ref_commit
-        git add .
-        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
-        git push origin process-replay --force
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      env:
-        ONNXCPU: 1
-      run: $PYTEST selfdrive/test/process_replay/test_regen.py
-
-  simulator_driving:
-    name: simulator driving
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: false  # FIXME: Started to timeout recently
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Driving test
-      timeout-minutes: 2
-      run: |
-        source selfdrive/test/setup_xvfb.sh
-        pytest -s tools/sim/tests/test_metadrive_bridge.py
-
-  create_ui_report:
-    name: Create UI Report
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-      - run: ./tools/op.sh setup
-      - name: Build openpilot
-        run: scons -j$(nproc)
-      - name: Create UI Report
-        run: |
-          source selfdrive/test/setup_xvfb.sh
-          python3 selfdrive/ui/tests/diff/replay.py
-          python3 selfdrive/ui/tests/diff/replay.py --big
-      - name: Upload UI Report
-        uses: actions/upload-artifact@v6
-        with:
-          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/diff/report

.github/workflows/ui_preview.yaml

@@ -1,175 +0,0 @@
-name: "ui preview"
-on:
-  push:
-    branches:
-      - master
-  pull_request_target:
-    types: [assigned, opened, synchronize, reopened, edited]
-    branches:
-      - 'master'
-    paths:
-      - 'selfdrive/assets/**'
-      - 'selfdrive/ui/**'
-      - 'system/ui/**'
-  workflow_dispatch:
-
-env:
-  UI_JOB_NAME: "Create UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
-  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
-
-  # variant:video_prefix:master_branch
-  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
-
-jobs:
-  preview:
-    if: github.repository == 'commaai/openpilot'
-    name: preview
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-
-      - name: Waiting for ui generation to end
-        uses: lewagon/wait-on-check-action@v1.3.4
-        with:
-          ref: ${{ env.SHA }}
-          check-name: ${{ env.UI_JOB_NAME }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allowed-conclusions: success
-          wait-interval: 20
-
-      - name: Getting workflow run ID
-        id: get_run_id
-        run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
-
-      - name: Getting proposed ui
-        uses: dawidd6/action-download-artifact@v6
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ steps.get_run_id.outputs.run_id }}
-          search_artifacts: true
-          name: ui-report-1-${{ env.REPORT_NAME }}
-          path: ${{ github.workspace }}/pr_ui
-
-      - name: Getting mici master ui
-        uses: actions/checkout@v6
-        with:
-          repository: commaai/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_mici
-          ref: openpilot_master_ui_mici_raylib
-
-      - name: Getting big master ui
-        uses: actions/checkout@v6
-        with:
-          repository: commaai/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_big
-          ref: openpilot_master_ui_big_raylib
-
-      - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video branch <<< "$variant"
-            master_dir="${{ github.workspace }}/master_${name}"
-            cd "$master_dir"
-            git checkout --orphan=new_branch
-            git rm -rf *
-            git branch -D "$branch"
-            git branch -m "$branch"
-            git config user.name "GitHub Actions Bot"
-            git config user.email "<>"
-            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
-            git add .
-            git commit -m "${name} video for commit ${{ env.SHA }}"
-            git push origin "$branch" --force
-          done
-
-      - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
-
-      - name: Finding diffs
-        if: github.event_name == 'pull_request_target'
-        id: find_diff
-        run: |
-          export PYTHONPATH=${{ github.workspace }}
-          baseurl="https://github.com/commaai/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
-
-          COMMENT=""
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video _ <<< "$variant"
-            diff_name="${name}_diff"
-
-            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
-            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
-
-            diff_exit_code=0
-            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
-              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
-              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
-              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
-
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
-
-            REPORT_URL="https://commaai.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
-            if [ $diff_exit_code -eq 0 ]; then
-              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
-            else
-              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
-            fi
-          done
-
-          {
-            echo "COMMENT<<EOF"
-            echo "$COMMENT"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Saving proposed ui
-        if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_mici
-        run: |
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          git checkout --orphan=${{ env.BRANCH_NAME }}
-          git rm -rf *
-          mv ${{ github.workspace }}/pr_ui/* .
-          git add .
-          git commit -m "ui videos for PR #${{ github.event.number }}"
-          git push origin ${{ env.BRANCH_NAME }} --force
-
-          # Append diff reports to report files branch
-          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
-          for variant in $VARIANTS; do
-            IFS=':' read -r name _ _ <<< "$variant"
-            diff_name="${name}_diff"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
-            git add "${diff_name}_pr_${{ github.event.number }}.html"
-          done
-          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
-          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-
-      - name: Comment on PR
-        if: github.event_name == 'pull_request_target'
-        uses: thollander/actions-comment-pull-request@v2
-        with:
-          message: |
-            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
-            ## UI Preview
-            ${{ steps.find_diff.outputs.COMMENT }}
-          comment_tag: run_id_ui_preview
-          pr_number: ${{ github.event.number }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -10,16 +10,13 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
+model2.png
 a.out
 .hypothesis
-.cache/
-bin/
 
-*.mp4
 *.dylib
 *.DSYM
 *.d
-*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -36,53 +33,53 @@ bin/
 *.class
 *.pyxbldc
 *.vcd
-*.mo
-*_pyx.cpp
-*.stats
-*.pkl
-*.pkl*
+*.qm
 config.json
+clcache
 compile_commands.json
 compare_runtime*.html
 
-# build artifacts
+persist
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
-cereal/messaging/bridge
+selfdrive/logcatd/logcatd
+selfdrive/mapd/default_speeds_by_region.json
+system/proclogd/proclogd
+selfdrive/ui/translations/alerts_generated.h
 selfdrive/ui/translations/tmp
+selfdrive/test/longitudinal_maneuvers/out
 selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
+selfdrive/modeld/_modeld
+selfdrive/modeld/_dmonitoringmodeld
+/src/
+
+notebooks
+hyperthneed
+provisioning
 
 .coverage*
 coverage.xml
 htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/thneed/compile
+selfdrive/modeld/models/*.thneed
+selfdrive/modeld/models/*.pkl
 
-# openpilot log files
 *.bz2
-*.zst
-*.rlog
 
 build/
 
 !**/.gitkeep
 
-
-### VisualStudioCode ###
-*.vsix
-.history
-.ionide
-.vscode/*
-.history/
-!.vscode/settings.json
-!.vscode/tasks.json
-!.vscode/launch.json
-!.vscode/extensions.json
-!.vscode/*.code-snippets
-
-# agents
-.claude/
-.context/
-PLAN.md
-TASK.md
+poetry.toml
+Pipfile

.gitmodules

@@ -1,18 +0,0 @@
-[submodule "panda"]
-  path = panda
-  url = ../../commaai/panda.git
-[submodule "opendbc"]
-  path = opendbc_repo
-  url = ../../commaai/opendbc.git
-[submodule "msgq"]
-  path = msgq_repo
-  url = ../../commaai/msgq.git
-[submodule "rednose_repo"]
-  path = rednose_repo
-  url = ../../commaai/rednose.git
-[submodule "teleoprtc_repo"]
-  path = teleoprtc_repo
-  url = ../../commaai/teleoprtc
-[submodule "tinygrad"]
-  path = tinygrad_repo
-  url = https://github.com/tinygrad/tinygrad.git

.lfsconfig

@@ -1,4 +0,0 @@
-[lfs]
-	url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-	pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
-	locksverify = false

.python-version

@@ -1 +0,0 @@
-3.12.13

.vscode/extensions.json

@@ -1,8 +0,0 @@
-{
-  "recommendations": [
-    "ms-python.python",
-    "ms-vscode.cpptools",
-    "elagil.pre-commit-helper",
-    "charliermarsh.ruff",
-  ]
-}

.vscode/launch.json

@@ -1,88 +0,0 @@
-{
-  "version": "0.2.0",
-  "inputs": [
-    {
-      "id": "python_process",
-      "type": "pickString",
-      "description": "Select the process to debug",
-      "options": [
-        "selfdrive/controls/controlsd.py",
-        "system/timed/timed.py",
-        "tools/sim/run_bridge.py"
-      ]
-    },
-    {
-      "id": "cpp_process",
-      "type": "pickString",
-      "description": "Select the process to debug",
-      "options": [
-        "selfdrive/ui/ui"
-      ]
-    },
-    {
-      "id": "args",
-      "description": "Arguments to pass to the process",
-      "type": "promptString"
-    },
-    {
-      "id": "replayArg",
-      "type": "promptString",
-      "description": "Enter route or segment to replay."
-    }
-  ],
-  "configurations": [
-    {
-      "name": "Python: openpilot Process",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${input:python_process}",
-      "console": "integratedTerminal",
-      "justMyCode": true,
-      "args": "${input:args}"
-    },
-    {
-      "name": "C++: openpilot Process",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}"
-    },
-    {
-      "name": "Attach LLDB to Replay drive",
-      "type": "lldb",
-      "request": "attach",
-      "pid": "${command:pickMyProcess}",
-      "sourceMap": {
-        ".": "${workspaceFolder}/opendbc/safety"
-      },
-      "initCommands": [
-        "script import time; time.sleep(3)"
-      ]
-    },
-    {
-      "name": "Replay drive",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
-      "args": [
-        "${input:replayArg}"
-      ],
-      "console": "integratedTerminal",
-      "justMyCode": false,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}"
-      },
-      "subProcess": true,
-      "stopOnEntry": false
-    }
-  ],
-  "compounds": [
-    {
-      "name": "Replay drive + Safety LLDB",
-      "configurations": [
-        "Replay drive",
-        "Attach LLDB to Replay drive"
-      ]
-    }
-  ]
-}

.vscode/settings.json

@@ -1,27 +0,0 @@
-{
-  "editor.tabSize": 2,
-  "editor.insertSpaces": true,
-  "editor.renderWhitespace": "trailing",
-  "files.trimTrailingWhitespace": true,
-  "search.exclude": {
-    "**/.git": true,
-    "**/.venv": true,
-    "**/__pycache__": true
-  },
-  "files.exclude": {
-    "**/.git": true,
-    "**/.venv": true,
-    "**/__pycache__": true
-  },
-  "python.analysis.exclude": [
-    "**/.git",
-    "**/.venv",
-    "**/__pycache__",
-    // exclude directories that should be using the symlinked version
-    "common/**",
-    "selfdrive/**",
-    "system/**",
-    "third_party/**",
-    "tools/**",
-  ]
-}

Dockerfile.openpilot

@@ -1,38 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-ENV OPENPILOT_PATH=/home/$USER/openpilot
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
-
-COPY --chown=$USER . ${OPENPILOT_PATH}/
-
-ENV UV_BIN="/home/$USER/.local/bin/"
-ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
-ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
-RUN tools/setup_dependencies.sh && \
-    sudo rm -rf /var/lib/apt/lists/*
-
-USER root
-RUN git config --global --add safe.directory '*'

Jenkinsfile

@@ -1,260 +0,0 @@
-def retryWithDelay(int maxRetries, int delay, Closure body) {
-  for (int i = 0; i < maxRetries; i++) {
-    try {
-      return body()
-    } catch (Exception e) {
-      sleep(delay)
-    }
-  }
-  throw Exception("Failed after ${maxRetries} retries")
-}
-
-def device(String ip, String step_label, String cmd) {
-  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
-    def ssh_cmd = """
-ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
-
-set -e
-
-export TERM=xterm-256color
-
-shopt -s huponexit # kill all child processes when the shell exits
-
-export CI=1
-export PYTHONWARNINGS=error
-#export LOGPRINT=debug # this has gotten too spammy...
-export TEST_DIR=${env.TEST_DIR}
-export SOURCE_DIR=${env.SOURCE_DIR}
-export GIT_BRANCH=${env.GIT_BRANCH}
-export GIT_COMMIT=${env.GIT_COMMIT}
-export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
-export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
-export AZURE_TOKEN='${env.AZURE_TOKEN}'
-# only use 1 thread for tici tests since most require HIL
-export PYTEST_ADDOPTS="-n0 -s"
-
-
-export GIT_SSH_COMMAND="ssh -i /data/gitkey"
-
-source ~/.bash_profile
-if [ -f /TICI ]; then
-  source /etc/profile
-
-  rm -rf /tmp/tmp*
-  rm -rf ~/.commacache
-  rm -rf /dev/shm/*
-  rm -rf /dev/tmp/tmp*
-
-  if ! systemctl is-active --quiet systemd-resolved; then
-    echo "restarting resolved"
-    sudo systemctl start systemd-resolved
-    sleep 3
-  fi
-
-  # restart aux USB
-  if [ -e /sys/bus/usb/drivers/hub/3-0:1.0 ]; then
-    echo "restarting aux usb"
-    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/unbind
-    sleep 0.5
-    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/bind
-  fi
-fi
-if [ -f /data/openpilot/launch_env.sh ]; then
-  source /data/openpilot/launch_env.sh
-fi
-
-ln -snf ${env.TEST_DIR} /data/pythonpath
-
-cd ${env.TEST_DIR} || true
-time ${cmd}
-END"""
-
-    sh script: ssh_cmd, label: step_label
-  }
-}
-
-def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
-  stage(stageName) {
-    if (currentBuild.result != null) {
-        return
-    }
-
-    if (isReplay()) {
-      error("REPLAYING TESTS IS NOT ALLOWED. FIX THEM INSTEAD.")
-    }
-
-    def extra = extra_env.collect { "export ${it}" }.join('\n');
-    def branch = env.BRANCH_NAME ?: 'master';
-    def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes'
-
-    lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
-      docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
-        timeout(time: 35, unit: 'MINUTES') {
-          retry (3) {
-            def date = sh(script: 'date', returnStdout: true).trim();
-            device(device_ip, "set time", "date -s '" + date + "'")
-            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
-          }
-          steps.each { item ->
-            def name = item[0]
-            def cmd = item[1]
-
-            def args = item[2]
-            def diffPaths = args.diffPaths ?: []
-            def cmdTimeout = args.timeout ?: 9999
-
-            if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
-              println "Skipping ${name}: no changes in ${diffPaths}."
-              return
-            } else {
-              timeout(time: cmdTimeout, unit: 'SECONDS') {
-                device(device_ip, name, cmd)
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}
-
-def hasPathChanged(String gitDiff, List<String> paths) {
-  for (path in paths) {
-    if (gitDiff.contains(path)) {
-      return true
-    }
-  }
-  return false
-}
-
-def isReplay() {
-  def replayClass = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause"
-  return currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClass) }
-}
-
-def setupCredentials() {
-  withCredentials([
-    string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
-  ]) {
-    env.AZURE_TOKEN = "${AZURE_TOKEN}"
-  }
-
-  withCredentials([
-    string(credentialsId: 'ci_artifacts_pat', variable: 'CI_ARTIFACTS_TOKEN'),
-  ]) {
-    env.CI_ARTIFACTS_TOKEN = "${CI_ARTIFACTS_TOKEN}"
-  }
-
-  withCredentials([
-    string(credentialsId: 'post_comments_github_pat', variable: 'GITHUB_COMMENTS_TOKEN'),
-  ]) {
-    env.GITHUB_COMMENTS_TOKEN = "${GITHUB_COMMENTS_TOKEN}"
-  }
-}
-
-def step(String name, String cmd, Map args = [:]) {
-  return [name, cmd, args]
-}
-
-node {
-  env.CI = "1"
-  env.PYTHONWARNINGS = "error"
-  env.TEST_DIR = "/data/openpilot"
-  env.SOURCE_DIR = "/data/openpilot_source/"
-  setupCredentials()
-
-  env.GIT_BRANCH = checkout(scm).GIT_BRANCH
-  env.GIT_COMMIT = checkout(scm).GIT_COMMIT
-
-  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tici', 'release-tizi', 'release-tizi-staging', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
-  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
-
-  if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
-    properties([
-        disableConcurrentBuilds(abortPrevious: true)
-    ])
-  }
-
-  try {
-    if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh && git push -f origin release-tizi-staging:release-mici-staging"),
-      ])
-    }
-
-    if (env.BRANCH_NAME == '__nightly') {
-      parallel (
-        'nightly': {
-          deviceStage("build nightly", "tizi-needs-can", [], [
-            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
-          ])
-        },
-        'nightly-dev': {
-          deviceStage("build nightly-dev", "tizi-needs-can", [], [
-            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
-          ])
-        },
-      )
-    }
-
-    if (!env.BRANCH_NAME.matches(excludeRegex)) {
-    parallel (
-      'onroad tests': {
-        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
-          step("build openpilot", "cd system/manager && ./build.py"),
-          step("check dirty", "release/check-dirty.sh"),
-          step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
-        ])
-      },
-      'HW + Unit Tests': {
-        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
-          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
-          step("test manager", "pytest system/manager/test/test_manager.py"),
-        ])
-      },
-      'camerad OX03C10': {
-        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
-        ])
-      },
-      'camerad OS04C10': {
-        deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
-        ])
-      },
-      'sensord': {
-        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
-        ])
-      },
-      'replay': {
-        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
-          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
-        ])
-      },
-      'tizi': {
-        deviceStage("tizi", "tizi", ["UNSAFE=1"], [
-          step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
-          step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
-          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
-        ])
-      },
-
-    )
-    }
-  } catch (Exception e) {
-    currentBuild.result = 'FAILED'
-    throw e
-  }
-}

LICENSE

@@ -1,7 +0,0 @@
-Copyright (c) 2018, Comma.ai, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -1,33 +1,9 @@
-<div align="center" style="text-align: center;">
+[![openpilot on the comma 3X](https://i.imgur.com/6l2qbf5.png)](https://comma.ai/shop/comma-3x)
 
-<h1>openpilot</h1>
+What is openpilot?
+------
 
-<p>
-  <b>openpilot is an operating system for robotics.</b>
-  <br>
-  Currently, it upgrades the driver assistance system in 300+ supported cars.
-</p>
-
-<h3>
-  <a href="https://docs.comma.ai">Docs</a>
-  <span> · </span>
-  <a href="https://docs.comma.ai/contributing/roadmap/">Roadmap</a>
-  <span> · </span>
-  <a href="https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md">Contribute</a>
-  <span> · </span>
-  <a href="https://discord.comma.ai">Community</a>
-  <span> · </span>
-  <a href="https://comma.ai/shop">Try it on a comma four</a>
-</h3>
-
-Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
-
-[![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/tests.yaml/badge.svg)](https://github.com/commaai/openpilot/actions/workflows/tests.yaml)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
-[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
-[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
-
-</div>
+[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](docs/INTEGRATION.md) and [limitations](docs/LIMITATIONS.md).
 
 <table>
   <tr>
@@ -37,56 +13,204 @@ Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
   </tr>
 </table>
 
-
-Using openpilot in a car
+What is FrogPilot? 🐸
 ------
 
-To use openpilot in a car, you need four things:
-1. **Supported Device:** a comma four, available at [comma.ai/shop/comma-four](https://www.comma.ai/shop/comma-four).
-2. **Software:** The setup procedure for the comma four allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
-3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
-4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma four to your car.
+FrogPilot is a fully open-sourced fork of openpilot, featuring clear and concise commits striving to be a resource for the openpilot developer community. It thrives on contributions from both users and developers, focusing on a collaborative, community-led approach to deliver an advanced openpilot experience for everyone!
 
-We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
+------
+FrogPilot was last updated on:
 
+**July 1st, 2024**
 
-### Branches
-
-Running `master` and other branches directly is supported, but it's recommended to run one of the following prebuilt branches:
-
-| comma four branch      | comma 3X branch        | URL                                    | description                                                                         |
-|------------------------|------------------------|----------------------------------------|-------------------------------------------------------------------------------------|
-| `release-mici`         | `release-tizi`         | openpilot.comma.ai                     | This is openpilot's release branch.                                                 |
-| `release-mici-staging` | `release-tizi-staging` | openpilot-test.comma.ai                | This is the staging branch for releases. Use it to get new releases slightly early. |
-| `nightly`              | `nightly`              | openpilot-nightly.comma.ai             | This is the bleeding edge development branch. Do not expect this to be stable.      |
-| `nightly-dev`          | `nightly-dev`          | installer.comma.ai/commaai/nightly-dev | Same as nightly, but includes experimental development features for some cars.      |
-
-To start developing openpilot
+Features
 ------
 
-openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot).
+FrogPilot offers a wide range of customizable features that are easily toggled on or off to suit your preferences. Whether you want a completely stock openpilot experience, or want to add some fun and personal touches, FrogPilot has you covered! Some of the features include:
 
-* Join the [community Discord](https://discord.comma.ai)
-* Check out [the contributing docs](docs/CONTRIBUTING.md)
-* Check out the [openpilot tools](tools/)
-* Code documentation lives at https://docs.comma.ai
-* Information about running openpilot lives on the [community wiki](https://github.com/commaai/openpilot/wiki)
+------
+⚡ **Advanced Customizations:**
 
-Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs#open-positions) and offers lots of [bounties](https://comma.ai/bounties) for external contributors.
+  - "Alert Volume Controller" to set the volume level for each of of openpilot's sounds
+  - Customize the following distance and jerk values for each personality profile
+  - Fine tune your car's "Steer Ratio" to perfect your lateral control
+  - Increase the distance when stopped behind lead vehicles
+  - Increase the max set speed by a custom interval (i.e. 2, 3, 4, 5, 6, etc. instead of just 1)
+  - Select between past, present, and future openpilot driving models
+------
+🎨 **Custom Themes:**
 
-Safety and Testing
-----
+  - 🐸 Frog theme (with a bonus 🐐 sound effect)
+  - <img src="https://images.emojiterra.com/google/noto-emoji/unicode-15/color/512px/1f1f7-1f1fa.png" width="15" height="12"> Russia / Joseph Stalin theme
+  - 🔌 Tesla theme
+  - Holiday themes! Minor holidays last a day, while major holidays (Easter, Halloween, Thanksgiving, Christmas) last a week
+  - Random events triggered by specific actions while driving with openpilot
 
-* openpilot observes [ISO26262](https://en.wikipedia.org/wiki/ISO_26262) guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
-* openpilot has software-in-the-loop [tests](.github/workflows/tests.yaml) that run on every commit.
-* The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
-* panda has software-in-the-loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
-* Internally, we have a hardware-in-the-loop Jenkins test suite that builds and unit tests the various processes.
-* panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
-* We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
+  - 📢 Want to add a theme? Request one in the "feature-request" channel in the FrogPilot Discord!
+------
+🚀 **Conditional Experimental Mode:**
 
-<details>
-<summary>MIT Licensed</summary>
+  - Auto-activates "Experimental Mode" under several conditions, including:
+    - Approaching intersections and turns while using navigation
+    - Approaching slower vehicles to take advantage of "Experimental Mode"'s smoother braking
+    - Curve and stop light/stop sign detection
+    - Driving below a set speed
+    - Turn signal activation below 55mph for turn assistance
+------
+📊 **Developer UI:**
+
+  - Display various driving logics such as the distance, speed, and the desired following distance to your lead vehicle
+  - Lane measuring of the adjacent lanes for lane detection
+  - Tap the "VEHICLE ONLINE"/"CPU"/"GPU" gauge to toggle between CPU and GPU monitoring
+  - Tap the "CONNECT ONLINE"/"MEMORY"/"LEFT"/"USED" gauge to toggle between RAM and storage monitoring
+------
+🛠 **Device Management:**
+
+  - Adjustable screen brightness for both onroad and offroad states
+  - Adjustable screen timeout times for both onroad and offroad states
+  - Backup and restore previous versions of FrogPilot
+  - Backup and restore previous versions of toggle configurations
+  - Battery level threshold to automatically shut the device down after you car's battery falls below a set voltage limit when offroad
+  - Delete stored driving data for increased privacy/space via the "Device" panel
+  - Device can operate offline indefinitely
+  - Disable logging and/or uploading
+  - Disable uploads while onroad to help reduce data usage
+  - Flash the Panda within the "Device" menu
+  - "Standby Mode" that wakes the screen up between engagement states or when important alerts are triggered
+  - Timer to automatically shut down after going offroad
+------
+🚖 **Lateral Adjustments:**
+
+  - Activate lateral control by simply pressing the "Cruise Control" button
+  - Force comma's auto tuning for unsupported vehicles
+  - Lateral control won't disengage on gas or brake
+  - Nudgeless lane changes with lane detection to prevent driving into curbs or going offroad
+  - Pause lateral control when below a set speed
+  - Pause lateral control when pressing the brake
+  - Pause lateral control when turn signals are active
+  - Precise turns by using turn desires when below the minimum lane change speed
+  - [Twilsonco's NNFF](https://github.com/twilsonco/openpilot) for smoother steering control
+------
+🚘 **Longitudinal Adjustments:**
+
+  - Aggressive acceleration when following a lead vehicle from a stop
+  - "Map Turn Speed Controller" to slow down for curves based on stored map data
+    - With an additional toggle to fine tune the speed aggressiveness
+  - Smoother braking behind slower leads
+  - "Speed Limit Controller" to adjust your speed to the posted speed limit
+    - With additional toggles to set offsets for "0-34 mph", "35-54 mph", "55-64 mph", and "65-99 mph"
+  - "Sport" and "Eco" acceleration and deceleration profiles
+  - "Traffic Mode" tailored towards driving in traffic
+  - Tweak the lead detection threshold to detect leads sooner for smoother braking on stopped/slower vehicles
+  - "Vision Turn Speed Controller" for smoother handling of curves
+    - With additional toggles to fine tune the speed aggressiveness and curve detection sensitivity
+------
+🗺️ **Navigation:**
+
+  - 3D buildings
+  - Custom map styles
+  - Full screen map that takes up the entire screen for a more expansive map view
+  - iOS shortcuts to quickly set navigation destinations
+  - Navigate on openpilot without a comma prime subscription
+  - Offline maps
+  - "Open Street Maps" integration for speed limit control and road name view
+------
+🎮 **Onroad UI:**
+
+  - Compass that rotates according to the direction you're driving
+  - FPS counter in the screen's border
+  - Hide various UI elements on the screen for a cleaner UI
+  - Pedals on the onroad UI indicate when the gas/brake pedals are being used
+  - Road UI Customizations:
+    - Acceleration path to show the model's desired acceleration/deceleration
+    - Blind spot path when a vehicle is detected in your blind spot
+    - Increase/decrease the lane line, path, and road edge widths
+    - Path edge colors based on specific driving statuses:
+      - 🔵 Blue - Navigation active
+      - 🟦 Light Blue - "Always On Lateral" active
+      - 🟢 Green - Default
+      - 🟠 Orange - "Experimental Mode" active
+      - 🔴 Red - "Traffic Mode" active
+      - 🟡 Yellow - "Conditional Experimental Mode" overridden
+    - "Unlimited" road UI that extends out as far as the model can see
+  - Sidebar retains it's previous position between reboots/ignition cycles
+  - Steering wheel icons
+    - 📢 Request your own steering wheel icon in the "feature-request" channel!
+  - Steering wheel in the onroad UI rotates alongside your physical steering wheel
+------
+🚙 **Vehicle Specific Additions:**
+
+  - Automatic/manual fingerprint selection to force a selected fingerprint
+  - Custom longitudinal tunings for GM EVs and trucks for smoother gas and brake control
+  - Custom longitudinal tunings for Toyota/Lexus for smoother gas and brake control
+  - Disable openpilot longitudinal control to use your car's stock cruise control
+  - GM Volt support
+  - Honda Clarity support
+  - Increased torque for the Subaru Crosstrek
+  - Lock doors automatically when in the drive gear for Toyota/Lexus and unlock when in park
+  - openpilot longitudinal control for GM vehicles without ACC
+  - Pedal interceptor support for GM vehicles
+  - "Stop and Go" hack for Toyota's without stop and go functionality
+  - ZSS support for the Toyota Prius and Sienna
+------
+🚦 **Quality of Life Features:**
+
+  - Automatic updates for a completely "set and forget" experience
+  - Camera view selection
+  - Custom alerts for green lights, vehicles in blindspot, lead departing, and the current speed limit changing
+  - Display the driver camera when in reverse
+  - Driving statistics to show how many hours and miles you've driven on the home screen
+  - Fleet Manager to easily access your driving data and screen recordings
+  - Numerical temperature gauge
+  - Retain tethering status between reboots
+  - Screenrecorder
+  - Toggle "Experimental Mode" via the "Lane Departure Alert" button, holding down the "Distance" button for 0.5+ seconds, or by double tapping the screen
+
+How to Install
+------
+
+Easiest way to install FrogPilot is via this URL at the installation screen:
+
+```
+frogpilot.download
+```
+
+DO NOT install the "FrogPilot-Development" branch. I'm constantly breaking things on there, so unless you don't want to use openpilot, NEVER install it!
+
+![](https://i.imgur.com/swr0kqJ.png)
+
+Bug reports / Feature Requests
+------
+
+If you encounter any issues or bugs while using FrogPilot, or if you have any suggestions for new features or improvements, please don't hesitate to post about it on the Discord! I'm always looking for ways to improve the fork and provide a better experience for everyone!
+
+To report a bug or request a new feature, make a post in the #bug-reports or #feature-requests channel respectively on the FrogPilot Discord. Please provide as much detail as possible about the issue you're experiencing or the feature you'd like to see added. Photos, videos, log files, or other relevant information are very helpful!
+
+I will do my best to respond to bug reports and feature requests in a timely manner, but please understand that I may not be able to address every request immediately. Your feedback and suggestions are valuable, and I appreciate your help in making FrogPilot the best it can be!
+
+Discord
+------
+
+[Join the FrogPilot Community Discord!](https://discord.gg/frogpilot)
+
+Credits
+------
+
+* [AlexandreSato](https://github.com/AlexandreSato)
+* [Crwusiz](https://github.com/crwusiz)
+* [DragonPilot](https://github.com/dragonpilot-community)
+* [ErichMoraga](https://github.com/ErichMoraga)
+* [Garrettpall](https://github.com/garrettpall)
+* [Mike8643](https://github.com/mike8643)
+* [Neokii](https://github.com/Neokii)
+* [OPGM](https://github.com/opgm)
+* [OPKR](https://github.com/openpilotkr)
+* [Pfeiferj](https://github.com/pfeiferj)
+* [ServerDummy](https://github.com/ServerDummy)
+* [Twilsonco](https://github.com/twilsonco)
+
+Licensing
+------
 
 openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
@@ -95,17 +219,10 @@ Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and i
 **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
 YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
 NO WARRANTY EXPRESSED OR IMPLIED.**
-</details>
 
-<details>
-<summary>User Data and comma Account</summary>
+---
 
-By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
+<img src="https://d1qb2nb5cznatu.cloudfront.net/startups/i/1061157-bc7e9bf3b246ece7322e6ffe653f6af8-medium_jpg.jpg?buster=1458363130" width="75"></img> <img src="https://cdn-images-1.medium.com/max/1600/1*C87EjxGeMPrkTuVRVWVg4w.png" width="225"></img>
 
-openpilot is open source software: the user is free to disable data collection if they wish to do so.
-
-openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
-
-By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
-</details>
+![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)

RELEASES.md

@@ -1,89 +1,3 @@
-Version 0.11.1 (2026-04-22)
-========================
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-Version 0.11.0 (2026-03-17)
-========================
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-
-Version 0.10.3 (2025-12-17)
-========================
-* New driving model #36249
-  * New temporal policy architecture
-  * New on-policy training physics noise model
-* New driver monitoring model #36409
-  * Trained on a new dataset, including comma four data
-* Improved inter-process communication memory efficiency
-
-Version 0.10.2 (2025-11-19)
-========================
-* comma four support
-
-Version 0.10.1 (2025-09-08)
-========================
-* New driving model #36276
-  * World Model: removed global localization inputs
-  * World Model: 2x the number of parameters
-  * World Model: trained on 4x the number of segments
-  * VAE Compression Model: new architecture and training objective
-  * Driving Vision Model: trained on 4x the number of segments
-* New Driver Monitoring model #36198
-* Acura TLX 2021 support thanks to MVL!
-* Honda City 2023 support thanks to vanillagorillaa and drFritz!
-* Honda N-Box 2018 support thanks to miettal!
-* Honda Odyssey 2021-25 support thanks to csouers and MVL!
-* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
-
-Version 0.10.0 (2025-08-05)
-========================
-* New driving model
-  * New training architecture
-     * Described in our CVPR paper: "Learning to Drive from a World Model"
-     * Longitudinal MPC replaced by E2E planning from World Model in Experimental Mode
-     * Action from lateral MPC as training objective replaced by E2E planning from World Model
-  * Low-speed lead car ground-truth fixes
-* Enable live-learned steering actuation delay
-* Opt-in audio recording for dashcam video
-* Acura MDX 2025 support thanks to vanillagorillaa and MVL!
-* Honda Accord 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda CR-V 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda Pilot 2023-25 support thanks to vanillagorillaa and MVL!
-
-Version 0.9.9 (2025-05-23)
-========================
-* New driving model
-  * New training architecture using parts from MLSIM
-* Steering actuation delay is now learned online
-* Ford Escape 2023-24 support thanks to incognitojam!
-* Ford Kuga 2024 support thanks to incognitojam!
-* Hyundai Nexo 2021 support thanks to sunnyhaibin!
-* Tesla Model 3 and Y support thanks to lukasloetkolben!
-* Lexus RC 2023 support thanks to nelsonjchen!
-
-Version 0.9.8 (2025-02-28)
-========================
-* New driving model
-  * Model now gates applying positive acceleration in Chill mode
-* New driver monitoring model
-  * Reduced false positives related to passengers
-* Image processing pipeline moved to the ISP
-  * More GPU time for bigger driving models
-  * Power draw reduced 0.5W, which means your device runs cooler
-* Added toggle to enable driver monitoring even when openpilot is not engaged
-* Localizer rewritten to remove GPS dependency at runtime
-* Firehose Mode for maximizing your training data uploads
-* Enable openpilot longitudinal control for Ford Q3 vehicles
-* New Toyota TSS2 longitudinal tune
-* Rivian R1S and R1T support thanks to lukasloetkolben!
-* Ford F-150, F-150 Hybrid, Mach-E, and Ranger support
-
 Version 0.9.7 (2024-06-13)
 ========================
 * New driving model

SConstruct

@@ -3,240 +3,388 @@ import subprocess
 import sys
 import sysconfig
 import platform
-import shlex
-import importlib
 import numpy as np
 
 import SCons.Errors
-from SCons.Defaults import _stripixes
 
 SCons.Warnings.warningAsException(True)
 
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
+TICI = os.path.isfile('/TICI')
+AGNOS = TICI
+
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+SetOption('num_jobs', int(os.cpu_count()/2))
+
+AddOption('--kaitai',
+          action='store_true',
+          help='Regenerate kaitai struct parsers')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--compile_db',
+          action='store_true',
+          help='build clang compilation database')
+
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
+
+AddOption('--snpe',
+          action='store_true',
+          help='use SNPE on PC')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+AddOption('--pc-thneed',
+          action='store_true',
+          dest='pc_thneed',
+          help='use thneed on pc')
 
-AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
-AddOption('--verbose', action='store_true', default=False, help='show full build commands')
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-# Detect platform
-arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+## Architecture name breakdown (arch)
+## - larch64: linux tici aarch64
+## - aarch64: linux pc aarch64
+## - x86_64:  linux pc x64
+## - Darwin:  mac x64 or arm64
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and os.path.isfile('/TICI'):
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and AGNOS:
   arch = "larch64"
-assert arch in [
-  "larch64",  # linux tici arm64
-  "aarch64",  # linux pc arm64
-  "x86_64",   # linux pc x64
-  "Darwin",   # macOS arm64 (x86 not supported)
-]
+assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
 
-pkg_names = ['bzip2', 'capnproto', 'eigen', 'ffmpeg', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
-pkgs = [importlib.import_module(name) for name in pkg_names]
+lenv = {
+  "PATH": os.environ['PATH'],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
 
-
-# ***** enforce a whitelist of system libraries *****
-# this prevents silently relying on a 3rd party package,
-# e.g. apt-installed libusb. all libraries should either
-# be distributed with all Linux distros and macOS, or
-# vendored in commaai/dependencies.
-allowed_system_libs = {
-  "EGL", "GLESv2", "GL",
-  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
-  "dl", "drm", "gbm", "m", "pthread",
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
 
-def _resolve_lib(env, name):
-  for d in env.Flatten(env.get('LIBPATH', [])):
-    p = Dir(str(d)).abspath
-    for ext in ('.a', '.so', '.dylib'):
-      f = File(os.path.join(p, f'lib{name}{ext}'))
-      if f.exists() or f.has_builder():
-        return name
-  if name in allowed_system_libs:
-    return name
-  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
+rpath = lenv["LD_LIBRARY_PATH"].copy()
 
-def _libflags(target, source, env, for_signature):
-  libs = []
-  lp = env.subst('$LIBLITERALPREFIX')
-  for lib in env.Flatten(env.get('LIBS', [])):
-    if isinstance(lib, str):
-      if os.sep in lib or lib.startswith('#'):
-        libs.append(File(lib))
-      elif lib.startswith('-') or (lp and lib.startswith(lp)):
-        libs.append(lib)
-      else:
-        libs.append(_resolve_lib(env, lib))
-    else:
-      libs.append(lib)
-  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
-                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
+if arch == "larch64":
+  cpppath = [
+    "#third_party/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/local/lib",
+    "/usr/lib",
+    "/system/vendor/lib64",
+    f"#third_party/acados/{arch}/lib",
+  ]
+
+  libpath += [
+    "#third_party/snpe/larch64",
+    "#third_party/libyuv/larch64/lib",
+    "/usr/lib/aarch64-linux-gnu"
+  ]
+  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  rpath += ["/usr/local/lib"]
+else:
+  cflags = []
+  cxxflags = []
+  cpppath = []
+  rpath += []
+
+  # MacOS
+  if arch == "Darwin":
+    libpath = [
+      f"#third_party/libyuv/{arch}/lib",
+      f"#third_party/acados/{arch}/lib",
+      f"{brew_prefix}/lib",
+      f"{brew_prefix}/opt/openssl@3.0/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += [
+      f"{brew_prefix}/include",
+      f"{brew_prefix}/opt/openssl@3.0/include",
+    ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
+  # Linux
+  else:
+    libpath = [
+      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{arch}/lib",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
+
+if GetOption('asan'):
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
+else:
+  ccflags = []
+  ldflags = []
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
+
+# Enable swaglog include in submodules
+cflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
+cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
+
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
 
 env = Environment(
-  ENV={
-    "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
-  },
+  ENV=lenv,
   CCFLAGS=[
     "-g",
     "-fPIC",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
+    "-Wshadow",
     "-Wno-unknown-warning-option",
+    "-Wno-deprecated-register",
+    "-Wno-register",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
+    "-Wno-error=unused-but-set-variable",
     "-Wno-vla-cxx-extension",
-  ],
-  CFLAGS=["-std=gnu11"],
-  CXXFLAGS=["-std=c++1z"],
-  CPPPATH=[
+  ] + cflags + ccflags,
+
+  CPPPATH=cpppath + [
     "#",
-    "#msgq",
-    "#third_party",
-    "#third_party/json11",
-    "#third_party/linux/include",
     "#third_party/acados/include",
     "#third_party/acados/include/blasfeo/include",
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
-    [x.INCLUDE_DIR for x in pkgs],
+    "#third_party/libyuv/include",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/snpe/include",
+    "#third_party/qrcode",
+    "#third_party",
+    "#cereal",
+    "#msgq",
+    "#opendbc/can",
+    "#third_party/maplibre-native-qt/include",
+    f"#third_party/maplibre-native-qt/{arch}/include"
   ],
-  LIBPATH=[
-    "#common",
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
+  LIBPATH=libpath + [
     "#msgq_repo",
     "#third_party",
     "#selfdrive/pandad",
+    "#common",
     "#rednose/helpers",
-    f"#third_party/acados/{arch}/lib",
-    [x.LIB_DIR for x in pkgs],
   ],
-  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
-  toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
+  toolpath=["#rednose_repo/site_scons/site_tools"],
 )
-if arch != "larch64":
-  env['_LIBFLAGS'] = _libflags
 
-# Arch-specific flags and paths
-if arch == "larch64":
-  env["CC"] = "clang"
-  env["CXX"] = "clang++"
-  env.Append(LIBPATH=[
-    "/usr/lib/aarch64-linux-gnu",
-  ])
-  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57"]
-  env.Append(CCFLAGS=arch_flags)
-  env.Append(CXXFLAGS=arch_flags)
-elif arch == "Darwin":
-  env.Append(LIBPATH=[
-    "/System/Library/Frameworks/OpenGL.framework/Libraries",
-  ])
-  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
-  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+if arch == "Darwin":
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
 
-_extra_cc = shlex.split(GetOption('ccflags') or '')
-if _extra_cc:
-  env.Append(CCFLAGS=_extra_cc)
+if GetOption('compile_db'):
+  env.CompilationDatabase('compile_commands.json')
 
-# no --as-needed on mac linker
-if arch != "Darwin":
-  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
+# Setup cache dir
+cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
 
-# Shorter build output: show brief descriptions instead of full commands.
-# Full command lines are still printed on failure by scons.
-if not GetOption('verbose'):
-  for action, short in (
-    ("CC",     "CC"),
-    ("CXX",    "CXX"),
-    ("LINK",   "LINK"),
-    ("SHCC",   "CC"),
-    ("SHCXX",  "CXX"),
-    ("SHLINK", "LINK"),
-    ("AR",     "AR"),
-    ("RANLIB", "RANLIB"),
-    ("AS",     "AS"),
-  ):
-    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
-
-# progress output
 node_interval = 5
 node_count = 0
 def progress_function(node):
   global node_count
   node_count += node_interval
   sys.stderr.write("progress: %d\n" % node_count)
+
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-# ********** Cython build environment **********
+# Cython build environment
+py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
-envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
-np_version = SCons.Script.Value(np.__version__)
-Export('envCython', 'np_version')
+Export('envCython')
 
-Export('env', 'arch')
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus", "Xml"]
 
-# Setup cache dir
-cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+qt_libs = []
+if arch == "Darwin":
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
+else:
+  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
+  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
 
-# ********** start building stuff **********
+  qt_env['QTDIR'] = qt_install_prefix
+  qt_dirs = [
+    f"{qt_install_headers}",
+  ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
+  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+qt_env['QT3DIR'] = qt_env['QTDIR']
+
+# compatibility for older SCons versions
+try:
+  qt_env.Tool('qt3')
+except SCons.Errors.UserError:
+  qt_env.Tool('qt')
+
+qt_env['CPPPATH'] += qt_dirs# + ["#selfdrive/ui/qt/"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_QUICK_LIB",
+  "-DQT_QUICKWIDGETS_LIB",
+  "-DQT_QML_LIB",
+  "-DQT_CORE_LIB",
+  "-DQT_MESSAGELOGCONTEXT",
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui', f"#third_party/maplibre-native-qt/{arch}/lib"]
+qt_env['RPATH'] += [Dir(f"#third_party/maplibre-native-qt/{arch}/lib").srcnode().abspath]
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+
+Export('env', 'qt_env', 'arch', 'real_arch')
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common')
-common = [_common, 'json11', 'zmq']
-Export('common')
+Import('_common', '_gpucommon')
+
+common = [_common, 'json11']
+gpucommon = [_gpucommon]
+
+Export('common', 'gpucommon')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
-# Enable swaglog include in submodules
-env_swaglog = env.Clone()
-env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
-SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
-
+SConscript(['msgq_repo/SConscript'])
 SConscript(['cereal/SConscript'])
-
 Import('socketmaster', 'msgq')
-messaging = [socketmaster, msgq, 'capnp', 'kj',]
+messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
 Export('messaging')
 
 
 # Build other submodules
-SConscript(['panda/SConscript'])
+SConscript([
+  'body/board/SConscript',
+  'opendbc/can/SConscript',
+  'panda/SConscript',
+])
 
 # Build rednose library
 SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
+  'system/proclogd/SConscript',
+  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
+if arch != "Darwin":
+  SConscript([
+    'system/sensord/SConscript',
+    'system/logcatd/SConscript',
+  ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
@@ -244,23 +392,13 @@ if arch == "larch64":
 # Build openpilot
 SConscript(['third_party/SConscript'])
 
-# Build selfdrive
-SConscript([
-  'selfdrive/pandad/SConscript',
-  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
-  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'selfdrive/locationd/SConscript',
-  'selfdrive/modeld/SConscript',
-  'selfdrive/ui/SConscript',
-])
+SConscript(['selfdrive/SConscript'])
 
-# Build tools
-if arch != "larch64":
-  SConscript([
-    'tools/replay/SConscript',
-    'tools/cabana/SConscript',
-    'tools/jotpluggler/SConscript',
-  ])
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
+  SConscript(['tools/replay/SConscript'])
+  if arch != "larch64":
+    SConscript(['tools/cabana/SConscript'])
 
-
-env.CompilationDatabase('compile_commands.json')
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

SECURITY.md

@@ -1,5 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-Suspected vulnerabilities can be reported to both `adeeb@comma.ai` and `security@comma.ai`.

body/.gitignore

@@ -0,0 +1,15 @@
+*.pyc
+.*.swp
+.*.swo
+*.o
+*.so
+*.os
+*.d
+*.dump
+a.out
+*~
+.#*
+.vscode*
+.DS_Store
+board/obj/
+.sconsign.dblite

body/SConstruct

@@ -0,0 +1 @@
+SConscript('board/SConscript')

body/board/SConscript

@@ -0,0 +1,197 @@
+import os
+import subprocess
+
+PREFIX = "arm-none-eabi-"
+BUILDER = "DEV"
+
+common_flags = []
+build_projects = {}
+
+build_projects["body"] = {
+  "MAIN": "main.c",
+  "STARTUP_FILE": "startup_stm32f413xx.s",
+  "LINKER_SCRIPT": "stm32fx_flash.ld",
+  "APP_START_ADDRESS": "0x08004000",
+  "PROJECT_FLAGS": [
+    "-mcpu=cortex-m4",
+    "-mhard-float",
+    "-DSTM32F4",
+    "-DSTM32F413xx",
+    "-mfpu=fpv4-sp-d16",
+    "-fsingle-precision-constant",
+    "-Os",
+    "-g",
+  ],
+}
+
+if os.getenv("RELEASE"):
+  BUILD_TYPE = "RELEASE"
+  cert_fn = os.getenv("CERT")
+  assert cert_fn is not None, 'No certificate file specified. Please set CERT env variable'
+  assert os.path.exists(cert_fn), 'Certificate file not found. Please specify absolute path'
+else:
+  BUILD_TYPE = "DEBUG"
+  cert_fn = File("../certs/debug").srcnode().relpath
+  common_flags += ["-DALLOW_DEBUG"]
+
+if os.getenv("DEBUG"):
+  common_flags += ["-DDEBUG"]
+
+includes = [
+  "inc/STM32F4xx_HAL_Driver/Inc",
+  "inc",
+  "..",
+  ".",
+]
+
+c_sources = [
+  ["hal_flash", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_flash.c"],
+  ["hal_pwr", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_pwr.c"],
+  ["hal_rcc", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_rcc.c"],
+  ["hal_i2c", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_i2c.c"],
+  ["hal_i2c_ex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_i2c_ex.c"],
+  ["hal_tim", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_tim.c"],
+  ["hal_tim_ex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_tim_ex.c"],
+  ["hal_adc_ex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_adc_ex.c"],
+  ["hal_cortex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_cortex.c"],
+  ["hal_flash_ex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_flash_ex.c"],
+  ["hal_gpio", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_gpio.c"],
+  ["hal_rcc_ex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_rcc_ex.c"],
+  ["hal", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal.c"],
+  ["hal_adc", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_adc.c"],
+  ["hal_dma", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_dma.c"],
+  ["system", "inc/system_stm32f4xx.c"],
+  ["it", "inc/stm32f4xx_it.c"],
+  ["bldc", "bldc/bldc.c"],
+  ["bldc_data", "bldc/BLDC_controller_data.c"],
+  ["bldc_con", "bldc/BLDC_controller.c"],
+  ["util", "util.c"],
+]
+
+c_bstub_sources = [
+  ["hal_pwr", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_pwr.c"],
+  ["hal_rcc", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_rcc.c"],
+  ["hal_cortex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_cortex.c"],
+  ["hal_gpio", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_gpio.c"],
+  ["hal_rcc_ex", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal_rcc_ex.c"],
+  ["hal", "inc/STM32F4xx_HAL_Driver/Src/stm32f4xx_hal.c"],
+  ["system", "inc/system_stm32f4xx.c"],
+  ["it", "inc/stm32f4xx_it.c"],
+  ["util", "util.c"],
+]
+
+def get_version(builder, build_type):
+  try:
+    git = subprocess.check_output(["git", "rev-parse", "--short=8", "HEAD"], encoding='utf8').strip()
+  except subprocess.CalledProcessError:
+    git = "00000000"
+  return f"{git}"
+
+
+def to_c_uint32(x):
+  nums = []
+  for _ in range(0x20):
+    nums.append(x % (2**32))
+    x //= (2**32)
+  return "{" + 'U,'.join(map(str, nums)) + "U}"
+
+
+def get_key_header(name):
+  from Crypto.PublicKey import RSA
+
+  public_fn = f'../certs/{name}.pub'
+  rsa = RSA.importKey(open(public_fn).read())
+  assert(rsa.size_in_bits() == 1024)
+
+  rr = pow(2**1024, 2, rsa.n)
+  n0inv = 2**32 - pow(rsa.n, -1, 2**32)
+
+  r = [
+    f"RSAPublicKey {name}_rsa_key = {{",
+    f"  .len = 0x20,",
+    f"  .n0inv = {n0inv}U,",
+    f"  .n = {to_c_uint32(rsa.n)},",
+    f"  .rr = {to_c_uint32(rr)},",
+    f"  .exponent = {rsa.e},",
+    f"}};",
+  ]
+  return r
+
+def objcopy(source, target, env, for_signature):
+    return '$OBJCOPY -O binary %s %s' % (source[0], target[0])
+
+# Common autogenerated includes
+git_ver = get_version(BUILDER, BUILD_TYPE)
+with open("obj/gitversion.h", "w") as f:
+  f.write(f'const uint8_t gitversion[8] = "{git_ver}";\n')
+
+certs = [get_key_header(n) for n in ["debug", "release"]]
+with open("obj/cert.h", "w") as f:
+  for cert in certs:
+    f.write("\n".join(cert) + "\n")
+
+for project_name in build_projects:
+  project = build_projects[project_name]
+  linkerscript_fn = File(project["LINKER_SCRIPT"]).srcnode().relpath
+
+  flags = [
+    "-Wall",
+    "-Wextra",
+    "-Wstrict-prototypes",
+    "-Werror",
+    "-mlittle-endian",
+    "-mthumb",
+    "-nostdlib",
+    "-fno-builtin",
+    f"-T{linkerscript_fn}",
+    "-std=gnu11",
+    "-fdata-sections",
+    "-ffunction-sections",
+    "-Wl,--gc-sections",
+  ] + project["PROJECT_FLAGS"] + common_flags
+
+  project_env = Environment(
+    ENV=os.environ,
+    CC=PREFIX + 'gcc',
+    AS=PREFIX + 'gcc',
+    OBJCOPY=PREFIX + 'objcopy',
+    OBJDUMP=PREFIX + 'objdump',
+    ASCOM="$AS $ASFLAGS -o $TARGET -c $SOURCES",
+    CFLAGS=flags,
+    ASFLAGS=flags,
+    LINKFLAGS=flags,
+    LIBS = ['gcc',],
+    CPPPATH=includes,
+    BUILDERS={
+      'Objcopy': Builder(generator=objcopy, suffix='.bin', src_suffix='.elf')
+    }
+  )
+  startup = project_env.Object(project["STARTUP_FILE"])
+
+  c_bstub_obj_list = []
+  for c in c_bstub_sources:
+    c_bstub_obj_list.append(project_env.Object(f"{c[0]}-{project_name}", c[1]))
+
+  # Bootstub
+  crypto_obj = [
+    project_env.Object(f"rsa-{project_name}", "../crypto/rsa.c"),
+    project_env.Object(f"sha-{project_name}", "../crypto/sha.c")
+  ]
+  bootstub_obj = project_env.Object(f"bootstub-{project_name}", "bootstub.c")
+  bootstub_elf = project_env.Program(f"obj/bootstub.{project_name}.elf", [startup] + crypto_obj + c_bstub_obj_list + [bootstub_obj])
+  bootstub_bin = project_env.Objcopy(f"obj/bootstub.{project_name}.bin", bootstub_elf)
+
+  c_obj_list = []
+  for c in c_sources:
+    c_obj_list.append(project_env.Object(f"{c[0]}-{project_name}", c[1]))
+
+  # Build main
+  main_obj = project_env.Object(f"main-{project_name}", project["MAIN"])
+  obj_list = [startup, main_obj] + c_obj_list
+  main_elf = project_env.Program(f"obj/{project_name}.elf", obj_list,
+    LINKFLAGS=[f"-Wl,--section-start,.isr_vector={project['APP_START_ADDRESS']}"] + flags)
+  main_bin = project_env.Objcopy(f"obj/{project_name}.bin", main_elf)
+
+  # Sign main
+  sign_py = File("../crypto/sign.py").srcnode().relpath
+  panda_bin_signed = project_env.Command(f"obj/{project_name}.bin.signed", main_bin, f"SETLEN=1 {sign_py} $SOURCE $TARGET {cert_fn}")

body/certs/debug

@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQC948lnRo4x44Rd7Y8bQAML4aKDC4XRx958fHV8K6+FbCaP1Z42
+U2kX0yygak0LjoDutpgObmGHZA+Iz3HeUD6VGjr/teN24vPk+A95cRsjt8rgmGQ9
+6HNjaNgjR+gl1F9XxFimMzir82Xpl1ekTueJNXa7ia5HVH1nFdiksOKHGQIDAQAB
+AoGAQuPw2I6EHJLW1/eNB75e1FqhUqRGeYV8nEGDaUBCTi+wzc4kM2LijF/5QnDv
+vvht9qkfm0XK2VSoHDtnEzcVM/l1ksb68n4R/1nUooAWY6cQI7dCSk/A6yS1EJFg
+BXsgGbT/65khw9pzBW2zVtMVcVNWFayqfCO1I9WcDdA1x1kCQQDfrhoZTZNoDEUE
+JKM4fiUdWr1h3Aw8KLJFFexSWeGDwo+qqnujYcKWkHa9qaH1RG5x8Kir9s9Oi4Js
+mzKwov8fAkEA2VPJPWxJ4vVQpXle6wC1nyoL7s739yxMWFcabvkzDDhlIVBNdVJd
+gZKsFWV7QnVNdDMjn9D27FwKu3i2D+kKxwJBANp1SMojqO765MEKI1t+YDNONx6H
+cm+i85Fjuv4nCIjOEdCGVuCYDxtMFpxgO2y3HAMuHx5sm8XDnWsDHLvFRdMCQD7V
+XqWHnYUk8AAnqy2+ssQl3/VXmZG5GQmhhV74Za3u0C5ljT+SZL6FrYMyKAT67T3f
+WzllrT6BDglNyTWoZxkCQQCt0XSoGM3603GGYNt6AUlGSgtXSo/2Px7odGUtQoKA
+FH9q6FVMYpQJ38spZxIGufZJmLP8LLg6YIWJj1F+akxr
+-----END RSA PRIVATE KEY-----

body/crypto/sign.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+import os
+import sys
+import struct
+import hashlib
+from Crypto.PublicKey import RSA
+import binascii
+
+# increment this to make new hardware not run old versions
+VERSION = 2
+
+if __name__ == "__main__":
+  with open(sys.argv[3], encoding='utf8') as k:
+    rsa = RSA.importKey(k.read())
+
+  with open(sys.argv[1], "rb") as f:
+    dat = f.read()
+
+  print("signing", len(dat), "bytes")
+
+  with open(sys.argv[2], "wb") as f:
+    if os.getenv("SETLEN") is not None:
+      # add the version at the end
+      dat += b"VERS" + struct.pack("I", VERSION)
+      # add the length at the beginning
+      x = struct.pack("I", len(dat)) + dat[4:]
+      # mock signature of dat[4:]
+      dd = hashlib.sha1(dat[4:]).digest()
+    else:
+      x = dat
+      dd = hashlib.sha1(dat).digest()
+
+    print("hash:", str(binascii.hexlify(dd), "utf-8"))
+    dd = b"\x00\x01" + b"\xff" * 0x69 + b"\x00" + dd
+    rsa_out = pow(int.from_bytes(dd, byteorder='big', signed=False), rsa.d, rsa.n)
+    sig = (hex(rsa_out)[2:].rjust(0x100, '0'))
+    x += binascii.unhexlify(sig)
+    f.write(x)

body/requirements.txt

@@ -0,0 +1,5 @@
+cffi
+scons
+ruff
+pre-commit
+pycryptodome==3.9.8

cereal/README.md

@@ -1,95 +0,0 @@
-# What is cereal?
-
-cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
-
-
-## Messaging Spec
-
-You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
-
-All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
-
-### Best Practices
-
-- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
-- In the context of the message they are in, field names should be completely unambiguous.
-- All values should be easy to plot and be human-readable with minimal parsing.
-
-### Maintaining backwards-compatibility
-
-When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
-be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
-things are not. Read more details [here](https://capnproto.org/language.html).
-
-### Custom forks
-
-Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
-spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
-then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
-[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
-fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
-
-An example of compatible changes:
-```diff
-diff --git a/cereal/custom.capnp b/cereal/custom.capnp
-index 3348e859e..3365c7b98 100644
---- a/cereal/custom.capnp
-+++ b/cereal/custom.capnp
-@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
- # DO rename the structs
- # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-
--struct CustomReserved0 @0x81c2f05a394cf4af {
-+struct SteeringInfo @0x81c2f05a394cf4af {
-+  active @0 :Bool;
-+  steeringAngleDeg @1 :Float32;
-+  steeringRateDeg @2 :Float32;
-+  steeringAccelDeg @3 :Float32;
- }
-
- struct CustomReserved1 @0xaedffd8f31e7b55d {
-diff --git a/cereal/log.capnp b/cereal/log.capnp
-index 1209f3fd9..b189f58b6 100644
---- a/cereal/log.capnp
-+++ b/cereal/log.capnp
-@@ -2558,14 +2558,14 @@ struct Event {
-
-     # DO change the name of the field
-     # DON'T change anything after the "@"
--    customReservedRawData0 @124 :Data;
-+    rawCanData @124 :Data;
-     customReservedRawData1 @125 :Data;
-     customReservedRawData2 @126 :Data;
-
-     # DO change the name of the field and struct
-     # DON'T change the ID (e.g. @107)
-     # DON'T change which struct it points to
--    customReserved0 @107 :Custom.CustomReserved0;
-+    steeringInfo @107 :Custom.SteeringInfo;
-     customReserved1 @108 :Custom.CustomReserved1;
-     customReserved2 @109 :Custom.CustomReserved2;
-     customReserved3 @110 :Custom.CustomReserved3;
-```
-
----
-
-Example
----
-```python
-import cereal.messaging as messaging
-
-# in subscriber
-sm = messaging.SubMaster(['sensorEvents'])
-while 1:
-  sm.update()
-  print(sm['sensorEvents'])
-
-```
-
-```python
-# in publisher
-pm = messaging.PubMaster(['sensorEvents'])
-dat = messaging.new_message('sensorEvents', size=1)
-dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
-pm.send('sensorEvents', dat)
-```

cereal/SConscript

@@ -1,20 +1,31 @@
-Import('env', 'common', 'msgq')
+Import('env', 'envCython', 'arch', 'common', 'msgq')
+
+import shutil
 
 cereal_dir = Dir('.')
 gen_dir = Dir('gen')
+other_dir = Dir('#msgq')
 
 # Build cereal
 schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
+env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
 env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
             schema_files,
             f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
 
-cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
+# TODO: remove non shared cereal and messaging
+cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files])
+
+cereal = env.Library('cereal', cereal_objects)
+env.SharedLibrary('cereal_shared', cereal_objects)
 
 # Build messaging
-services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
-socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[msgq, 'zmq', common])
+
+
+socketmaster = env.SharedObject(['messaging/socketmaster.cc'])
+socketmaster = env.Library('socketmaster', socketmaster)
 
 Export('cereal', 'socketmaster')

cereal/__init__.py

@@ -1,11 +1,9 @@
 import os
 import capnp
-from importlib.resources import as_file, files
 
+CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
 capnp.remove_import_hook()
 
-with as_file(files("cereal")) as fspath:
-  CEREAL_PATH = fspath.as_posix()
-  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
-  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
-  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))
+log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -1 +0,0 @@
-../opendbc_repo/opendbc/car/car.capnp

cereal/car.capnp

@@ -0,0 +1,744 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0x8e2af1e708af8b8d;
+
+# ******* events causing controls state machine transition *******
+
+struct CarEvent @0x9b1657f34caf3ad3 {
+  name @0 :EventName;
+
+  # event types
+  enable @1 :Bool;
+  noEntry @2 :Bool;
+  warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
+  userDisable @4 :Bool;
+  softDisable @5 :Bool;
+  immediateDisable @6 :Bool;
+  preEnable @7 :Bool;
+  permanent @8 :Bool; # alerts presented regardless of openpilot state
+  overrideLateral @10 :Bool;
+  overrideLongitudinal @9 :Bool;
+
+  enum EventName @0xbaa8c5d505f727de {
+    canError @0;
+    steerUnavailable @1;
+    wrongGear @4;
+    doorOpen @5;
+    seatbeltNotLatched @6;
+    espDisabled @7;
+    wrongCarMode @8;
+    steerTempUnavailable @9;
+    reverseGear @10;
+    buttonCancel @11;
+    buttonEnable @12;
+    pedalPressed @13;  # exits active state
+    preEnableStandstill @73;  # added during pre-enable state with brake
+    gasPressedOverride @108;  # added when user is pressing gas with no disengage on gas
+    steerOverride @114;
+    cruiseDisabled @14;
+    speedTooLow @17;
+    outOfSpace @18;
+    overheat @19;
+    calibrationIncomplete @20;
+    calibrationInvalid @21;
+    calibrationRecalibrating @117;
+    controlsMismatch @22;
+    pcmEnable @23;
+    pcmDisable @24;
+    radarFault @26;
+    brakeHold @28;
+    parkBrake @29;
+    manualRestart @30;
+    lowSpeedLockout @31;
+    joystickDebug @34;
+    steerTempUnavailableSilent @35;
+    resumeRequired @36;
+    preDriverDistracted @37;
+    promptDriverDistracted @38;
+    driverDistracted @39;
+    preDriverUnresponsive @43;
+    promptDriverUnresponsive @44;
+    driverUnresponsive @45;
+    belowSteerSpeed @46;
+    lowBattery @48;
+    accFaulted @51;
+    sensorDataInvalid @52;
+    commIssue @53;
+    commIssueAvgFreq @109;
+    tooDistracted @54;
+    posenetInvalid @55;
+    soundsUnavailable @56;
+    preLaneChangeLeft @57;
+    preLaneChangeRight @58;
+    laneChange @59;
+    lowMemory @63;
+    stockAeb @64;
+    ldw @65;
+    carUnrecognized @66;
+    invalidLkasSetting @69;
+    speedTooHigh @70;
+    laneChangeBlocked @71;
+    relayMalfunction @72;
+    stockFcw @74;
+    startup @75;
+    startupNoCar @76;
+    startupNoControl @77;
+    startupMaster @78;
+    startupNoFw @104;
+    fcw @79;
+    steerSaturated @80;
+    belowEngageSpeed @84;
+    noGps @85;
+    wrongCruiseMode @87;
+    modeldLagging @89;
+    deviceFalling @90;
+    fanMalfunction @91;
+    cameraMalfunction @92;
+    cameraFrameRate @110;
+    processNotRunning @95;
+    dashcamMode @96;
+    controlsInitializing @98;
+    usbError @99;
+    roadCameraError @100;
+    driverCameraError @101;
+    wideRoadCameraError @102;
+    highCpuUsage @105;
+    cruiseMismatch @106;
+    lkasDisabled @107;
+    canBusMissing @111;
+    controlsdLagging @112;
+    resumeBlocked @113;
+    steerTimeLimit @115;
+    vehicleSensorsInvalid @116;
+    locationdTemporaryError @103;
+    locationdPermanentError @118;
+    paramsdTemporaryError @50;
+    paramsdPermanentError @119;
+    actuatorsApiUnavailable @120;
+
+    # FrogPilot Events
+    accel30 @121;
+    accel35 @122;
+    accel40 @123;
+    blockUser @124;
+    dejaVuCurve @125;
+    firefoxSteerSaturated @126;
+    goatSteerSaturated @127;
+    greenLight @128;
+    holidayActive @129;
+    laneChangeBlockedLoud @130;
+    leadDeparting @131;
+    noLaneAvailable @132;
+    openpilotCrashed @133;
+    openpilotCrashedRandomEvents @134;
+    pedalInterceptorNoBrake @135;
+    speedLimitChanged @136;
+    torqueNNLoad @137;
+    trafficModeActive @138;
+    trafficModeInactive @139;
+    turningLeft @140;
+    turningRight @141;
+    vCruise69 @142;
+    yourFrogTriedToKillMe @143;
+
+    radarCanErrorDEPRECATED @15;
+    communityFeatureDisallowedDEPRECATED @62;
+    radarCommIssueDEPRECATED @67;
+    driverMonitorLowAccDEPRECATED @68;
+    gasUnavailableDEPRECATED @3;
+    dataNeededDEPRECATED @16;
+    modelCommIssueDEPRECATED @27;
+    ipasOverrideDEPRECATED @33;
+    geofenceDEPRECATED @40;
+    driverMonitorOnDEPRECATED @41;
+    driverMonitorOffDEPRECATED @42;
+    calibrationProgressDEPRECATED @47;
+    invalidGiraffeHondaDEPRECATED @49;
+    invalidGiraffeToyotaDEPRECATED @60;
+    internetConnectivityNeededDEPRECATED @61;
+    whitePandaUnsupportedDEPRECATED @81;
+    commIssueWarningDEPRECATED @83;
+    focusRecoverActiveDEPRECATED @86;
+    neosUpdateRequiredDEPRECATED @88;
+    modelLagWarningDEPRECATED @93;
+    startupOneplusDEPRECATED @82;
+    startupFuzzyFingerprintDEPRECATED @97;
+    noTargetDEPRECATED @25;
+    brakeUnavailableDEPRECATED @2;
+    plannerErrorDEPRECATED @32;
+    gpsMalfunctionDEPRECATED @94;
+  }
+}
+
+# ******* main car state @ 100hz *******
+# all speeds in m/s
+
+struct CarState {
+  events @13 :List(CarEvent);
+
+  # CAN health
+  canValid @26 :Bool;       # invalid counter/checksums
+  canTimeout @40 :Bool;     # CAN bus dropped out
+  canErrorCounter @48 :UInt32;
+
+  # car speed
+  vEgo @1 :Float32;          # best estimate of speed
+  aEgo @16 :Float32;         # best estimate of acceleration
+  vEgoRaw @17 :Float32;      # unfiltered speed from CAN sensors
+  vEgoCluster @44 :Float32;  # best estimate of speed shown on car's instrument cluster, used for UI
+
+  yawRate @22 :Float32;     # best estimate of yaw rate
+  standstill @18 :Bool;
+  wheelSpeeds @2 :WheelSpeeds;
+
+  # gas pedal, 0.0-1.0
+  gas @3 :Float32;        # this is user pedal only
+  gasPressed @4 :Bool;    # this is user pedal only
+
+  engineRpm @46 :Float32;
+
+  # brake pedal, 0.0-1.0
+  brake @5 :Float32;      # this is user pedal only
+  brakePressed @6 :Bool;  # this is user pedal only
+  regenBraking @45 :Bool; # this is user pedal only
+  parkingBrake @39 :Bool;
+  brakeHoldActive @38 :Bool;
+
+  # steering wheel
+  steeringAngleDeg @7 :Float32;
+  steeringAngleOffsetDeg @37 :Float32; # Offset betweens sensors in case there multiple
+  steeringRateDeg @15 :Float32;
+  steeringTorque @8 :Float32;      # TODO: standardize units
+  steeringTorqueEps @27 :Float32;  # TODO: standardize units
+  steeringPressed @9 :Bool;        # if the user is using the steering wheel
+  steerFaultTemporary @35 :Bool;   # temporary EPS fault
+  steerFaultPermanent @36 :Bool;   # permanent EPS fault
+  stockAeb @30 :Bool;
+  stockFcw @31 :Bool;
+  espDisabled @32 :Bool;
+  accFaulted @42 :Bool;
+  carFaultedNonCritical @47 :Bool;  # some ECU is faulted, but car remains controllable
+
+  # cruise state
+  cruiseState @10 :CruiseState;
+
+  # gear
+  gearShifter @14 :GearShifter;
+
+  # button presses
+  buttonEvents @11 :List(ButtonEvent);
+  leftBlinker @20 :Bool;
+  rightBlinker @21 :Bool;
+  genericToggle @23 :Bool;
+
+  # lock info
+  doorOpen @24 :Bool;
+  seatbeltUnlatched @25 :Bool;
+
+  # clutch (manual transmission only)
+  clutchPressed @28 :Bool;
+
+  # blindspot sensors
+  leftBlindspot @33 :Bool; # Is there something blocking the left lane change
+  rightBlindspot @34 :Bool; # Is there something blocking the right lane change
+
+  fuelGauge @41 :Float32; # battery or fuel tank level from 0.0 to 1.0
+  charging @43 :Bool;
+
+  # process meta
+  cumLagMs @50 :Float32;
+
+  struct WheelSpeeds {
+    # optional wheel speeds
+    fl @0 :Float32;
+    fr @1 :Float32;
+    rl @2 :Float32;
+    rr @3 :Float32;
+  }
+
+  struct CruiseState {
+    enabled @0 :Bool;
+    speed @1 :Float32;
+    speedCluster @6 :Float32;  # Set speed as shown on instrument cluster
+    available @2 :Bool;
+    speedOffset @3 :Float32;
+    standstill @4 :Bool;
+    nonAdaptive @5 :Bool;
+  }
+
+  enum GearShifter {
+    unknown @0;
+    park @1;
+    drive @2;
+    neutral @3;
+    reverse @4;
+    sport @5;
+    low @6;
+    brake @7;
+    eco @8;
+    manumatic @9;
+  }
+
+  # send on change
+  struct ButtonEvent {
+    pressed @0 :Bool;
+    type @1 :Type;
+
+    enum Type {
+      unknown @0;
+      leftBlinker @1;
+      rightBlinker @2;
+      accelCruise @3;
+      decelCruise @4;
+      cancel @5;
+      altButton1 @6;
+      altButton2 @7;
+      altButton3 @8;
+      setCruise @9;
+      resumeCruise @10;
+      gapAdjustCruise @11;
+    }
+  }
+
+  # deprecated
+  errorsDEPRECATED @0 :List(CarEvent.EventName);
+  brakeLightsDEPRECATED @19 :Bool;
+  steeringRateLimitedDEPRECATED @29 :Bool;
+  canMonoTimesDEPRECATED @12: List(UInt64);
+  canRcvTimeoutDEPRECATED @49 :Bool;
+}
+
+# ******* radar state @ 20hz *******
+
+struct RadarData @0x888ad6581cf0aacb {
+  errors @0 :List(Error);
+  points @1 :List(RadarPoint);
+
+  enum Error {
+    canError @0;
+    fault @1;
+    wrongConfig @2;
+  }
+
+  # similar to LiveTracks
+  # is one timestamp valid for all? I think so
+  struct RadarPoint {
+    trackId @0 :UInt64;  # no trackId reuse
+
+    # these 3 are the minimum required
+    dRel @1 :Float32; # m from the front bumper of the car
+    yRel @2 :Float32; # m
+    vRel @3 :Float32; # m/s
+
+    # these are optional and valid if they are not NaN
+    aRel @4 :Float32; # m/s^2
+    yvRel @5 :Float32; # m/s
+
+    # some radars flag measurements VS estimates
+    measured @6 :Bool;
+  }
+
+  # deprecated
+  canMonoTimesDEPRECATED @2 :List(UInt64);
+}
+
+# ******* car controls @ 100hz *******
+
+struct CarControl {
+  # must be true for any actuator commands to work
+  enabled @0 :Bool;
+  latActive @11: Bool;
+  longActive @12: Bool;
+
+  # Actuator commands as computed by controlsd
+  actuators @6 :Actuators;
+
+  # moved to CarOutput
+  actuatorsOutputDEPRECATED @10 :Actuators;
+
+  leftBlinker @15: Bool;
+  rightBlinker @16: Bool;
+
+  orientationNED @13 :List(Float32);
+  angularVelocity @14 :List(Float32);
+
+  cruiseControl @4 :CruiseControl;
+  hudControl @5 :HUDControl;
+
+  struct Actuators {
+    # range from 0.0 - 1.0
+    gas @0: Float32;
+    brake @1: Float32;
+    # range from -1.0 - 1.0
+    steer @2: Float32;
+    # value sent over can to the car
+    steerOutputCan @8: Float32;
+    steeringAngleDeg @3: Float32;
+
+    curvature @7: Float32;
+
+    speed @6: Float32; # m/s
+    accel @4: Float32; # m/s^2
+    longControlState @5: LongControlState;
+
+    enum LongControlState @0xe40f3a917d908282{
+      off @0;
+      pid @1;
+      stopping @2;
+      starting @3;
+    }
+  }
+
+  struct CruiseControl {
+    cancel @0: Bool;
+    resume @1: Bool;
+    override @4: Bool;
+    speedOverrideDEPRECATED @2: Float32;
+    accelOverrideDEPRECATED @3: Float32;
+  }
+
+  struct HUDControl {
+    speedVisible @0: Bool;
+    setSpeed @1: Float32;
+    lanesVisible @2: Bool;
+    leadVisible @3: Bool;
+    visualAlert @4: VisualAlert;
+    audibleAlert @5: AudibleAlert;
+    rightLaneVisible @6: Bool;
+    leftLaneVisible @7: Bool;
+    rightLaneDepart @8: Bool;
+    leftLaneDepart @9: Bool;
+    leadDistanceBars @10: Int8;  # 1-3: 1 is closest, 3 is farthest. some ports may utilize 2-4 bars instead
+
+    enum VisualAlert {
+      # these are the choices from the Honda
+      # map as good as you can for your car
+      none @0;
+      fcw @1;
+      steerRequired @2;
+      brakePressed @3;
+      wrongGear @4;
+      seatbeltUnbuckled @5;
+      speedTooHigh @6;
+      ldw @7;
+    }
+
+    enum AudibleAlert {
+      none @0;
+
+      engage @1;
+      disengage @2;
+      refuse @3;
+
+      warningSoft @4;
+      warningImmediate @5;
+
+      prompt @6;
+      promptRepeat @7;
+      promptDistracted @8;
+
+      # Random Events
+      angry @9;
+      dejaVu @10;
+      doc @11;
+      fart @12;
+      firefox @13;
+      nessie @14;
+      noice @15;
+      uwu @16;
+
+      # Other
+      goat @17;
+    }
+  }
+
+  gasDEPRECATED @1 :Float32;
+  brakeDEPRECATED @2 :Float32;
+  steeringTorqueDEPRECATED @3 :Float32;
+  activeDEPRECATED @7 :Bool;
+  rollDEPRECATED @8 :Float32;
+  pitchDEPRECATED @9 :Float32;
+}
+
+struct CarOutput {
+  # Any car specific rate limits or quirks applied by
+  # the CarController are reflected in actuatorsOutput
+  # and matches what is sent to the car
+  actuatorsOutput @0 :CarControl.Actuators;
+}
+
+# ****** car param ******
+
+struct CarParams {
+  carName @0 :Text;
+  carFingerprint @1 :Text;
+  fuzzyFingerprint @55 :Bool;
+
+  notCar @66 :Bool;  # flag for non-car robotics platforms
+
+  pcmCruise @3 :Bool;        # is openpilot's state tied to the PCM's cruise state?
+  enableDsu @5 :Bool;        # driving support unit
+  enableBsm @56 :Bool;       # blind spot monitoring
+  flags @64 :UInt32;         # flags for car specific quirks
+  experimentalLongitudinalAvailable @71 :Bool;
+
+  minEnableSpeed @7 :Float32;
+  minSteerSpeed @8 :Float32;
+  safetyConfigs @62 :List(SafetyConfig);
+  alternativeExperience @65 :Int16;      # panda flag for features like no disengage on gas
+
+  # Car docs fields
+  maxLateralAccel @68 :Float32;
+  autoResumeSng @69 :Bool;               # describes whether car can resume from a stop automatically
+
+  # things about the car in the manual
+  mass @17 :Float32;            # [kg] curb weight: all fluids no cargo
+  wheelbase @18 :Float32;       # [m] distance from rear axle to front axle
+  centerToFront @19 :Float32;   # [m] distance from center of mass to front axle
+  steerRatio @20 :Float32;      # [] ratio of steering wheel angle to front wheel angle
+  steerRatioRear @21 :Float32;  # [] ratio of steering wheel angle to rear wheel angle (usually 0)
+
+  # things we can derive
+  rotationalInertia @22 :Float32;    # [kg*m2] body rotational inertia
+  tireStiffnessFactor @72 :Float32;  # scaling factor used in calculating tireStiffness[Front,Rear]
+  tireStiffnessFront @23 :Float32;   # [N/rad] front tire coeff of stiff
+  tireStiffnessRear @24 :Float32;    # [N/rad] rear tire coeff of stiff
+
+  longitudinalTuning @25 :LongitudinalPIDTuning;
+  lateralParams @48 :LateralParams;
+  lateralTuning :union {
+    pid @26 :LateralPIDTuning;
+    indiDEPRECATED @27 :LateralINDITuning;
+    lqrDEPRECATED @40 :LateralLQRTuning;
+    torque @67 :LateralTorqueTuning;
+  }
+
+  steerLimitAlert @28 :Bool;
+  steerLimitTimer @47 :Float32;  # time before steerLimitAlert is issued
+
+  vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state
+  vEgoStarting @59 :Float32; # Speed at which the car goes into starting state
+  stoppingControl @31 :Bool; # Does the car allow full control even at lows speeds when stopping
+  steerControlType @34 :SteerControlType;
+  radarUnavailable @35 :Bool; # True when radar objects aren't visible on CAN or aren't parsed out
+  stopAccel @60 :Float32; # Required acceleration to keep vehicle stationary
+  stoppingDecelRate @52 :Float32; # m/s^2/s while trying to stop
+  startAccel @32 :Float32; # Required acceleration to get car moving
+  startingState @70 :Bool; # Does this car make use of special starting state
+
+  steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds
+  longitudinalActuatorDelay @58 :Float32; # Gas/Brake actuator delay in seconds
+  openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control?
+  carVin @38 :Text; # VIN number queried during fingerprinting
+  dashcamOnly @41: Bool;
+  passive @73: Bool;   # is openpilot in control?
+  transmissionType @43 :TransmissionType;
+  carFw @44 :List(CarFw);
+
+  radarTimeStep @45: Float32 = 0.05;  # time delta between radar updates, 20Hz is very standard
+  fingerprintSource @49: FingerprintSource;
+  networkLocation @50 :NetworkLocation;  # Where Panda/C2 is integrated into the car's CAN network
+
+  wheelSpeedFactor @63 :Float32; # Multiplier on wheels speeds to computer actual speeds
+
+  struct SafetyConfig {
+    safetyModel @0 :SafetyModel;
+    safetyParam @3 :UInt16;
+    safetyParamDEPRECATED @1 :Int16;
+    safetyParam2DEPRECATED @2 :UInt32;
+  }
+
+  struct LateralParams {
+    torqueBP @0 :List(Int32);
+    torqueV @1 :List(Int32);
+  }
+
+  struct LateralPIDTuning {
+    kpBP @0 :List(Float32);
+    kpV @1 :List(Float32);
+    kiBP @2 :List(Float32);
+    kiV @3 :List(Float32);
+    kf @4 :Float32;
+  }
+
+  struct LateralTorqueTuning {
+    useSteeringAngle @0 :Bool;
+    kp @1 :Float32;
+    ki @2 :Float32;
+    friction @3 :Float32;
+    kf @4 :Float32;
+    steeringAngleDeadzoneDeg @5 :Float32;
+    latAccelFactor @6 :Float32;
+    latAccelOffset @7 :Float32;
+  }
+
+  struct LongitudinalPIDTuning {
+    kpBP @0 :List(Float32);
+    kpV @1 :List(Float32);
+    kiBP @2 :List(Float32);
+    kiV @3 :List(Float32);
+    kf @6 :Float32;
+    deadzoneBP @4 :List(Float32);
+    deadzoneV @5 :List(Float32);
+  }
+
+  struct LateralINDITuning {
+    outerLoopGainBP @4 :List(Float32);
+    outerLoopGainV @5 :List(Float32);
+    innerLoopGainBP @6 :List(Float32);
+    innerLoopGainV @7 :List(Float32);
+    timeConstantBP @8 :List(Float32);
+    timeConstantV @9 :List(Float32);
+    actuatorEffectivenessBP @10 :List(Float32);
+    actuatorEffectivenessV @11 :List(Float32);
+
+    outerLoopGainDEPRECATED @0 :Float32;
+    innerLoopGainDEPRECATED @1 :Float32;
+    timeConstantDEPRECATED @2 :Float32;
+    actuatorEffectivenessDEPRECATED @3 :Float32;
+  }
+
+  struct LateralLQRTuning {
+    scale @0 :Float32;
+    ki @1 :Float32;
+    dcGain @2 :Float32;
+
+    # State space system
+    a @3 :List(Float32);
+    b @4 :List(Float32);
+    c @5 :List(Float32);
+
+    k @6 :List(Float32);  # LQR gain
+    l @7 :List(Float32);  # Kalman gain
+  }
+
+  enum SafetyModel {
+    silent @0;
+    hondaNidec @1;
+    toyota @2;
+    elm327 @3;
+    gm @4;
+    hondaBoschGiraffe @5;
+    ford @6;
+    cadillac @7;
+    hyundai @8;
+    chrysler @9;
+    tesla @10;
+    subaru @11;
+    gmPassive @12;
+    mazda @13;
+    nissan @14;
+    volkswagen @15;
+    toyotaIpas @16;
+    allOutput @17;
+    gmAscm @18;
+    noOutput @19;  # like silent but without silent CAN TXs
+    hondaBosch @20;
+    volkswagenPq @21;
+    subaruPreglobal @22;  # pre-Global platform
+    hyundaiLegacy @23;
+    hyundaiCommunity @24;
+    volkswagenMlb @25;
+    hongqi @26;
+    body @27;
+    hyundaiCanfd @28;
+    volkswagenMqbEvo @29;
+    chryslerCusw @30;
+    psa @31;
+  }
+
+  enum SteerControlType {
+    torque @0;
+    angle @1;
+
+    curvatureDEPRECATED @2;
+  }
+
+  enum TransmissionType {
+    unknown @0;
+    automatic @1;  # Traditional auto, including DSG
+    manual @2;  # True "stick shift" only
+    direct @3;  # Electric vehicle or other direct drive
+    cvt @4;
+  }
+
+  struct CarFw {
+    ecu @0 :Ecu;
+    fwVersion @1 :Data;
+    address @2 :UInt32;
+    subAddress @3 :UInt8;
+    responseAddress @4 :UInt32;
+    request @5 :List(Data);
+    brand @6 :Text;
+    bus @7 :UInt8;
+    logging @8 :Bool;
+    obdMultiplexing @9 :Bool;
+  }
+
+  enum Ecu {
+    eps @0;
+    abs @1;
+    fwdRadar @2;
+    fwdCamera @3;
+    engine @4;
+    unknown @5;
+    transmission @8; # Transmission Control Module
+    hybrid @18; # hybrid control unit, e.g. Chrysler's HCP, Honda's IMA Control Unit, Toyota's hybrid control computer
+    srs @9; # airbag
+    gateway @10; # can gateway
+    hud @11; # heads up display
+    combinationMeter @12; # instrument cluster
+    electricBrakeBooster @15;
+    shiftByWire @16;
+    adas @19;
+    cornerRadar @21;
+    hvac @20;
+    parkingAdas @7;  # parking assist system ECU, e.g. Toyota's IPAS, Hyundai's RSPA, etc.
+    epb @22;  # electronic parking brake
+    telematics @23;
+    body @24;  # body control module
+
+    # Toyota only
+    dsu @6;
+
+    # Honda only
+    vsa @13; # Vehicle Stability Assist
+    programmedFuelInjection @14;
+
+    debug @17;
+  }
+
+  enum FingerprintSource {
+    can @0;
+    fw @1;
+    fixed @2;
+  }
+
+  enum NetworkLocation {
+    fwdCamera @0;  # Standard/default integration at LKAS camera
+    gateway @1;    # Integration at vehicle's CAN gateway
+  }
+
+  enableGasInterceptor @2 :Bool;
+  enableCameraDEPRECATED @4 :Bool;
+  enableApgsDEPRECATED @6 :Bool;
+  steerRateCostDEPRECATED @33 :Float32;
+  isPandaBlackDEPRECATED @39 :Bool;
+  hasStockCameraDEPRECATED @57 :Bool;
+  safetyParamDEPRECATED @10 :Int16;
+  safetyModelDEPRECATED @9 :SafetyModel;
+  safetyModelPassiveDEPRECATED @42 :SafetyModel = silent;
+  minSpeedCanDEPRECATED @51 :Float32;
+  communityFeatureDEPRECATED @46: Bool;
+  startingAccelRateDEPRECATED @53 :Float32;
+  steerMaxBPDEPRECATED @11 :List(Float32);
+  steerMaxVDEPRECATED @12 :List(Float32);
+  gasMaxBPDEPRECATED @13 :List(Float32);
+  gasMaxVDEPRECATED @14 :List(Float32);
+  brakeMaxBPDEPRECATED @15 :List(Float32);
+  brakeMaxVDEPRECATED @16 :List(Float32);
+  directAccelControlDEPRECATED @30 :Bool;
+  maxSteeringAngleDegDEPRECATED @54 :Float32;
+  longitudinalActuatorDelayLowerBoundDEPRECATEDDEPRECATED @61 :Float32;
+}

cereal/custom.capnp

@@ -7,22 +7,68 @@ $Cxx.namespace("cereal");
 # These structs are guaranteed to remain reserved and empty in mainline
 # cereal, so use these if you want custom events in your fork.
 
-# DO rename the structs
-# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-
-struct CustomReserved0 @0x81c2f05a394cf4af {
+# you can rename the struct, but don't change the identifier
+struct FrogPilotCarControl @0x81c2f05a394cf4af {
+  alwaysOnLateral @0 :Bool;
+  speedLimitChanged @1 :Bool;
 }
 
-struct CustomReserved1 @0xaedffd8f31e7b55d {
+struct FrogPilotCarState @0xaedffd8f31e7b55d {
+  struct ButtonEvent {
+    enum Type {
+      lkas @0;
+    }
+  }
+
+  alwaysOnLateralDisabled @0 :Bool;
+  brakeLights @1 :Bool;
+  dashboardSpeedLimit @2 :Float32;
+  distanceLongPressed @3 :Bool;
+  ecoGear @4 :Bool;
+  hasMenu @5 :Bool;
+  sportGear @6 :Bool;
+  trafficModeActive @7 :Bool;
 }
 
-struct CustomReserved2 @0xf35cc4560bbf6ec2 {
+struct FrogPilotDeviceState @0xf35cc4560bbf6ec2 {
+  freeSpace @0 :Int16;
+  usedSpace @1 :Int16;
 }
 
-struct CustomReserved3 @0xda96579883444c35 {
+struct FrogPilotNavigation @0xda96579883444c35 {
+  approachingIntersection @0 :Bool;
+  approachingTurn @1 :Bool;
+  navigationSpeedLimit @2 :Float32;
 }
 
-struct CustomReserved4 @0x80ae746ee2596b11 {
+struct FrogPilotPlan @0x80ae746ee2596b11 {
+  accelerationJerk @0 :Float32;
+  accelerationJerkStock @1 :Float32;
+  adjustedCruise @2 :Float32;
+  conditionalExperimentalActive @3 :Bool;
+  dangerJerk @4 :Float32;
+  desiredFollowDistance @5 :Int16;
+  greenLight @6 :Bool;
+  laneWidthLeft @7 :Float32;
+  laneWidthRight @8 :Float32;
+  leadDeparting @9 :Bool;
+  maxAcceleration @10 :Float32;
+  minAcceleration @11 :Float32;
+  roadCurvature @12 :Float32;
+  safeObstacleDistance @13 :Int16;
+  safeObstacleDistanceStock @14 :Int16;
+  slcOverridden @15 :Bool;
+  slcOverriddenSpeed @16 :Float32;
+  slcSpeedLimit @17 :Float32;
+  slcSpeedLimitOffset @18 :Float32;
+  speedJerk @19 :Float32;
+  speedJerkStock @20 :Float32;
+  stoppedEquivalenceFactor @21 :Int16;
+  takingCurveQuickly @22 :Bool;
+  tFollow @23 :Float32;
+  unconfirmedSlcSpeedLimit @24 :Float32;
+  vCruise @25 :Float32;
+  vtscControllingCurve @26 :Bool;
 }
 
 struct CustomReserved5 @0xa5cd762cd951a455 {
@@ -39,33 +85,3 @@ struct CustomReserved8 @0xf416ec09499d9d19 {
 
 struct CustomReserved9 @0xa1680744031fdb2d {
 }
-
-struct CustomReserved10 @0xcb9fd56c7057593a {
-}
-
-struct CustomReserved11 @0xc2243c65e0340384 {
-}
-
-struct CustomReserved12 @0x9ccdc8676701b412 {
-}
-
-struct CustomReserved13 @0xcd96dafb67a082d0 {
-}
-
-struct CustomReserved14 @0xb057204d7deadf3f {
-}
-
-struct CustomReserved15 @0xbd443b539493bc68 {
-}
-
-struct CustomReserved16 @0xfc6241ed8877b611 {
-}
-
-struct CustomReserved17 @0xa30662f84033036c {
-}
-
-struct CustomReserved18 @0xc86a3d38d13eb3ef {
-}
-
-struct CustomReserved19 @0xa4f1eb3323f5f582 {
-}

cereal/log.capnp

@@ -17,126 +17,6 @@ struct Map(Key, Value) {
   }
 }
 
-struct OnroadEvent @0xc4fa6047f024e718 {
-  name @0 :EventName;
-
-  # event types
-  enable @1 :Bool;
-  noEntry @2 :Bool;
-  warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
-  userDisable @4 :Bool;
-  softDisable @5 :Bool;
-  immediateDisable @6 :Bool;
-  preEnable @7 :Bool;
-  permanent @8 :Bool; # alerts presented regardless of openpilot state
-  overrideLateral @10 :Bool;
-  overrideLongitudinal @9 :Bool;
-
-  enum EventName @0x91f1992a1f77fb03 {
-    canError @0;
-    steerUnavailable @1;
-    wrongGear @2;
-    doorOpen @3;
-    seatbeltNotLatched @4;
-    espDisabled @5;
-    wrongCarMode @6;
-    steerTempUnavailable @7;
-    reverseGear @8;
-    buttonCancel @9;
-    buttonEnable @10;
-    pedalPressed @11;  # exits active state
-    preEnableStandstill @12;  # added during pre-enable state with brake
-    gasPressedOverride @13;  # added when user is pressing gas with no disengage on gas
-    steerOverride @14;
-    steerDisengage @94;  # exits active state
-    cruiseDisabled @15;
-    speedTooLow @16;
-    outOfSpace @17;
-    overheat @18;
-    calibrationIncomplete @19;
-    calibrationInvalid @20;
-    calibrationRecalibrating @21;
-    controlsMismatch @22;
-    pcmEnable @23;
-    pcmDisable @24;
-    radarFault @25;
-    radarTempUnavailable @93;
-    brakeHold @26;
-    parkBrake @27;
-    manualRestart @28;
-    joystickDebug @29;
-    longitudinalManeuver @30;
-    steerTempUnavailableSilent @31;
-    resumeRequired @32;
-    preDriverDistracted @33;
-    promptDriverDistracted @34;
-    driverDistracted @35;
-    preDriverUnresponsive @36;
-    promptDriverUnresponsive @37;
-    driverUnresponsive @38;
-    belowSteerSpeed @39;
-    lowBattery @40;
-    accFaulted @41;
-    sensorDataInvalid @42;
-    commIssue @43;
-    commIssueAvgFreq @44;
-    tooDistracted @45;
-    posenetInvalid @46;
-    preLaneChangeLeft @48;
-    preLaneChangeRight @49;
-    laneChange @50;
-    lowMemory @51;
-    stockAeb @52;
-    stockLkas @98;
-    lateralManeuver @99;
-    ldw @53;
-    carUnrecognized @54;
-    invalidLkasSetting @55;
-    speedTooHigh @56;
-    laneChangeBlocked @57;
-    relayMalfunction @58;
-    stockFcw @59;
-    startup @60;
-    startupNoCar @61;
-    startupNoControl @62;
-    startupNoSecOcKey @63;
-    startupMaster @64;
-    fcw @65;
-    steerSaturated @66;
-    belowEngageSpeed @67;
-    noGps @68;
-    wrongCruiseMode @69;
-    modeldLagging @70;
-    deviceFalling @71;
-    fanMalfunction @72;
-    cameraMalfunction @73;
-    cameraFrameRate @74;
-    processNotRunning @75;
-    dashcamMode @76;
-    selfdriveInitializing @77;
-    usbError @78;
-    cruiseMismatch @79;
-    canBusMissing @80;
-    selfdrivedLagging @81;
-    resumeBlocked @82;
-    steerTimeLimit @83;
-    vehicleSensorsInvalid @84;
-    locationdTemporaryError @85;
-    locationdPermanentError @86;
-    paramsdTemporaryError @87;
-    paramsdPermanentError @88;
-    actuatorsApiUnavailable @89;
-    espActive @90;
-    personalityChanged @91;
-    aeb @92;
-    userBookmark @95;
-    excessiveActuation @96;
-    audioFeedback @97;
-
-    soundsUnavailableDEPRECATED @47;
-  }
-}
-
 enum LongitudinalPersonality {
   aggressive @0;
   standard @1;
@@ -158,10 +38,6 @@ struct InitData {
   gitBranch @11 :Text;
   gitRemote @13 :Text;
 
-  # this is source commit for prebuilt branches
-  gitSrcCommit @23 :Text;
-  gitSrcCommitDate @24 :Text;
-
   androidProperties @16 :Map(Text, Text);
 
   pandaInfo @8 :PandaInfo;
@@ -261,6 +137,8 @@ struct FrameData {
   requestId @28 :UInt32;
   encodeId @1 :UInt32;
 
+  frameType @7 :FrameType;
+
   # Timestamps
   timestampEof @2 :UInt64;
   timestampSof @8 :UInt64;
@@ -280,7 +158,7 @@ struct FrameData {
 
   temperaturesC @24 :List(Float32);
 
-  enum FrameTypeDEPRECATED {
+  enum FrameType {
     unknown @0;
     neo @1;
     chffrAndroid @2;
@@ -297,7 +175,6 @@ struct FrameData {
 
   frameLengthDEPRECATED @3 :Int32;
   globalGainDEPRECATED @5 :Int32;
-  frameTypeDEPRECATED @7 :FrameTypeDEPRECATED;
   androidCaptureResultDEPRECATED @9 :AndroidCaptureResult;
   lensPosDEPRECATED @11 :Int32;
   lensSagDEPRECATED @12 :Float32;
@@ -420,7 +297,6 @@ struct GpsLocationData {
   speedAccuracy @12 :Float32;
 
   hasFix @13 :Bool;
-  satelliteCount @14 :Int8;
 
   enum SensorSource {
     android @0;
@@ -459,11 +335,17 @@ enum LaneChangeDirection {
   right @2;
 }
 
+enum TurnDirection {
+  none @0;
+  turnLeft @1;
+  turnRight @2;
+}
+
 struct CanData {
   address @0 :UInt32;
+  busTime @1 :UInt16;
   dat     @2 :Data;
   src     @3 :UInt8;
-  busTimeDEPRECATED @1 :UInt16;
 }
 
 struct DeviceState @0xa4d8b5af2aa492eb {
@@ -494,14 +376,10 @@ struct DeviceState @0xa4d8b5af2aa492eb {
   # device thermals
   cpuTempC @26 :List(Float32);
   gpuTempC @27 :List(Float32);
-  dspTempC @49 :Float32;
   memoryTempC @28 :Float32;
+  nvmeTempC @35 :List(Float32);
   modemTempC @36 :List(Float32);
   pmicTempC @39 :List(Float32);
-  intakeTempC @46 :Float32;
-  exhaustTempC @47 :Float32;
-  gnssTempC @48 :Float32;
-  bottomSocTempC @50 :Float32;
   maxTempC @44 :Float32;  # max of other temps, used to control fan
   thermalZones @38 :List(ThermalZone);
   thermalStatus @14 :ThermalStatus;
@@ -572,7 +450,6 @@ struct DeviceState @0xa4d8b5af2aa492eb {
   chargingDisabledDEPRECATED @18 :Bool;
   usbOnlineDEPRECATED @12 :Bool;
   ambientTempCDEPRECATED @30 :Float32;
-  nvmeTempCDEPRECATED @35 :List(Float32);
 }
 
 struct PandaState @0xa7649e2575e4591e {
@@ -588,13 +465,13 @@ struct PandaState @0xa7649e2575e4591e {
   heartbeatLost @22 :Bool;
   interruptLoad @25 :Float32;
   fanPower @28 :UInt8;
+  fanStallCount @34 :UInt8;
 
-  spiErrorCount @33 :UInt16;
+  spiChecksumErrorCount @33 :UInt16;
 
   harnessStatus @21 :HarnessStatus;
   sbu1Voltage @35 :Float32;
   sbu2Voltage @36 :Float32;
-  soundOutputLevel @37 :UInt16;
 
   # can health
   canState0 @29 :PandaCanState;
@@ -613,11 +490,6 @@ struct PandaState @0xa7649e2575e4591e {
   voltage @0 :UInt32;
   current @1 :UInt32;
 
-  # these fields are not used by openpilot, but they're
-  # reserved for forks building alternate experiences.
-  controlsAllowedRESERVED1 @38 :Bool;
-  controlsAllowedRESERVED2 @39 :Bool;
-
   enum FaultStatus {
     none @0;
     faultTemp @1;
@@ -722,7 +594,6 @@ struct PandaState @0xa7649e2575e4591e {
   usbPowerModeDEPRECATED @12 :PeripheralState.UsbPowerModeDEPRECATED;
   safetyParamDEPRECATED @20 :Int16;
   safetyParam2DEPRECATED @26 :UInt32;
-  fanStallCountDEPRECATED @34 :UInt8;
 }
 
 struct PeripheralState {
@@ -743,10 +614,11 @@ struct PeripheralState {
 struct RadarState @0x9a185389d6fdd05f {
   mdMonoTime @6 :UInt64;
   carStateMonoTime @11 :UInt64;
-  radarErrors @13 :Car.RadarData.Error;
+  radarErrors @12 :List(Car.RadarData.Error);
 
   leadOne @3 :LeadData;
   leadTwo @4 :LeadData;
+  cumLagMs @5 :Float32;
 
   struct LeadData {
     dRel @0 :Float32;
@@ -776,8 +648,6 @@ struct RadarState @0x9a185389d6fdd05f {
   calCycleDEPRECATED @8 :Int32;
   calPercDEPRECATED @9 :Int8;
   canMonoTimesDEPRECATED @10 :List(UInt64);
-  cumLagMsDEPRECATED @5 :Float32;
-  radarErrorsDEPRECATED @12 :List(Car.RadarData.ErrorDEPRECATED);
 }
 
 struct LiveCalibrationData {
@@ -808,7 +678,7 @@ struct LiveCalibrationData {
   }
 }
 
-struct LiveTracksDEPRECATED {
+struct LiveTracks {
   trackId @0 :Int32;
   dRel @1 :Float32;
   yRel @2 :Float32;
@@ -821,25 +691,53 @@ struct LiveTracksDEPRECATED {
   oncoming @9 :Bool;
 }
 
-struct SelfdriveState {
-  # high level system state
-  state @0 :OpenpilotState;
-  enabled @1 :Bool;
-  active @2 :Bool;
-  engageable @9 :Bool;  # can OP be engaged?
+struct ControlsState @0x97ff69c53601abf1 {
+  startMonoTime @48 :UInt64;
+  longitudinalPlanMonoTime @28 :UInt64;
+  lateralPlanMonoTime @50 :UInt64;
+
+  state @31 :OpenpilotState;
+  enabled @19 :Bool;
+  active @36 :Bool;
+
+  experimentalMode @64 :Bool;
+  personality @66 :LongitudinalPersonality;
+
+  longControlState @30 :Car.CarControl.Actuators.LongControlState;
+  vPid @2 :Float32;
+  vTargetLead @3 :Float32;
+  vCruise @22 :Float32;  # actual set speed
+  vCruiseCluster @63 :Float32;  # set speed to display in the UI
+  upAccelCmd @4 :Float32;
+  uiAccelCmd @5 :Float32;
+  ufAccelCmd @33 :Float32;
+  aTarget @35 :Float32;
+  curvature @37 :Float32;  # path curvature from vehicle model
+  desiredCurvature @61 :Float32;  # lag adjusted curvatures used by lateral controllers
+  forceDecel @51 :Bool;
 
   # UI alerts
-  alertText1 @3 :Text;
-  alertText2 @4 :Text;
-  alertStatus @5 :AlertStatus;
-  alertSize @6 :AlertSize;
-  alertType @7 :Text;
-  alertSound @8 :Car.CarControl.HUDControl.AudibleAlert;
-  alertHudVisual @12 :Car.CarControl.HUDControl.VisualAlert;
+  alertText1 @24 :Text;
+  alertText2 @25 :Text;
+  alertStatus @38 :AlertStatus;
+  alertSize @39 :AlertSize;
+  alertBlinkingRate @42 :Float32;
+  alertType @44 :Text;
+  alertSound @56 :Car.CarControl.HUDControl.AudibleAlert;
+  engageable @41 :Bool;  # can OP be engaged?
 
-  # configurable driving settings
-  experimentalMode @10 :Bool;
-  personality @11 :LongitudinalPersonality;
+  cumLagMs @15 :Float32;
+
+  lateralControlState :union {
+    indiState @52 :LateralINDIState;
+    pidState @53 :LateralPIDState;
+    angleState @58 :LateralAngleState;
+    debugState @59 :LateralDebugState;
+    torqueState @60 :LateralTorqueState;
+
+    curvatureStateDEPRECATED @65 :LateralCurvatureState;
+    lqrStateDEPRECATED @55 :LateralLQRState;
+  }
 
   enum OpenpilotState @0xdbe58b96d2d1ac61 {
     disabled @0;
@@ -849,41 +747,18 @@ struct SelfdriveState {
     overriding @4;  # superset of overriding with steering or accelerator
   }
 
-  enum AlertStatus @0xa0d0dcd113193c62 {
-    normal @0;
-    userPrompt @1;
-    critical @2;
+  enum AlertStatus {
+    normal @0;       # low priority alert for user's convenience
+    userPrompt @1;   # mid priority alert that might require user intervention
+    critical @2;     # high priority alert that needs immediate user intervention
+    frogpilot @3;    # FrogPilot startup alert
   }
 
-  enum AlertSize @0xe98bb99d6e985f64 {
-    none @0;
-    small @1;
-    mid @2;
-    full @3;
-  }
-}
-
-struct ControlsState @0x97ff69c53601abf1 {
-  longitudinalPlanMonoTime @28 :UInt64;
-  lateralPlanMonoTime @50 :UInt64;
-
-  longControlState @30 :Car.CarControl.Actuators.LongControlState;
-  upAccelCmd @4 :Float32;
-  uiAccelCmd @5 :Float32;
-  ufAccelCmd @33 :Float32;
-  curvature @37 :Float32;  # path curvature from vehicle model
-  desiredCurvature @61 :Float32;  # lag adjusted curvatures used by lateral controllers
-  forceDecel @51 :Bool;
-
-  lateralControlState :union {
-    pidState @53 :LateralPIDState;
-    angleState @58 :LateralAngleState;
-    debugState @59 :LateralDebugState;
-    torqueState @60 :LateralTorqueState;
-
-    curvatureStateDEPRECATED @65 :LateralCurvatureState;
-    lqrStateDEPRECATED @55 :LateralLQRState;
-    indiStateDEPRECATED @52 :LateralINDIState;
+  enum AlertSize {
+    none @0;    # don't display the alert
+    small @1;   # small box
+    mid @2;     # mid screen
+    full @3;    # full screen
   }
 
   struct LateralINDIState {
@@ -927,8 +802,7 @@ struct ControlsState @0x97ff69c53601abf1 {
     saturated @7 :Bool;
     actualLateralAccel @9 :Float32;
     desiredLateralAccel @10 :Float32;
-    desiredLateralJerk @11 :Float32;
-    version @12 :Int32;
+    nnLog @11 :List(Float32);
    }
 
   struct LateralLQRState {
@@ -1000,58 +874,6 @@ struct ControlsState @0x97ff69c53601abf1 {
   canMonoTimesDEPRECATED @21 :List(UInt64);
   desiredCurvatureRateDEPRECATED @62 :Float32;
   canErrorCounterDEPRECATED @57 :UInt32;
-  vPidDEPRECATED @2 :Float32;
-  alertBlinkingRateDEPRECATED @42 :Float32;
-  alertText1DEPRECATED @24 :Text;
-  alertText2DEPRECATED @25 :Text;
-  alertStatusDEPRECATED @38 :SelfdriveState.AlertStatus;
-  alertSizeDEPRECATED @39 :SelfdriveState.AlertSize;
-  alertTypeDEPRECATED @44 :Text;
-  alertSound2DEPRECATED @56 :Car.CarControl.HUDControl.AudibleAlert;
-  engageableDEPRECATED @41 :Bool;  # can OP be engaged?
-  stateDEPRECATED @31 :SelfdriveState.OpenpilotState;
-  enabledDEPRECATED @19 :Bool;
-  activeDEPRECATED @36 :Bool;
-  experimentalModeDEPRECATED @64 :Bool;
-  personalityDEPRECATED @66 :LongitudinalPersonality;
-  vCruiseDEPRECATED @22 :Float32;  # actual set speed
-  vCruiseClusterDEPRECATED @63 :Float32;  # set speed to display in the UI
-  startMonoTimeDEPRECATED @48 :UInt64;
-  cumLagMsDEPRECATED @15 :Float32;
-  aTargetDEPRECATED @35 :Float32;
-  vTargetLeadDEPRECATED @3 :Float32;
-}
-
-struct DrivingModelData {
-  frameId @0 :UInt32;
-  frameIdExtra @1 :UInt32;
-  frameDropPerc @6 :Float32;
-  modelExecutionTime @7 :Float32;
-
-  action @2 :ModelDataV2.Action;
-
-  laneLineMeta @3 :LaneLineMeta;
-  meta @4 :MetaData;
-
-  path @5 :PolyPath;
-
-  struct PolyPath {
-    xCoefficients @0 :List(Float32);
-    yCoefficients @1 :List(Float32);
-    zCoefficients @2 :List(Float32);
-  }
-
-  struct LaneLineMeta {
-    leftY @0 :Float32;
-    rightY @1 :Float32;
-    leftProb @2 :Float32;
-    rightProb @3 :Float32;
-  }
-
-  struct MetaData {
-    laneChangeState @0 :LaneChangeState;
-    laneChangeDirection @1 :LaneChangeDirection;
-  }
 }
 
 # All SI units and in device frame
@@ -1072,6 +894,7 @@ struct ModelDataV2 {
   frameDropPerc @2 :Float32;
   timestampEof @3 :UInt64;
   modelExecutionTime @15 :Float32;
+  gpuExecutionTime @17 :Float32;
   rawPredictions @16 :Data;
 
   # predicted future position, orientation, etc..
@@ -1096,15 +919,14 @@ struct ModelDataV2 {
   confidence @23: ConfidenceClass;
 
   # Model perceived motion
-  temporalPoseDEPRECATED @21 :Pose;
+  temporalPose @21 :Pose;
+
+  navEnabled @22 :Bool;
+  locationMonoTime @24 :UInt64;
 
   # e2e lateral planner
-  action @26: Action;
-
-  gpuExecutionTimeDEPRECATED @17 :Float32;
-  navEnabledDEPRECATED @22 :Bool;
-  locationMonoTimeDEPRECATED @24 :UInt64;
   lateralPlannerSolutionDEPRECATED @25: LateralPlannerSolution;
+  action @26: Action;
 
   struct LeadDataV2 {
     prob @0 :Float32; # probability that car is your lead at time t
@@ -1144,6 +966,7 @@ struct ModelDataV2 {
     hardBrakePredicted @7 :Bool;
     laneChangeState @8 :LaneChangeState;
     laneChangeDirection @9 :LaneChangeDirection;
+    turnDirection @10 :TurnDirection;
 
 
     # deprecated
@@ -1166,8 +989,6 @@ struct ModelDataV2 {
     brake3MetersPerSecondSquaredProbs @4 :List(Float32);
     brake4MetersPerSecondSquaredProbs @5 :List(Float32);
     brake5MetersPerSecondSquaredProbs @6 :List(Float32);
-    gasPressProbs @7 :List(Float32);
-    brakePressProbs @8 :List(Float32);
   }
 
   struct Pose {
@@ -1190,8 +1011,6 @@ struct ModelDataV2 {
 
   struct Action {
     desiredCurvature @0 :Float32;
-    desiredAcceleration @1 :Float32;
-    shouldStop @2 :Bool;
   }
 }
 
@@ -1239,18 +1058,6 @@ struct AndroidLogEntry {
   message @6 :Text;
 }
 
-struct DriverAssistance {
-  # Lane Departure Warnings
-  leftLaneDeparture @0 :Bool;
-  rightLaneDeparture @1 :Bool;
-
-  # FCW, AEB, etc. will go here
-}
-
-struct LateralManeuverPlan {
-  desiredCurvature @0 :Float32;  # 1/m
-}
-
 struct LongitudinalPlan @0xe00b5b3eba12876c {
   modelMonoTime @9 :UInt64;
   hasLead @7 :Bool;
@@ -1302,7 +1109,7 @@ struct LongitudinalPlan @0xe00b5b3eba12876c {
   radarValidDEPRECATED @28 :Bool;
   radarCanErrorDEPRECATED @30 :Bool;
   commIssueDEPRECATED @31 :Bool;
-  eventsDEPRECATED @13 :List(Car.OnroadEventDEPRECATED);
+  eventsDEPRECATED @13 :List(Car.CarEvent);
   gpsTrajectoryDEPRECATED @12 :GpsTrajectory;
   gpsPlannerActiveDEPRECATED @19 :Bool;
   personalityDEPRECATED @36 :LongitudinalPersonality;
@@ -1423,48 +1230,6 @@ struct LiveLocationKalman {
   }
 }
 
-
-struct LivePose {
-  # More info on reference frames:
-  # https://github.com/commaai/openpilot/tree/master/common/transformations
-  orientationNED @0 :XYZMeasurement;
-  velocityDevice @1 :XYZMeasurement;
-  accelerationDevice @2 :XYZMeasurement;
-  angularVelocityDevice @3 :XYZMeasurement;
-
-  inputsOK @4 :Bool = false;
-  posenetOK @5 :Bool = false;
-  sensorsOK @6 :Bool = false;
-
-  timestamp @8 :UInt64;
-
-  debugFilterState @7 :FilterState;
-
-  struct XYZMeasurement {
-    x @0 :Float32;
-    y @1 :Float32;
-    z @2 :Float32;
-    xStd @3 :Float32;
-    yStd @4 :Float32;
-    zStd @5 :Float32;
-    valid @6 :Bool;
-  }
-
-  struct FilterState {
-    value @0 : List(Float64);
-    std @1 : List(Float64);
-    valid @2 : Bool;
-
-    observations @3 :List(Observation);
-
-    struct Observation {
-      kind @0 :Int32;
-      value @1 :List(Float32);
-      error @2 :List(Float32);
-    }
-  }
-}
-
 struct ProcLog {
   cpuTimes @0 :List(CPUTimes);
   mem @1 :Mem;
@@ -1492,11 +1257,6 @@ struct ProcLog {
 
     cmdline @15 :List(Text);
     exe @16 :Text;
-
-    # from /proc/<pid>/smaps_rollup (proportional/private memory)
-    memPss @17 :UInt64;        # Pss — shared pages split by mapper count
-    memPssAnon @18 :UInt64;    # Pss_Anon — private anonymous (heap, stack)
-    memPssShmem @19 :UInt64;   # Pss_Shmem — proportional MSGQ/tmpfs share
   }
 
   struct CPUTimes {
@@ -1615,10 +1375,6 @@ struct UbloxGnss {
       svId @0 :UInt8;
       gnssId @1 :UInt8;
       flagsBitfield @2 :UInt32;
-      cno @3 :UInt8;
-      elevationDeg @4 :Int8;
-      azimuthDeg @5 :Int16;
-      pseudorangeResidual @6 :Float32;
     }
   }
 
@@ -2168,10 +1924,12 @@ struct Joystick {
 struct DriverStateV2 {
   frameId @0 :UInt32;
   modelExecutionTime @1 :Float32;
-  gpuExecutionTime @8 :Float32;
+  dspExecutionTime @2 :Float32;
   rawPredictions @3 :Data;
 
+  poorVisionProb @4 :Float32;
   wheelOnRightProb @5 :Float32;
+
   leftDriverData @6 :DriverData;
   rightDriverData @7 :DriverData;
 
@@ -2181,21 +1939,15 @@ struct DriverStateV2 {
     facePosition @2 :List(Float32);
     facePositionStd @3 :List(Float32);
     faceProb @4 :Float32;
-    eyesVisibleProb @14 :Float32;
-    eyesClosedProb @15 :Float32;
-    phoneProb @13 :Float32;
-    leftEyeProbDEPRECATED @5 :Float32;
-    rightEyeProbDEPRECATED @6 :Float32;
-    leftBlinkProbDEPRECATED @7 :Float32;
-    rightBlinkProbDEPRECATED @8 :Float32;
-    sunglassesProbDEPRECATED @9 :Float32;
-    notReadyProbDEPRECATED @12 :List(Float32);
-    occludedProbDEPRECATED @10 :Float32;
-    readyProbDEPRECATED @11 :List(Float32);
+    leftEyeProb @5 :Float32;
+    rightEyeProb @6 :Float32;
+    leftBlinkProb @7 :Float32;
+    rightBlinkProb @8 :Float32;
+    sunglassesProb @9 :Float32;
+    occludedProb @10 :Float32;
+    readyProb @11 :List(Float32);
+    notReadyProb @12 :List(Float32);
   }
-
-  dspExecutionTimeDEPRECATED @2 :Float32;
-  poorVisionProbDEPRECATED @4 :Float32;
 }
 
 struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
@@ -2231,7 +1983,7 @@ struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
 }
 
 struct DriverMonitoringState @0xb83cda094a1da284 {
-  events @18 :List(OnroadEvent);
+  events @0 :List(Car.CarEvent);
   faceDetected @1 :Bool;
   isDistracted @2 :Bool;
   distractedType @17 :UInt32;
@@ -2247,13 +1999,9 @@ struct DriverMonitoringState @0xb83cda094a1da284 {
   hiStdCount @14 :UInt32;
   isActiveMode @16 :Bool;
   isRHD @4 :Bool;
-  uncertainCount @19 :UInt32;
 
-  phoneProbOffsetDEPRECATED @20 :Float32;
-  phoneProbValidCountDEPRECATED @21 :UInt32;
   isPreviewDEPRECATED @15 :Bool;
   rhdCheckedDEPRECATED @5 :Bool;
-  eventsDEPRECATED @0 :List(Car.OnroadEventDEPRECATED);
 }
 
 struct Boot {
@@ -2281,20 +2029,9 @@ struct LiveParametersData {
   stiffnessFactorStd @12 :Float32;
   steerRatioStd @13 :Float32;
   roll @14 :Float32;
-  debugFilterState @16 :FilterState;
-
-  angleOffsetValid @17 :Bool = true;
-  angleOffsetAverageValid @18 :Bool = true;
-  steerRatioValid @19 :Bool = true;
-  stiffnessFactorValid @20 :Bool = true;
+  filterState @15 :LiveLocationKalman.Measurement;
 
   yawRateDEPRECATED @7 :Float32;
-  filterStateDEPRECATED @15 :LiveLocationKalman.Measurement;
-
-  struct FilterState {
-    value @0 : List(Float64);
-    std @1 : List(Float64);
-  }
 }
 
 struct LiveTorqueParametersData {
@@ -2311,24 +2048,6 @@ struct LiveTorqueParametersData {
   points @10 :List(List(Float32));
   version @11 :Int32;
   useParams @12 :Bool;
-  calPerc @13 :Int8;
-}
-
-struct LiveDelayData {
-  lateralDelay @0 :Float32;
-  validBlocks @1 :Int32;
-  status @2 :Status;
-
-  lateralDelayEstimate @3 :Float32;
-  lateralDelayEstimateStd @5 :Float32;
-  points @4 :List(Float32);
-  calPerc @6 :Int8;
-
-  enum Status {
-    unestimated @0;
-    estimated @1;
-    invalid @2;
-  }
 }
 
 struct LiveMapDataDEPRECATED {
@@ -2492,40 +2211,16 @@ struct EncodeData {
   height @5 :UInt32;
 }
 
-struct DebugAlert {
-  alertText1 @0 :Text;
-  alertText2 @1 :Text;
+struct UserFlag {
 }
 
-struct UserBookmark @0xfe346a9de48d9b50 {
-}
-
-struct SoundPressure @0xdc24138990726023 {
+struct Microphone {
   soundPressure @0 :Float32;
 
   # uncalibrated, A-weighted
   soundPressureWeighted @3 :Float32;
   soundPressureWeightedDb @1 :Float32;
-
-  filteredSoundPressureWeightedDbDEPRECATED @2 :Float32;
-}
-
-struct AudioData {
-  data @0 :Data;
-  sampleRate @1 :UInt32;
-}
-
-struct AudioFeedback {
-  audio @0 :AudioData;
-  blockNum @1 :UInt16;
-}
-
-struct Touch {
-  sec @0 :Int64;
-  usec @1 :Int64;
-  type @2 :UInt8;
-  code @3 :Int32;
-  value @4 :Int32;
+  filteredSoundPressureWeightedDb @2 :Float32;
 }
 
 struct Event {
@@ -2544,23 +2239,25 @@ struct Event {
     gpsNMEA @3 :GPSNMEAData;
     can @5 :List(CanData);
     controlsState @7 :ControlsState;
-    selfdriveState @130 :SelfdriveState;
     gyroscope @99 :SensorEventData;
+    gyroscope2 @100 :SensorEventData;
     accelerometer @98 :SensorEventData;
+    accelerometer2 @101 :SensorEventData;
     magnetometer @95 :SensorEventData;
     lightSensor @96 :SensorEventData;
     temperatureSensor @97 :SensorEventData;
+    temperatureSensor2 @123 :SensorEventData;
     pandaStates @81 :List(PandaState);
     peripheralState @80 :PeripheralState;
     radarState @13 :RadarState;
-    liveTracks @131 :Car.RadarData;
+    liveTracks @16 :List(LiveTracks);
     sendcan @17 :List(CanData);
     liveCalibration @19 :LiveCalibrationData;
     carState @22 :Car.CarState;
     carControl @23 :Car.CarControl;
     carOutput @127 :Car.CarOutput;
     longitudinalPlan @24 :LongitudinalPlan;
-    driverAssistance @132 :DriverAssistance;
+    uiPlan @106 :UiPlan;
     ubloxGnss @34 :UbloxGnss;
     ubloxRaw @39 :Data;
     qcomGnss @31 :QcomGnss;
@@ -2569,15 +2266,13 @@ struct Event {
     gnssMeasurements @91 :GnssMeasurements;
     liveParameters @61 :LiveParametersData;
     liveTorqueParameters @94 :LiveTorqueParametersData;
-    liveDelay @146 : LiveDelayData;
     cameraOdometry @63 :CameraOdometry;
     thumbnail @66: Thumbnail;
-    onroadEvents @134: List(OnroadEvent);
+    onroadEvents @68: List(Car.CarEvent);
     carParams @69: Car.CarParams;
     driverMonitoringState @71: DriverMonitoringState;
-    livePose @129 :LivePose;
+    liveLocationKalman @72 :LiveLocationKalman;
     modelV2 @75 :ModelDataV2;
-    drivingModelData @128 :DrivingModelData;
     driverStateV2 @92 :DriverStateV2;
 
     # camera stuff, each camera state has a matching encode idx
@@ -2594,8 +2289,7 @@ struct Event {
     livestreamDriverEncodeIdx @119 :EncodeIndex;
 
     # microphone data
-    soundPressure @103 :SoundPressure;
-    rawAudioData @147 :AudioData;
+    microphone @103 :Microphone;
 
     # systems stuff
     androidLog @20 :AndroidLogEntry;
@@ -2607,9 +2301,6 @@ struct Event {
     logMessage @18 :Text;
     errorLogMessage @85 :Text;
 
-    # touch frame
-    touch @135 :List(Touch);
-
     # navigation
     navInstruction @82 :NavInstruction;
     navRoute @83 :NavRoute;
@@ -2617,64 +2308,41 @@ struct Event {
     mapRenderState @105: MapRenderState;
 
     # UI services
+    userFlag @93 :UserFlag;
     uiDebug @102 :UIDebug;
 
-    # driving feedback
-    userBookmark @93 :UserBookmark;
-    bookmarkButton @148 :UserBookmark;
-    audioFeedback @149 :AudioFeedback;
-
-    lateralManeuverPlan @150 :LateralManeuverPlan;
-
     # *********** debug ***********
     testJoystick @52 :Joystick;
     roadEncodeData @86 :EncodeData;
     driverEncodeData @87 :EncodeData;
     wideRoadEncodeData @88 :EncodeData;
     qRoadEncodeData @89 :EncodeData;
-    alertDebug @133 :DebugAlert;
 
     livestreamRoadEncodeData @120 :EncodeData;
     livestreamWideRoadEncodeData @121 :EncodeData;
     livestreamDriverEncodeData @122 :EncodeData;
 
-    # *********** Custom: reserved for forks ***********
-
-    # DO change the name of the field
-    # DON'T change anything after the "@"
     customReservedRawData0 @124 :Data;
     customReservedRawData1 @125 :Data;
     customReservedRawData2 @126 :Data;
 
-    # DO change the name of the field and struct
-    # DON'T change the ID (e.g. @107)
-    # DON'T change which struct it points to
-    customReserved0 @107 :Custom.CustomReserved0;
-    customReserved1 @108 :Custom.CustomReserved1;
-    customReserved2 @109 :Custom.CustomReserved2;
-    customReserved3 @110 :Custom.CustomReserved3;
-    customReserved4 @111 :Custom.CustomReserved4;
+    # *********** Custom: reserved for forks ***********
+    frogpilotCarControl @107 :Custom.FrogPilotCarControl;
+    frogpilotCarState @108 :Custom.FrogPilotCarState;
+    frogpilotDeviceState @109 :Custom.FrogPilotDeviceState;
+    frogpilotNavigation @110 :Custom.FrogPilotNavigation;
+    frogpilotPlan @111 :Custom.FrogPilotPlan;
     customReserved5 @112 :Custom.CustomReserved5;
     customReserved6 @113 :Custom.CustomReserved6;
     customReserved7 @114 :Custom.CustomReserved7;
     customReserved8 @115 :Custom.CustomReserved8;
     customReserved9 @116 :Custom.CustomReserved9;
-    customReserved10 @136 :Custom.CustomReserved10;
-    customReserved11 @137 :Custom.CustomReserved11;
-    customReserved12 @138 :Custom.CustomReserved12;
-    customReserved13 @139 :Custom.CustomReserved13;
-    customReserved14 @140 :Custom.CustomReserved14;
-    customReserved15 @141 :Custom.CustomReserved15;
-    customReserved16 @142 :Custom.CustomReserved16;
-    customReserved17 @143 :Custom.CustomReserved17;
-    customReserved18 @144 :Custom.CustomReserved18;
-    customReserved19 @145 :Custom.CustomReserved19;
 
     # *********** legacy + deprecated ***********
     model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated
     liveMpcDEPRECATED @36 :LiveMpcData;
     liveLongitudinalMpcDEPRECATED @37 :LiveLongitudinalMpcData;
-    liveLocationKalmanLegacyDEPRECATED @51 :Legacy.LiveLocationData;
+    liveLocationKalmanDEPRECATED @51 :Legacy.LiveLocationData;
     orbslamCorrectionDEPRECATED @45 :Legacy.OrbslamCorrection;
     liveUIDEPRECATED @14 :Legacy.LiveUI;
     sensorEventDEPRECATED @4 :SensorEventData;
@@ -2709,13 +2377,6 @@ struct Event {
     driverStateDEPRECATED @59 :DriverStateDEPRECATED;
     sensorEventsDEPRECATED @11 :List(SensorEventData);
     lateralPlanDEPRECATED @64 :LateralPlan;
-    navModelDEPRECATED @104 :NavModelData;
-    uiPlanDEPRECATED @106 :UiPlan;
-    liveLocationKalmanDEPRECATED @72 :LiveLocationKalman;
-    liveTracksDEPRECATED @16 :List(LiveTracksDEPRECATED);
-    onroadEventsDEPRECATED @68: List(Car.OnroadEventDEPRECATED);
-    gyroscope2DEPRECATED @100 :SensorEventData;
-    accelerometer2DEPRECATED @101 :SensorEventData;
-    temperatureSensor2DEPRECATED @123 :SensorEventData;
+    navModel @104 :NavModelData;
   }
 }

cereal/maptile.capnp

@@ -0,0 +1,49 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0xa086df597ef5d7a0;
+
+# Geometry
+struct Point {
+  x @0: Float64;
+  y @1: Float64;
+  z @2: Float64;
+}
+
+struct PolyLine {
+  points @0: List(Point);
+}
+
+# Map features
+struct Lane {
+  id @0 :Text;
+
+  leftBoundary @1 :LaneBoundary;
+  rightBoundary @2 :LaneBoundary;
+
+  leftAdjacentId @3 :Text;
+  rightAdjacentId @4 :Text;
+
+  inboundIds @5 :List(Text);
+  outboundIds @6 :List(Text);
+
+  struct LaneBoundary {
+    polyLine @0 :PolyLine;
+    startHeading @1 :Float32; # WRT north
+  }
+}
+
+# Map tiles
+struct TileSummary {
+  version @0 :Text;
+  updatedAt @1 :UInt64; # Millis since epoch
+
+  level @2 :UInt8;
+  x @3 :UInt16;
+  y @4 :UInt16;
+}
+
+struct MapTile {
+  summary @0 :TileSummary;
+  lanes @1 :List(Lane);
+}

cereal/messaging/.gitignore

@@ -0,0 +1,10 @@
+demo
+bridge
+test_runner
+*.o
+*.os
+*.d
+*.a
+*.so
+messaging_pyx.cpp
+build/

cereal/messaging/__init__.py

@@ -1,41 +1,24 @@
 # must be built with scons
-from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
-                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-import msgq
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw, context
+
 import os
 import capnp
 import time
 
-from typing import Optional, List, Union, Dict
+from typing import Optional, List, Union, Dict, Deque
+from collections import deque
 
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.utils import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def pub_sock(endpoint: str) -> PubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.pub_sock(endpoint, segment_size)
-
-
-def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
-                       timeout=timeout, segment_size=segment_size)
-
-
-def reset_context():
-  msgq.context = Context()
-
-
-def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
-  with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
+def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
+  with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
     return msg
 
 
@@ -104,71 +87,26 @@ def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
       return log_from_bytes(dat)
 
 
-class FrequencyTracker:
-  def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
-    freq = max(min(service_freq, update_freq), 1.)
-    if is_poll:
-      min_freq = max_freq = freq
-    else:
-      max_freq = min(freq, update_freq)
-      if service_freq >= 2 * update_freq:
-        min_freq = update_freq
-      elif update_freq >= 2* service_freq:
-        min_freq = freq
-      else:
-        min_freq = min(freq, freq / 2.)
-
-    self.min_freq = min_freq * 0.8
-    self.max_freq = max_freq * 1.2
-    self.avg_dt = MovingAverage(int(10 * freq))
-    self.recent_avg_dt = MovingAverage(int(freq))
-    self.prev_time = 0.0
-
-  def record_recv_time(self, cur_time: float) -> None:
-    # TODO: Handle case where cur_time is less than prev_time
-    if self.prev_time > 1e-5:
-      dt = cur_time - self.prev_time
-
-      self.avg_dt.add_value(dt)
-      self.recent_avg_dt.add_value(dt)
-
-    self.prev_time = cur_time
-
-  @property
-  def valid(self) -> bool:
-    if self.avg_dt.count == 0:
-      return False
-
-    avg_freq = 1.0 / self.avg_dt.get_average()
-    if self.min_freq <= avg_freq <= self.max_freq:
-      return True
-
-    avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
-    return self.min_freq <= avg_freq_recent <= self.max_freq
-
-
 class SubMaster:
   def __init__(self, services: List[str], poll: Optional[str] = None,
                ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
                ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
     self.frame = -1
-    self.services = services
     self.seen = {s: False for s in services}
     self.updated = {s: False for s in services}
     self.recv_time = {s: 0. for s in services}
     self.recv_frame = {s: 0 for s in services}
+    self.alive = {s: False for s in services}
+    self.freq_ok = {s: False for s in services}
+    self.recv_dts: Dict[str, Deque[float]] = {}
     self.sock = {}
     self.data = {}
-    self.logMonoTime = {s: 0 for s in services}
+    self.valid = {}
+    self.logMonoTime = {}
 
-    # zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
-    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
-    self.static_freq_services = set(s for s in services if not on_demand[s])
-    self.alive = {s: on_demand[s] for s in services}
-    self.freq_ok = {s: on_demand[s] for s in services}
-    self.valid = {s: on_demand[s] for s in services}
+    self.max_freq = {}
+    self.min_freq = {}
 
-    self.freq_tracker: Dict[str, FrequencyTracker] = {}
     self.poller = Poller()
     polled_services = set([poll, ] if poll is not None else services)
     self.non_polled_services = set(services) - polled_services
@@ -193,7 +131,24 @@ class SubMaster:
         data = new_message(s, 0) # lists
 
       self.data[s] = getattr(data.as_reader(), s)
-      self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
+      self.logMonoTime[s] = 0
+      self.valid[s] = True  # FIXME: this should default to False
+
+      freq = max(min([SERVICE_LIST[s].frequency, self.update_freq]), 1.)
+      if s == poll:
+        max_freq = freq
+        min_freq = freq
+      else:
+        max_freq = min(freq, self.update_freq)
+        if SERVICE_LIST[s].frequency >= 2*self.update_freq:
+          min_freq = self.update_freq
+        elif self.update_freq >= 2*SERVICE_LIST[s].frequency:
+          min_freq = freq
+        else:
+          min_freq = min(freq, freq / 2.)
+      self.max_freq[s] = max_freq*1.2
+      self.min_freq[s] = min_freq*0.8
+      self.recv_dts[s] = deque(maxlen=int(10*freq))
 
   def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
     return self.data[s]
@@ -213,7 +168,7 @@ class SubMaster:
 
   def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1
-    self.updated = dict.fromkeys(self.services, False)
+    self.updated = dict.fromkeys(self.updated, False)
     for msg in msgs:
       if msg is None:
         continue
@@ -222,26 +177,54 @@ class SubMaster:
       self.seen[s] = True
       self.updated[s] = True
 
-      self.freq_tracker[s].record_recv_time(cur_time)
+      if self.recv_time[s] > 1e-5:
+        self.recv_dts[s].append(cur_time - self.recv_time[s])
       self.recv_time[s] = cur_time
       self.recv_frame[s] = self.frame
       self.data[s] = getattr(msg, s)
       self.logMonoTime[s] = msg.logMonoTime
       self.valid[s] = msg.valid
 
-    for s in self.static_freq_services:
-      # alive if delay is within 10x the expected frequency; checks relaxed in simulator
-      self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
-      self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
+    for s in self.data:
+      if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation:
+        # alive if delay is within 10x the expected frequency
+        self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency)
+
+        # check average frequency; slow to fall, quick to recover
+        dts = self.recv_dts[s]
+        assert dts.maxlen is not None
+        recent_dts = list(dts)[-int(dts.maxlen / 10):]
+        try:
+          avg_freq = 1 / (sum(dts) / len(dts))
+          avg_freq_recent = 1 / (sum(recent_dts) / len(recent_dts))
+        except ZeroDivisionError:
+          avg_freq = 0
+          avg_freq_recent = 0
+
+        avg_freq_ok = self.min_freq[s] <= avg_freq <= self.max_freq[s]
+        recent_freq_ok = self.min_freq[s] <= avg_freq_recent <= self.max_freq[s]
+        self.freq_ok[s] = avg_freq_ok or recent_freq_ok
+      else:
+        self.freq_ok[s] = True
+        if self.simulation:
+          self.alive[s] = self.seen[s] # alive is defined as seen when simulation flag set
+        else:
+          self.alive[s] = True
 
   def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
-    return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
+    if service_list is None:
+      service_list = list(self.sock.keys())
+    return all(self.alive[s] for s in service_list if s not in self.ignore_alive)
 
   def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
-    return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
+    if service_list is None:
+      service_list = list(self.sock.keys())
+    return all(self.freq_ok[s] for s in service_list if self._check_avg_freq(s))
 
   def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
-    return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
+    if service_list is None:
+      service_list = list(self.sock.keys())
+    return all(self.valid[s] for s in service_list if s not in self.ignore_valid)
 
   def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
     return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)

cereal/messaging/bridge.cc

@@ -1,72 +0,0 @@
-#include <cassert>
-
-#include "cereal/messaging/msgq_to_zmq.h"
-#include "cereal/services.h"
-#include "common/util.h"
-
-ExitHandler do_exit;
-
-static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
-  std::vector<std::string> service_list;
-  for (const auto& it : services) {
-    std::string name = it.second.name;
-    bool in_whitelist = whitelist_str.find(name) != std::string::npos;
-    if (zmq_to_msgq && !in_whitelist) {
-      continue;
-    }
-    service_list.push_back(name);
-  }
-  return service_list;
-}
-
-void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  MsgqToZmq bridge;
-  bridge.run(endpoints, ip);
-}
-
-void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<BridgeZmqPoller>();
-  auto pub_context = std::make_unique<Context>();
-  auto sub_context = std::make_unique<BridgeZmqContext>();
-  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
-
-  for (auto endpoint : endpoints) {
-    auto pub_sock = new PubSocket();
-    auto sub_sock = new BridgeZmqSubSocket();
-    size_t queue_size = services.at(endpoint).queue_size;
-    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
-    sub_sock->connect(sub_context.get(), endpoint, ip, false);
-
-    poller->registerSocket(sub_sock);
-    sub2pub[sub_sock] = pub_sock;
-  }
-
-  while (!do_exit) {
-    for (auto sub_sock : poller->poll(100)) {
-      std::unique_ptr<Message> msg(sub_sock->receive(true));
-      if (msg) {
-        sub2pub[sub_sock]->sendMessage(msg.get());
-      }
-    }
-  }
-
-  // Clean up allocated sockets
-  for (auto &[sub_sock, pub_sock] : sub2pub) {
-    delete sub_sock;
-    delete pub_sock;
-  }
-}
-
-int main(int argc, char **argv) {
-  bool is_zmq_to_msgq = argc > 2;
-  std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1";
-  std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : "";
-  std::vector<std::string> endpoints = get_services(whitelist_str, is_zmq_to_msgq);
-
-  if (is_zmq_to_msgq) {
-    zmq_to_msgq(endpoints, ip);
-  } else {
-    msgq_to_zmq(endpoints, ip);
-  }
-  return 0;
-}

cereal/messaging/bridge_zmq.cc

@@ -1,170 +0,0 @@
-#include "cereal/messaging/bridge_zmq.h"
-
-#include <cassert>
-#include <cstring>
-#include <unistd.h>
-
-static size_t fnv1a_hash(const std::string &str) {
-  const size_t fnv_prime = 0x100000001b3;
-  size_t hash_value = 0xcbf29ce484222325;
-  for (char c : str) {
-    hash_value ^= (unsigned char)c;
-    hash_value *= fnv_prime;
-  }
-  return hash_value;
-}
-
-// FIXME: This is a hack to get the port number from the socket name, might have collisions.
-static int get_port(std::string endpoint) {
-  size_t hash_value = fnv1a_hash(endpoint);
-  int start_port = 8023;
-  int max_port = 65535;
-  return start_port + (hash_value % (max_port - start_port));
-}
-
-BridgeZmqContext::BridgeZmqContext() {
-  context = zmq_ctx_new();
-}
-
-BridgeZmqContext::~BridgeZmqContext() {
-  if (context != nullptr) {
-    zmq_ctx_term(context);
-  }
-}
-
-void BridgeZmqMessage::init(size_t sz) {
-  size = sz;
-  data = new char[size];
-}
-
-void BridgeZmqMessage::init(char *d, size_t sz) {
-  size = sz;
-  data = new char[size];
-  memcpy(data, d, size);
-}
-
-void BridgeZmqMessage::close() {
-  if (size > 0) {
-    delete[] data;
-  }
-  data = nullptr;
-  size = 0;
-}
-
-BridgeZmqMessage::~BridgeZmqMessage() {
-  close();
-}
-
-int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
-
-  if (conflate) {
-    int arg = 1;
-    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
-  }
-
-  int reconnect_ivl = 500;
-  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
-
-  full_endpoint = "tcp://" + address + ":";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  return zmq_connect(sock, full_endpoint.c_str());
-}
-
-void BridgeZmqSubSocket::setTimeout(int timeout) {
-  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
-}
-
-Message *BridgeZmqSubSocket::receive(bool non_blocking) {
-  zmq_msg_t msg;
-  assert(zmq_msg_init(&msg) == 0);
-
-  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
-  int rc = zmq_msg_recv(&msg, sock, flags);
-
-  Message *ret = nullptr;
-  if (rc >= 0) {
-    ret = new BridgeZmqMessage;
-    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
-  }
-
-  zmq_msg_close(&msg);
-  return ret;
-}
-
-BridgeZmqSubSocket::~BridgeZmqSubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  full_endpoint = "tcp://*:";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
-  pid = getpid();
-
-  return zmq_bind(sock, full_endpoint.c_str());
-}
-
-int BridgeZmqPubSocket::sendMessage(Message *message) {
-  assert(pid == getpid());
-  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
-}
-
-int BridgeZmqPubSocket::send(char *data, size_t size) {
-  assert(pid == getpid());
-  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
-}
-
-BridgeZmqPubSocket::~BridgeZmqPubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
-  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
-  polls[num_polls].socket = socket->getRawSocket();
-  polls[num_polls].events = ZMQ_POLLIN;
-
-  sockets.push_back(socket);
-  num_polls++;
-}
-
-std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
-  std::vector<BridgeZmqSubSocket *> ret;
-
-  int rc = zmq_poll(polls, num_polls, timeout);
-  if (rc < 0) {
-    return ret;
-  }
-
-  for (size_t i = 0; i < num_polls; i++) {
-    if (polls[i].revents) {
-      ret.push_back(sockets[i]);
-    }
-  }
-
-  return ret;
-}

cereal/messaging/bridge_zmq.h

@@ -1,72 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <string>
-#include <vector>
-
-#include <zmq.h>
-
-#include "msgq/ipc.h"
-
-class BridgeZmqContext {
-public:
-  BridgeZmqContext();
-  void *getRawContext() { return context; }
-  ~BridgeZmqContext();
-
-private:
-  void *context = nullptr;
-};
-
-class BridgeZmqMessage : public Message {
-public:
-  void init(size_t size);
-  void init(char *data, size_t size);
-  void close();
-  size_t getSize() { return size; }
-  char *getData() { return data; }
-  ~BridgeZmqMessage();
-
-private:
-  char *data = nullptr;
-  size_t size = 0;
-};
-
-class BridgeZmqSubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
-  void setTimeout(int timeout);
-  Message *receive(bool non_blocking = false);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqSubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-};
-
-class BridgeZmqPubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
-  int sendMessage(Message *message);
-  int send(char *data, size_t size);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqPubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-  int pid = -1;
-};
-
-class BridgeZmqPoller {
-public:
-  void registerSocket(BridgeZmqSubSocket *socket);
-  std::vector<BridgeZmqSubSocket *> poll(int timeout);
-
-private:
-  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
-  std::vector<BridgeZmqSubSocket *> sockets;
-  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
-  size_t num_polls = 0;
-};

cereal/messaging/messaging.h

@@ -1,102 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <map>
-#include <string>
-#include <vector>
-#include <utility>
-
-#include <capnp/serialize.h>
-
-#include "cereal/gen/cpp/log.capnp.h"
-#include "common/timing.h"
-#include "msgq/ipc.h"
-
-class SubMaster {
-public:
-  SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll = {},
-            const char *address = nullptr, const std::vector<const char *> &ignore_alive = {});
-  void update(int timeout = 1000);
-  void update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages);
-  inline bool allAlive(const std::vector<const char *> &service_list = {}) { return all_(service_list, false, true); }
-  inline bool allValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, false); }
-  inline bool allAliveAndValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, true); }
-  void drain();
-  ~SubMaster();
-
-  uint64_t frame = 0;
-  bool updated(const char *name) const;
-  bool alive(const char *name) const;
-  bool valid(const char *name) const;
-  uint64_t rcv_frame(const char *name) const;
-  uint64_t rcv_time(const char *name) const;
-  cereal::Event::Reader &operator[](const char *name) const;
-
-private:
-  bool all_(const std::vector<const char *> &service_list, bool valid, bool alive);
-  Poller *poller_ = nullptr;
-  struct SubMessage;
-  std::map<SubSocket *, SubMessage *> messages_;
-  std::map<std::string, SubMessage *> services_;
-};
-
-class MessageBuilder : public capnp::MallocMessageBuilder {
-public:
-  MessageBuilder() = default;
-
-  cereal::Event::Builder initEvent(bool valid = true) {
-    cereal::Event::Builder event = initRoot<cereal::Event>();
-    event.setLogMonoTime(nanos_since_boot());
-    event.setValid(valid);
-    return event;
-  }
-
-  kj::ArrayPtr<capnp::byte> toBytes() {
-    heapArray_ = capnp::messageToFlatArray(*this);
-    return heapArray_.asBytes();
-  }
-
-  size_t getSerializedSize() {
-    return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
-  }
-
-  int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
-    size_t serialized_size = getSerializedSize();
-    if (serialized_size > buffer_size) { return -1; }
-    kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
-    capnp::writeMessage(out, *this);
-    return serialized_size;
-  }
-
-private:
-  kj::Array<capnp::word> heapArray_;
-};
-
-class PubMaster {
-public:
-  PubMaster(const std::vector<const char *> &service_list);
-  inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
-  int send(const char *name, MessageBuilder &msg);
-  ~PubMaster();
-
-private:
-  std::map<std::string, PubSocket *> sockets_;
-};
-
-class AlignedBuffer {
-public:
-  kj::ArrayPtr<const capnp::word> align(const char *data, const size_t size) {
-    words_size = size / sizeof(capnp::word) + 1;
-    if (aligned_buf.size() < words_size) {
-      aligned_buf = kj::heapArray<capnp::word>(words_size < 512 ? 512 : words_size);
-    }
-    memcpy(aligned_buf.begin(), data, size);
-    return aligned_buf.slice(0, words_size);
-  }
-  inline kj::ArrayPtr<const capnp::word> align(Message *m) {
-    return align(m->getData(), m->getSize());
-  }
-private:
-  kj::Array<capnp::word> aligned_buf;
-  size_t words_size;
-};

cereal/messaging/msgq_to_zmq.cc

@@ -1,146 +0,0 @@
-#include "cereal/messaging/msgq_to_zmq.h"
-
-#include <cassert>
-
-#include "cereal/services.h"
-#include "common/util.h"
-
-extern ExitHandler do_exit;
-
-// Max messages to process per socket per poll
-constexpr int MAX_MESSAGES_PER_SOCKET = 50;
-
-static std::string recv_zmq_msg(void *sock) {
-  zmq_msg_t msg;
-  zmq_msg_init(&msg);
-  std::string ret;
-  if (zmq_msg_recv(&msg, sock, 0) > 0) {
-    ret.assign((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
-  }
-  zmq_msg_close(&msg);
-  return ret;
-}
-
-void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
-  msgq_context = std::make_unique<Context>();
-
-  // Create ZMQPubSockets for each endpoint
-  for (const auto &endpoint : endpoints) {
-    auto &socket_pair = socket_pairs.emplace_back();
-    socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
-    int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
-    if (ret != 0) {
-      printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
-      return;
-    }
-  }
-
-  // Start ZMQ monitoring thread to monitor socket events
-  std::thread thread(&MsgqToZmq::zmqMonitorThread, this);
-
-  // Main loop for processing messages
-  while (!do_exit) {
-    {
-      std::unique_lock lk(mutex);
-      cv.wait(lk, [this]() { return do_exit || !sub2pub.empty(); });
-      if (do_exit) break;
-
-      for (auto sub_sock : msgq_poller->poll(100)) {
-        // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
-        for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
-          auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
-          if (!msg) break;
-
-          while (pub_sock->sendMessage(msg.get()) == -1) {
-            if (errno != EINTR) break;
-          }
-        }
-      }
-    }
-    util::sleep_for(1);  // Give zmqMonitorThread a chance to acquire the mutex
-  }
-
-  thread.join();
-}
-
-void MsgqToZmq::zmqMonitorThread() {
-  std::vector<zmq_pollitem_t> pollitems;
-
-  // Set up ZMQ monitor for each pub socket
-  for (int i = 0; i < socket_pairs.size(); ++i) {
-    std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
-
-    void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
-    zmq_connect(monitor_socket, addr.c_str());
-    pollitems.emplace_back(zmq_pollitem_t{.socket = monitor_socket, .events = ZMQ_POLLIN});
-  }
-
-  while (!do_exit) {
-    int ret = zmq_poll(pollitems.data(), pollitems.size(), 1000);
-    if (ret < 0) {
-      if (errno == EINTR) {
-        // Due to frequent EINTR signals from msgq, introduce a brief delay (200 ms)
-        // to reduce CPU usage during retry attempts.
-        util::sleep_for(200);
-      }
-      continue;
-    }
-
-    for (int i = 0; i < pollitems.size(); ++i) {
-      if (pollitems[i].revents & ZMQ_POLLIN) {
-        // First frame in message contains event number and value
-        std::string frame = recv_zmq_msg(pollitems[i].socket);
-        if (frame.empty()) continue;
-
-        uint16_t event_type = *(uint16_t *)(frame.data());
-
-        // Second frame in message contains event address
-        frame = recv_zmq_msg(pollitems[i].socket);
-        if (frame.empty()) continue;
-
-        std::unique_lock lk(mutex);
-        auto &pair = socket_pairs[i];
-        if (event_type & ZMQ_EVENT_ACCEPTED) {
-          printf("socket [%s] connected\n", pair.endpoint.c_str());
-          if (++pair.connected_clients == 1) {
-            // Create new MSGQ subscriber socket and map to ZMQ publisher
-            pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
-            sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
-            registerSockets();
-          }
-        } else if (event_type & ZMQ_EVENT_DISCONNECTED) {
-          printf("socket [%s] disconnected\n", pair.endpoint.c_str());
-          if (pair.connected_clients == 0 || --pair.connected_clients == 0) {
-            // Remove MSGQ subscriber socket from mapping and reset it
-            sub2pub.erase(pair.sub_sock.get());
-            pair.sub_sock.reset(nullptr);
-            registerSockets();
-          }
-        }
-        cv.notify_one();
-      }
-    }
-  }
-
-  // Clean up monitor sockets
-  for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
-    zmq_close(pollitems[i].socket);
-  }
-  cv.notify_one();
-}
-
-void MsgqToZmq::registerSockets() {
-  msgq_poller = std::make_unique<MSGQPoller>();
-  for (const auto &socket_pair : socket_pairs) {
-    if (socket_pair.sub_sock) {
-      msgq_poller->registerSocket(socket_pair.sub_sock.get());
-    }
-  }
-}

cereal/messaging/msgq_to_zmq.h

@@ -1,36 +0,0 @@
-#pragma once
-
-#include <condition_variable>
-#include <map>
-#include <memory>
-#include <mutex>
-#include <string>
-#include <vector>
-
-#include "msgq/impl_msgq.h"
-#include "cereal/messaging/bridge_zmq.h"
-
-class MsgqToZmq {
-public:
-  MsgqToZmq() {}
-  void run(const std::vector<std::string> &endpoints, const std::string &ip);
-
-protected:
-  void registerSockets();
-  void zmqMonitorThread();
-
-  struct SocketPair {
-    std::string endpoint;
-    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
-    std::unique_ptr<MSGQSubSocket> sub_sock;
-    int connected_clients = 0;
-  };
-
-  std::unique_ptr<Context> msgq_context;
-  std::unique_ptr<BridgeZmqContext> zmq_context;
-  std::mutex mutex;
-  std::condition_variable cv;
-  std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
-  std::vector<SocketPair> socket_pairs;
-};

cereal/messaging/socketmaster.cc

@@ -1,204 +0,0 @@
-#include <assert.h>
-#include <stdlib.h>
-#include <string>
-#include <mutex>
-
-#include "cereal/services.h"
-#include "cereal/messaging/messaging.h"
-
-const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
-
-static inline bool inList(const std::vector<const char *> &list, const char *value) {
-  for (auto &v : list) {
-    if (strcmp(value, v) == 0) return true;
-  }
-  return false;
-}
-
-class MessageContext {
-public:
-  MessageContext() : ctx_(nullptr) {}
-  ~MessageContext() { delete ctx_; }
-  inline Context *context() {
-    std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
-    return ctx_;
-  }
-private:
-  Context *ctx_;
-  std::once_flag init_flag;
-};
-
-MessageContext message_context;
-
-struct SubMaster::SubMessage {
-  std::string name;
-  SubSocket *socket = nullptr;
-  float freq = 0.0f;
-  bool updated = false, alive = false, valid = false, ignore_alive;
-  uint64_t rcv_time = 0, rcv_frame = 0;
-  void *allocated_msg_reader = nullptr;
-  bool is_polled = false;
-  capnp::FlatArrayMessageReader *msg_reader = nullptr;
-  AlignedBuffer aligned_buf;
-  cereal::Event::Reader event;
-};
-
-SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
-                     const char *address, const std::vector<const char *> &ignore_alive) {
-  poller_ = Poller::create();
-  for (auto name : service_list) {
-    assert(services.count(std::string(name)) > 0);
-
-    service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
-    assert(socket != 0);
-    bool is_polled = inList(poll, name) || poll.empty();
-    if (is_polled) poller_->registerSocket(socket);
-    SubMessage *m = new SubMessage{
-      .name = name,
-      .socket = socket,
-      .freq = serv.frequency,
-      .ignore_alive = inList(ignore_alive, name),
-      .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
-      .is_polled = is_polled};
-    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
-    messages_[socket] = m;
-    services_[name] = m;
-  }
-}
-
-void SubMaster::update(int timeout) {
-  for (auto &kv : messages_) kv.second->updated = false;
-
-  auto sockets = poller_->poll(timeout);
-
-  // add non-polled sockets for non-blocking receive
-  for (auto &kv : messages_) {
-    SubMessage *m = kv.second;
-    SubSocket *s = kv.first;
-    if (!m->is_polled) sockets.push_back(s);
-  }
-
-  uint64_t current_time = nanos_since_boot();
-
-  std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
-
-  for (auto s : sockets) {
-    Message *msg = s->receive(true);
-    if (msg == nullptr) continue;
-
-    SubMessage *m = messages_.at(s);
-
-    m->msg_reader->~FlatArrayMessageReader();
-    capnp::ReaderOptions options;
-    options.traversalLimitInWords = kj::maxValue; // Don't limit
-    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options);
-    delete msg;
-    messages.push_back({m->name, m->msg_reader->getRoot<cereal::Event>()});
-  }
-
-  update_msgs(current_time, messages);
-}
-
-void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
-  if (++frame == UINT64_MAX) frame = 1;
-
-  for (auto &kv : messages) {
-    auto m_find = services_.find(kv.first);
-    if (m_find == services_.end()){
-      continue;
-    }
-    SubMessage *m = m_find->second;
-    m->event = kv.second;
-    m->updated = true;
-    m->rcv_time = current_time;
-    m->rcv_frame = frame;
-    m->valid = m->event.getValid();
-    if (SIMULATION) m->alive = true;
-  }
-
-  if (!SIMULATION) {
-    for (auto &kv : messages_) {
-      SubMessage *m = kv.second;
-      m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
-    }
-  }
-}
-
-bool SubMaster::all_(const std::vector<const char *> &service_list, bool valid, bool alive) {
-  int found = 0;
-  for (auto &kv : messages_) {
-    SubMessage *m = kv.second;
-    if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
-      found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive));
-    }
-  }
-  return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
-}
-
-void SubMaster::drain() {
-  while (true) {
-    auto polls = poller_->poll(0);
-    if (polls.size() == 0)
-      break;
-
-    for (auto sock : polls) {
-      Message *msg = sock->receive(true);
-      delete msg;
-    }
-  }
-}
-
-bool SubMaster::updated(const char *name) const {
-  return services_.at(name)->updated;
-}
-
-bool SubMaster::alive(const char *name) const {
-  return services_.at(name)->alive;
-}
-
-bool SubMaster::valid(const char *name) const {
-  return services_.at(name)->valid;
-}
-
-uint64_t SubMaster::rcv_frame(const char *name) const {
-  return services_.at(name)->rcv_frame;
-}
-
-uint64_t SubMaster::rcv_time(const char *name) const {
-  return services_.at(name)->rcv_time;
-}
-
-cereal::Event::Reader &SubMaster::operator[](const char *name) const {
-  return services_.at(name)->event;
-}
-
-SubMaster::~SubMaster() {
-  delete poller_;
-  for (auto &kv : messages_) {
-    SubMessage *m = kv.second;
-    m->msg_reader->~FlatArrayMessageReader();
-    free(m->allocated_msg_reader);
-    delete m->socket;
-    delete m;
-  }
-}
-
-PubMaster::PubMaster(const std::vector<const char *> &service_list) {
-  for (auto name : service_list) {
-    assert(services.count(name) > 0);
-    service serv = services.at(std::string(name));
-    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
-    assert(socket);
-    sockets_[name] = socket;
-  }
-}
-
-int PubMaster::send(const char *name, MessageBuilder &msg) {
-  auto bytes = msg.toBytes();
-  return send(name, bytes.begin(), bytes.size());
-}
-
-PubMaster::~PubMaster() {
-  for (auto s : sockets_) delete s.second;
-}

cereal/messaging/tests/test_messaging.py

@@ -1,185 +0,0 @@
-import os
-import capnp
-import multiprocessing
-import numbers
-import random
-import threading
-import time
-from openpilot.common.parameterized import parameterized
-import pytest
-
-from cereal import log, car
-import cereal.messaging as messaging
-from cereal.services import SERVICE_LIST
-
-events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
-
-def random_sock():
-  return random.choice(events)
-
-def random_socks(num_socks=10):
-  return list({random_sock() for _ in range(num_socks)})
-
-def random_bytes(length=1000):
-  return bytes([random.randrange(0xFF) for _ in range(length)])
-
-def zmq_sleep(t=1):
-  if "ZMQ" in os.environ:
-    time.sleep(t)
-
-
-# TODO: this should take any capnp struct and returrn a msg with random populated data
-def random_carstate():
-  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
-  msg = messaging.new_message("carState")
-  cs = msg.carState
-  for f in fields:
-    setattr(cs, f, random.random() * 10)
-  return msg
-
-# TODO: this should compare any capnp structs
-def assert_carstate(cs1, cs2):
-  for f in car.CarState.schema.non_union_fields:
-    # TODO: check all types
-    val1, val2 = getattr(cs1, f), getattr(cs2, f)
-    if isinstance(val1, numbers.Number):
-      assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
-
-def delayed_send(delay, sock, dat):
-  def send_func():
-    sock.send(dat)
-  threading.Timer(delay, send_func).start()
-
-
-class TestMessaging:
-  def setUp(self):
-    # TODO: ZMQ tests are too slow; all sleeps will need to be
-    # replaced with logic to block on the necessary condition
-    if "ZMQ" in os.environ:
-      pytest.skip()
-
-    # ZMQ pub socket takes too long to die
-    # sleep to prevent multiple publishers error between tests
-    zmq_sleep()
-
-  @parameterized.expand(events)
-  def test_new_message(self, evt):
-    try:
-      msg = messaging.new_message(evt)
-    except capnp.lib.capnp.KjException:
-      msg = messaging.new_message(evt, random.randrange(200))
-    assert (time.monotonic() - msg.logMonoTime) < 0.1
-    assert not msg.valid
-    assert evt == msg.which()
-
-  @parameterized.expand(events)
-  def test_pub_sock(self, evt):
-    messaging.pub_sock(evt)
-
-  @parameterized.expand(events)
-  def test_sub_sock(self, evt):
-    messaging.sub_sock(evt)
-
-  @parameterized.expand([
-    (messaging.drain_sock, capnp._DynamicStructReader),
-    (messaging.drain_sock_raw, bytes),
-  ])
-  def test_drain_sock(self, func, expected_type):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=1000)
-    zmq_sleep()
-
-    # no wait and no msgs in queue
-    msgs = func(sub_sock)
-    assert isinstance(msgs, list)
-    assert len(msgs) == 0
-
-    # no wait but msgs are queued up
-    num_msgs = random.randrange(3, 10)
-    for _ in range(num_msgs):
-      pub_sock.send(messaging.new_message(sock).to_bytes())
-    time.sleep(0.1)
-    msgs = func(sub_sock)
-    assert isinstance(msgs, list)
-    assert all(isinstance(msg, expected_type) for msg in msgs)
-    assert len(msgs) == num_msgs
-
-  def test_recv_sock(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=100)
-    zmq_sleep()
-
-    # no wait and no msg in queue, socket should timeout
-    recvd = messaging.recv_sock(sub_sock)
-    assert recvd is None
-
-    # no wait and one msg in queue
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    time.sleep(0.01)
-    recvd = messaging.recv_sock(sub_sock)
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    # https://github.com/python/mypy/issues/13038
-    assert_carstate(msg.carState, recvd.carState)
-
-  def test_recv_one(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=1000)
-    zmq_sleep()
-
-    # no msg in queue, socket should timeout
-    recvd = messaging.recv_one(sub_sock)
-    assert recvd is None
-
-    # one msg in queue
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    recvd = messaging.recv_one(sub_sock)
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
-
-  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
-  def test_recv_one_or_none(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock)
-    zmq_sleep()
-
-    # no msg in queue, socket shouldn't block
-    recvd = messaging.recv_one_or_none(sub_sock)
-    assert recvd is None
-
-    # one msg in queue
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    recvd = messaging.recv_one_or_none(sub_sock)
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)
-
-  def test_recv_one_retry(self):
-    sock = "carState"
-    sock_timeout = 0.1
-    pub_sock = messaging.pub_sock(sock)
-    sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
-    zmq_sleep()
-
-    # this test doesn't work with ZMQ since multiprocessing interrupts it
-    if "ZMQ" not in os.environ:
-      # wait 5 socket timeouts and make sure it's still retrying
-      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
-      p.start()
-      time.sleep(sock_timeout*5)
-      assert p.is_alive()
-      p.terminate()
-
-    # wait 5 socket timeouts before sending
-    msg = random_carstate()
-    start_time = time.monotonic()
-    delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
-    recvd = messaging.recv_one_retry(sub_sock)
-    assert (time.monotonic() - start_time) >= sock_timeout*5
-    assert isinstance(recvd, capnp._DynamicStructReader)
-    assert_carstate(msg.carState, recvd.carState)

cereal/messaging/tests/test_pub_sub_master.py

@@ -1,160 +0,0 @@
-import random
-import time
-from typing import Sized, cast
-
-import cereal.messaging as messaging
-from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
-                                                  random_bytes, random_carstate, assert_carstate, \
-                                                  zmq_sleep
-from cereal.services import SERVICE_LIST
-
-
-class TestSubMaster:
-
-  def setup_method(self):
-    # ZMQ pub socket takes too long to die
-    # sleep to prevent multiple publishers error between tests
-    zmq_sleep(3)
-
-  def test_init(self):
-    sm = messaging.SubMaster(events)
-    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
-              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      assert len(cast(Sized, p)) == len(events)
-
-  def test_init_state(self):
-    socks = random_socks()
-    sm = messaging.SubMaster(socks)
-    assert sm.frame == -1
-    assert not any(sm.updated.values())
-    assert not any(sm.seen.values())
-    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
-    assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
-    assert all(t == 0. for t in sm.recv_time.values())
-    assert all(f == 0 for f in sm.recv_frame.values())
-    assert all(t == 0 for t in sm.logMonoTime.values())
-
-    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
-              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      assert len(cast(Sized, p)) == len(socks)
-
-  def test_getitem(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sm = messaging.SubMaster([sock,])
-    zmq_sleep()
-
-    msg = random_carstate()
-    pub_sock.send(msg.to_bytes())
-    sm.update(1000)
-    assert_carstate(msg.carState, sm[sock])
-
-  # TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
-  def test_update(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sm = messaging.SubMaster([sock,])
-    zmq_sleep()
-
-    for i in range(10):
-      msg = messaging.new_message(sock)
-      pub_sock.send(msg.to_bytes())
-      sm.update(1000)
-      assert sm.frame == i
-      assert all(sm.updated.values())
-
-  def test_update_timeout(self):
-    sock = random_sock()
-    sm = messaging.SubMaster([sock,])
-    timeout = random.randrange(1000, 3000)
-    start_time = time.monotonic()
-    sm.update(timeout)
-    t = time.monotonic() - start_time
-    assert t >= timeout/1000.
-    assert t < 3
-    assert not any(sm.updated.values())
-
-  def test_avg_frequency_checks(self):
-    for poll in (True, False):
-      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
-                               poll=("modelV2" if poll else None),
-                               frequency=(20. if not poll else None))
-
-      checks = {
-        "carState": (20, 20),
-        "modelV2": (20, 20 if poll else 10),
-        "cameraOdometry": (20, 10),
-        "liveCalibration": (4, 4),
-        "carParams": (None, None),
-        "userBookmark": (None, None),
-      }
-
-      for service, (max_freq, min_freq) in checks.items():
-        if max_freq is not None:
-          assert sm._check_avg_freq(service)
-          assert sm.freq_tracker[service].max_freq == max_freq*1.2
-          assert sm.freq_tracker[service].min_freq == min_freq*0.8
-        else:
-          assert not sm._check_avg_freq(service)
-
-  def test_alive(self):
-    pass
-
-  def test_ignore_alive(self):
-    pass
-
-  def test_valid(self):
-    pass
-
-  # SubMaster should always conflate
-  def test_conflate(self):
-    sock = "carState"
-    pub_sock = messaging.pub_sock(sock)
-    sm = messaging.SubMaster([sock,])
-
-    n = 10
-    for i in range(n+1):
-      msg = messaging.new_message(sock)
-      msg.carState.vEgo = i
-      pub_sock.send(msg.to_bytes())
-      time.sleep(0.01)
-    sm.update(1000)
-    assert sm[sock].vEgo == n
-
-
-class TestPubMaster:
-
-  def setup_method(self):
-    # ZMQ pub socket takes too long to die
-    # sleep to prevent multiple publishers error between tests
-    zmq_sleep(3)
-
-  def test_init(self):
-    messaging.PubMaster(events)
-
-  def test_send(self):
-    socks = random_socks()
-    pm = messaging.PubMaster(socks)
-    sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
-    zmq_sleep()
-
-    # PubMaster accepts either a capnp msg builder or bytes
-    for capnp in [True, False]:
-      for i in range(100):
-        sock = socks[i % len(socks)]
-
-        if capnp:
-          try:
-            msg = messaging.new_message(sock)
-          except Exception:
-            msg = messaging.new_message(sock, random.randrange(50))
-        else:
-          msg = random_bytes()
-
-        pm.send(sock, msg)
-        recvd = sub_socks[sock].receive()
-
-        if capnp:
-          msg.clear_write_flag()
-          msg = msg.to_bytes()
-        assert msg == recvd, i

cereal/messaging/tests/test_services.py

@@ -1,21 +0,0 @@
-import os
-import tempfile
-from typing import Dict
-from openpilot.common.parameterized import parameterized
-
-import cereal.services as services
-from cereal.services import SERVICE_LIST
-
-
-class TestServices:
-
-  @parameterized.expand(SERVICE_LIST.keys())
-  def test_services(self, s):
-    service = SERVICE_LIST[s]
-    assert service.frequency <= 104
-    assert service.decimation != 0
-
-  def test_generated_header(self):
-    with tempfile.NamedTemporaryFile(suffix=".h") as f:
-      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
-      assert ret == 0, "generated services header is not valid C"

cereal/services.py

@@ -1,57 +1,45 @@
 #!/usr/bin/env python3
-from enum import IntEnum
 from typing import Optional
 
 
-# TODO: this should be automatically determined using the capnp schema
-class QueueSize(IntEnum):
-  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
-  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
-  SMALL = 250 * 1024          # 250KB - most services
-
-
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
-               queue_size: QueueSize = QueueSize.SMALL):
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
-    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
-  "magnetometer": (True, 25.),
+  "accelerometer2": (True, 100., 100),
+  "magnetometer": (True, 25., 25),
   "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "touch": (True, 20., 1),
-  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
-  "selfdriveState": (True, 100., 10),
+  "can": (True, 100., 1223),  # decimation gives ~5 msgs in a full segment
+  "controlsState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
+  "sendcan": (True, 100., 139),
   "logMessage": (True, 0.),
   "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
-  "liveDelay": (True, 4., 1),
   "androidLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
-  "longitudinalPlan": (True, 20., 10),
-  "lateralManeuverPlan": (True, 20.),
-  "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15, QueueSize.BIG),
+  "longitudinalPlan": (True, 20., 5),
+  "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
@@ -59,10 +47,10 @@ _services: dict[str, tuple] = {
   "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
-  "livePose": (True, 20., 4),
+  "liveLocationKalman": (True, 20., 5),
   "liveParameters": (True, 20., 5),
-  "cameraOdometry": (True, 20., 10),
-  "thumbnail": (True, 1 / 60., 1),
+  "cameraOdometry": (True, 20., 5),
+  "thumbnail": (True, 0.2, 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
   "roadCameraState": (True, 20., 20),
@@ -72,34 +60,39 @@ _services: dict[str, tuple] = {
   "driverMonitoringState": (True, 20., 10),
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
-  "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., None, QueueSize.BIG),
+  "modelV2": (True, 20., 40),
   "managerState": (True, 2., 1),
   "uploaderState": (True, 0., 1),
   "navInstruction": (True, 1., 10),
   "navRoute": (True, 0.),
   "navThumbnail": (True, 0.),
+  "navModel": (True, 2., 4.),
+  "mapRenderState": (True, 2., 1.),
+  "uiPlan": (True, 20., 40.),
   "qRoadEncodeIdx": (False, 20.),
-  "userBookmark": (True, 0., 1),
-  "soundPressure": (True, 10., 10),
-  "rawAudioData": (False, 20.),
-  "bookmarkButton": (True, 0., 1),
-  "audioFeedback": (True, 0., 1),
-  "roadEncodeData": (False, 20., None, QueueSize.BIG),
-  "driverEncodeData": (False, 20., None, QueueSize.BIG),
-  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
+  "userFlag": (True, 0., 1),
+  "microphone": (True, 10., 10),
+
+  # FrogPilot
+  "frogpilotCarControl": (True, 100., 10),
+  "frogpilotCarState": (True, 100., 10),
+  "frogpilotDeviceState": (True, 2., 1),
+  "frogpilotNavigation": (True, 1., 10),
+  "frogpilotPlan": (True, 20., 5),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
-  "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamWideRoadEncodeData": (False, 20.),
+  "livestreamRoadEncodeData": (False, 20.),
+  "livestreamDriverEncodeData": (False, 20.),
   "customReservedRawData0": (True, 0.),
   "customReservedRawData1": (True, 0.),
   "customReservedRawData2": (True, 0.),
@@ -117,13 +110,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
+  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation, v.queue_size)
+    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation)
   h += "};\n"
 
   h += "#endif\n"

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -1,14 +1,25 @@
-Import('env', 'envCython')
+Import('env', 'envCython', 'arch')
 
 common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
-  'ratekeeper.cc',
+  'i2c.cc',
+  'watchdog.cc',
+  'ratekeeper.cc'
 ]
 
+if arch != "Darwin":
+  common_libs.append('gpio.cc')
+
 _common = env.Library('common', common_libs, LIBS="json11")
-Export('_common')
+
+files = [
+  'clutil.cc',
+]
+
+_gpucommon = env.Library('gpucommon', files)
+Export('_common', '_gpucommon')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
@@ -18,6 +29,11 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-common_python = [params_python]
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
 
 Export('common_python')

common/api.py

@@ -1,62 +0,0 @@
-import jwt
-import os
-import requests
-from datetime import datetime, timedelta, UTC
-from openpilot.system.hardware.hw import Paths
-from openpilot.system.version import get_version
-
-API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
-
-# name: jwt signature algorithm
-KEYS = {"id_rsa": "RS256",
-        "id_ecdsa": "ES256"}
-
-
-class Api:
-  def __init__(self, dongle_id):
-    self.dongle_id = dongle_id
-    self.jwt_algorithm, self.private_key, _ = get_key_pair()
-
-  def get(self, *args, **kwargs):
-    return self.request('GET', *args, **kwargs)
-
-  def post(self, *args, **kwargs):
-    return self.request('POST', *args, **kwargs)
-
-  def request(self, method, endpoint, timeout=None, access_token=None, **params):
-    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
-
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    now = datetime.now(UTC).replace(tzinfo=None)
-    payload = {
-      'identity': self.dongle_id,
-      'nbf': now,
-      'iat': now,
-      'exp': now + timedelta(hours=expiry_hours)
-    }
-    if payload_extra is not None:
-      payload.update(payload_extra)
-    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
-    if isinstance(token, bytes):
-      token = token.decode('utf8')
-    return token
-
-
-def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
-  headers = {}
-  if access_token is not None:
-    headers['Authorization'] = "JWT " + access_token
-
-  headers['User-Agent'] = "openpilot-" + get_version()
-
-  # TODO: add session to Api
-  req = requests if session is None else session
-  return req.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)
-
-
-def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-  for key in KEYS:
-    if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
-      with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
-        return KEYS[key], private.read(), public.read()
-  return None, None, None

common/api/__init__.py

@@ -0,0 +1,46 @@
+import jwt
+import os
+import requests
+from datetime import datetime, timedelta
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.version import get_version
+
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
+
+class Api:
+  def __init__(self, dongle_id):
+    self.dongle_id = dongle_id
+    with open(Paths.persist_root()+'/comma/id_rsa') as f:
+      self.private_key = f.read()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def get_token(self, expiry_hours=1):
+    now = datetime.utcnow()
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours)
+    }
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+
+
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+  headers = {}
+  if access_token is not None:
+    headers['Authorization'] = "JWT " + access_token
+
+  headers['User-Agent'] = "openpilot-" + get_version()
+
+  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)

common/constants.py

@@ -1,23 +0,0 @@
-import numpy as np
-
-# conversions
-class CV:
-  # Speed
-  MPH_TO_KPH = 1.609344
-  KPH_TO_MPH = 1. / MPH_TO_KPH
-  MS_TO_KPH = 3.6
-  KPH_TO_MS = 1. / MS_TO_KPH
-  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
-  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
-  MS_TO_KNOTS = 1.9438
-  KNOTS_TO_MS = 1. / MS_TO_KNOTS
-
-  # Angle
-  DEG_TO_RAD = np.pi / 180.
-  RAD_TO_DEG = 1. / DEG_TO_RAD
-
-  # Mass
-  LB_TO_KG = 0.453592
-
-
-ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

common/conversions.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+class Conversions:
+  # Speed
+  MPH_TO_KPH = 1.609344
+  KPH_TO_MPH = 1. / MPH_TO_KPH
+  MS_TO_KPH = 3.6
+  KPH_TO_MS = 1. / MS_TO_KPH
+  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
+  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
+  METER_TO_FOOT = 3.28084
+  FOOT_TO_METER = 1. / METER_TO_FOOT
+  CM_TO_INCH = 1. / 2.54
+  INCH_TO_CM = 1. / CM_TO_INCH
+  MS_TO_KNOTS = 1.9438
+  KNOTS_TO_MS = 1. / MS_TO_KNOTS
+
+  # Angle
+  DEG_TO_RAD = np.pi / 180.
+  RAD_TO_DEG = 1. / DEG_TO_RAD
+
+  # Mass
+  LB_TO_KG = 0.453592

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/ffi_wrapper.py

@@ -0,0 +1,8 @@
+import platform
+
+
+def suffix():
+  if platform.system() == "Darwin":
+    return ".dylib"
+  else:
+    return ".so"

common/file_chunker.py

@@ -1,37 +0,0 @@
-import math
-import os
-from pathlib import Path
-
-CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
-
-def get_chunk_name(name, idx, num_chunks):
-  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
-
-def get_manifest_path(name):
-  return f"{name}.chunkmanifest"
-
-def get_chunk_paths(path, file_size):
-  num_chunks = math.ceil(file_size / CHUNK_SIZE)
-  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
-
-def chunk_file(path, targets):
-  manifest_path, *chunk_paths = targets
-  with open(path, 'rb') as f:
-    data = f.read()
-  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
-  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
-  for i, chunk_path in enumerate(chunk_paths):
-    with open(chunk_path, 'wb') as f:
-      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
-  Path(manifest_path).write_text(str(len(chunk_paths)))
-  os.remove(path)
-
-
-def read_file_chunked(path):
-  manifest_path = get_manifest_path(path)
-  if os.path.isfile(manifest_path):
-    num_chunks = int(Path(manifest_path).read_text().strip())
-    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
-  if os.path.isfile(path):
-    return Path(path).read_bytes()
-  raise FileNotFoundError(path)

common/file_helpers.py

@@ -0,0 +1,37 @@
+import os
+import tempfile
+import contextlib
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+                        overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)

common/filter_simple.py

@@ -1,4 +1,5 @@
 class FirstOrderFilter:
+  # first order filter
   def __init__(self, x0, rc, dt, initialized=True):
     self.x = x0
     self.dt = dt
@@ -15,20 +16,3 @@ class FirstOrderFilter:
       self.initialized = True
       self.x = x
     return self.x
-
-
-class BounceFilter(FirstOrderFilter):
-  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
-    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
-    self.bounce = bounce
-    super().__init__(x0, rc, dt, initialized)
-
-  def update(self, x):
-    super().update(x)
-    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
-    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
-    self.velocity.update(0.0)
-    if abs(self.velocity.x) < 1e-3:
-      self.velocity.x = 0.0
-    self.x += self.velocity.x
-    return self.x

common/git.py

@@ -1,30 +1,30 @@
 from functools import cache
 import subprocess
-from openpilot.common.utils import run_cmd, run_cmd_default
+from openpilot.common.run import run_cmd, run_cmd_default
 
 
 @cache
-def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
   return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
 
 
 @cache
-def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
   return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
 
 
 @cache
-def get_short_branch(cwd: str | None = None) -> str:
+def get_short_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
 
 
 @cache
-def get_branch(cwd: str | None = None) -> str:
+def get_branch(cwd: str = None) -> str:
   return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
 
 
 @cache
-def get_origin(cwd: str | None = None) -> str:
+def get_origin(cwd: str = None) -> str:
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
@@ -34,7 +34,7 @@ def get_origin(cwd: str | None = None) -> str:
 
 
 @cache
-def get_normalized_origin(cwd: str | None = None) -> str:
+def get_normalized_origin(cwd: str = None) -> str:
   return get_origin(cwd) \
     .replace("git@", "", 1) \
     .replace(".git", "", 1) \

common/gpio.py

@@ -1,6 +1,4 @@
 import os
-import fcntl
-import ctypes
 from functools import cache
 
 def gpio_init(pin: int, output: bool) -> None:
@@ -54,36 +52,3 @@ def get_irqs_for_action(action: str) -> list[str]:
       if irq.isdigit() and action in get_irq_action(irq):
         ret.append(irq)
   return ret
-
-# *** gpiochip ***
-
-class gpioevent_data(ctypes.Structure):
-  _fields_ = [
-    ("timestamp", ctypes.c_uint64),
-    ("id", ctypes.c_uint32),
-  ]
-
-class gpioevent_request(ctypes.Structure):
-  _fields_ = [
-    ("lineoffset", ctypes.c_uint32),
-    ("handleflags", ctypes.c_uint32),
-    ("eventflags", ctypes.c_uint32),
-    ("label", ctypes.c_char * 32),
-    ("fd", ctypes.c_int)
-  ]
-
-def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
-  GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
-  GPIOHANDLE_REQUEST_INPUT = 0x1
-  GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
-
-  rq = gpioevent_request()
-  rq.lineoffset = pin
-  rq.handleflags = GPIOHANDLE_REQUEST_INPUT
-  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
-  rq.label = label.encode('utf-8')[:31] + b'\0'
-
-  fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
-  fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
-  os.close(fd)
-  return int(rq.fd)

common/gps.py

@@ -1,8 +0,0 @@
-from openpilot.common.params import Params
-
-
-def get_gps_location_service(params: Params) -> str:
-  if params.get_bool("UbloxAvailable"):
-    return "gpsLocationExternal"
-  else:
-    return "gpsLocation"

common/i2c.py

@@ -1,81 +0,0 @@
-import os
-import fcntl
-import ctypes
-
-# I2C constants from /usr/include/linux/i2c-dev.h
-I2C_SLAVE = 0x0703
-I2C_SLAVE_FORCE = 0x0706
-I2C_SMBUS = 0x0720
-
-# SMBus transfer types
-I2C_SMBUS_READ = 1
-I2C_SMBUS_WRITE = 0
-I2C_SMBUS_BYTE_DATA = 2
-I2C_SMBUS_I2C_BLOCK_DATA = 8
-
-I2C_SMBUS_BLOCK_MAX = 32
-
-
-class _I2cSmbusData(ctypes.Union):
-  _fields_ = [
-    ("byte", ctypes.c_uint8),
-    ("word", ctypes.c_uint16),
-    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
-  ]
-
-
-class _I2cSmbusIoctlData(ctypes.Structure):
-  _fields_ = [
-    ("read_write", ctypes.c_uint8),
-    ("command", ctypes.c_uint8),
-    ("size", ctypes.c_uint32),
-    ("data", ctypes.POINTER(_I2cSmbusData)),
-  ]
-
-
-class SMBus:
-  def __init__(self, bus: int):
-    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
-
-  def __enter__(self) -> 'SMBus':
-    return self
-
-  def __exit__(self, *args) -> None:
-    self.close()
-
-  def close(self) -> None:
-    if hasattr(self, '_fd') and self._fd >= 0:
-      os.close(self._fd)
-      self._fd = -1
-
-  def _set_address(self, addr: int, force: bool = False) -> None:
-    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
-    fcntl.ioctl(self._fd, ioctl_arg, addr)
-
-  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
-    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
-    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
-
-  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
-    return int(data.byte)
-
-  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
-    self._set_address(addr, force)
-    data = _I2cSmbusData()
-    data.byte = value & 0xFF
-    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
-
-  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
-    self._set_address(addr, force)
-    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
-      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
-
-    data = _I2cSmbusData()
-    data.block[0] = length
-    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
-    read_len = int(data.block[0]) or length
-    read_len = min(read_len, length)
-    return [int(b) for b in data.block[1 : read_len + 1]]

common/logging_extra.py

@@ -8,7 +8,6 @@ import uuid
 import socket
 import logging
 import traceback
-import numpy as np
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
@@ -16,8 +15,6 @@ from contextlib import contextmanager
 LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
 
 def json_handler(obj):
-  if isinstance(obj, np.bool_):
-    return bool(obj)
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
   return repr(obj)
@@ -199,6 +196,7 @@ class SwagLogger(logging.Logger):
       co = f.f_code
       filename = os.path.normcase(co.co_filename)
 
+      # TODO: is this pylint exception correct?
       if filename == _srcfile:
         f = f.f_back
         continue

common/mock/__init__.py

@@ -8,12 +8,12 @@ import functools
 import threading
 from cereal.messaging import PubMaster
 from cereal.services import SERVICE_LIST
-from openpilot.common.mock.generators import generate_livePose
+from openpilot.common.mock.generators import generate_liveLocationKalman
 from openpilot.common.realtime import Ratekeeper
 
 
 MOCK_GENERATOR = {
-  "livePose": generate_livePose
+  "liveLocationKalman": generate_liveLocationKalman
 }
 
 

common/mock/generators.py

@@ -1,14 +1,20 @@
 from cereal import messaging
 
 
-def generate_livePose():
-  msg = messaging.new_message('livePose')
-  meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
-  msg.livePose.orientationNED = meas
-  msg.livePose.velocityDevice = meas
-  msg.livePose.angularVelocityDevice = meas
-  msg.livePose.accelerationDevice = meas
-  msg.livePose.inputsOK = True
-  msg.livePose.posenetOK = True
-  msg.livePose.sensorsOK = True
+LOCATION1 = (32.7174, -117.16277)
+LOCATION2 = (32.7558, -117.2037)
+
+LLK_DECIMATION = 10
+RENDER_FRAMES = 15
+DEFAULT_ITERATIONS = RENDER_FRAMES * LLK_DECIMATION
+
+
+def generate_liveLocationKalman(location=LOCATION1):
+  msg = messaging.new_message('liveLocationKalman')
+  msg.liveLocationKalman.positionGeodetic = {'value': [*location, 0], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.positionECEF = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.calibratedOrientationNED = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.velocityCalibrated = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.status = 'valid'
+  msg.liveLocationKalman.gpsOK = True
   return msg

common/numpy_fast.py

@@ -0,0 +1,19 @@
+def clip(x, lo, hi):
+  return max(lo, min(hi, x))
+
+def interp(x, xp, fp):
+  N = len(xp)
+
+  def get_interp(xv):
+    hi = 0
+    while hi < N and xv > xp[hi]:
+      hi += 1
+    low = hi - 1
+    return fp[-1] if hi == N and xv > xp[low] else (
+      fp[0] if hi == 0 else
+      (xv - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]) + fp[low])
+
+  return [get_interp(v) for v in x] if hasattr(x, '__iter__') else get_interp(x)
+
+def mean(x):
+  return sum(x) / len(x)

common/parameterized.py

@@ -1,47 +0,0 @@
-import sys
-import pytest
-import inspect
-
-
-class parameterized:
-  @staticmethod
-  def expand(cases):
-    cases = list(cases)
-
-    if not cases:
-      return lambda func: pytest.mark.skip("no parameterized cases")(func)
-
-    def decorator(func):
-      params = [p for p in inspect.signature(func).parameters if p != 'self']
-      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
-      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
-      expand_params = params[: len(normalized[0])]
-      if len(expand_params) == 1:
-        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
-      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
-
-    return decorator
-
-
-def parameterized_class(attrs, input_list=None):
-  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
-    params_list = attrs
-  else:
-    assert input_list is not None
-    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
-    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
-
-  def decorator(cls):
-    globs = sys._getframe(1).f_globals
-    for i, params in enumerate(params_list):
-      name = f"{cls.__name__}_{i}"
-      new_cls = type(name, (cls,), dict(params))
-      new_cls.__module__ = cls.__module__
-      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
-      globs[name] = new_cls
-    # Don't collect the un-parametrised base, but return it so outer decorators
-    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
-    cls.__test__ = False
-    return cls
-
-  return decorator

common/params.cc

@@ -1,242 +0,0 @@
-#include "common/params.h"
-
-#include <dirent.h>
-#include <sys/file.h>
-
-#include <algorithm>
-#include <cassert>
-#include <csignal>
-#include <unordered_map>
-
-#include "common/params_keys.h"
-#include "common/queue.h"
-#include "common/swaglog.h"
-#include "common/util.h"
-#include "system/hardware/hw.h"
-
-namespace {
-
-volatile sig_atomic_t params_do_exit = 0;
-void params_sig_handler(int signal) {
-  params_do_exit = 1;
-}
-
-int fsync_dir(const std::string &path) {
-  int result = -1;
-  int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
-  if (fd >= 0) {
-    result = HANDLE_EINTR(fsync(fd));
-    HANDLE_EINTR(close(fd));
-  }
-  return result;
-}
-
-bool create_params_path(const std::string &param_path, const std::string &key_path) {
-  // Make sure params path exists
-  if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
-    return false;
-  }
-
-  // See if the symlink exists, otherwise create it
-  if (!util::file_exists(key_path)) {
-    // 1) Create temp folder
-    // 2) Symlink it to temp link
-    // 3) Move symlink to <params>/d
-
-    std::string tmp_path = param_path + "/.tmp_XXXXXX";
-    // this should be OK since mkdtemp just replaces characters in place
-    char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
-    if (tmp_dir == NULL) {
-      return false;
-    }
-
-    std::string link_path = std::string(tmp_dir) + ".link";
-    if (symlink(tmp_dir, link_path.c_str()) != 0) {
-      return false;
-    }
-
-    // don't return false if it has been created by other
-    if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
-  std::string params_path = path.empty() ? Path::params() : path;
-  if (!create_params_path(params_path, params_path + prefix)) {
-    throw std::runtime_error(util::string_format(
-        "Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
-        errno, params_path.c_str(), prefix.c_str()));
-  }
-  return params_path;
-}
-
-class FileLock {
-public:
-  FileLock(const std::string &fn) {
-    fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
-    if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
-      LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
-    }
-  }
-  ~FileLock() { close(fd_); }
-
-private:
-  int fd_ = -1;
-};
-
-} // namespace
-
-
-Params::Params(const std::string &path) {
-  params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
-  params_path = ensure_params_path(params_prefix, path);
-}
-
-Params::~Params() {
-  if (future.valid()) {
-    future.wait();
-  }
-  assert(queue.empty());
-}
-
-std::vector<std::string> Params::allKeys() const {
-  std::vector<std::string> ret;
-  for (auto &p : keys) {
-    ret.push_back(p.first);
-  }
-  return ret;
-}
-
-bool Params::checkKey(const std::string &key) {
-  return keys.find(key) != keys.end();
-}
-
-ParamKeyFlag Params::getKeyFlag(const std::string &key) {
-  return static_cast<ParamKeyFlag>(keys[key].flags);
-}
-
-ParamKeyType Params::getKeyType(const std::string &key) {
-  return keys[key].type;
-}
-
-std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
-  return keys[key].default_value;
-}
-
-int Params::put(const char* key, const char* value, size_t value_size) {
-  // Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
-  // 1) Create temp file
-  // 2) Write data to temp file
-  // 3) fsync() the temp file
-  // 4) rename the temp file to the real name
-  // 5) fsync() the containing directory
-  std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
-  int tmp_fd = mkstemp((char*)tmp_path.c_str());
-  if (tmp_fd < 0) return -1;
-
-  int result = -1;
-  do {
-    // Write value to temp.
-    ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
-    if (bytes_written < 0 || (size_t)bytes_written != value_size) {
-      result = -20;
-      break;
-    }
-
-    // fsync to force persist the changes.
-    if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break;
-
-    FileLock file_lock(params_path + "/.lock");
-
-    // Move temp into place.
-    if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
-
-    // fsync parent directory
-    result = fsync_dir(getParamPath());
-  } while (false);
-
-  close(tmp_fd);
-  if (result != 0) {
-    ::unlink(tmp_path.c_str());
-  }
-  return result;
-}
-
-int Params::remove(const std::string &key) {
-  FileLock file_lock(params_path + "/.lock");
-  int result = unlink(getParamPath(key).c_str());
-  if (result != 0) {
-    return result;
-  }
-  return fsync_dir(getParamPath());
-}
-
-std::string Params::get(const std::string &key, bool block) {
-  if (!block) {
-    return util::read_file(getParamPath(key));
-  } else {
-    // blocking read until successful
-    params_do_exit = 0;
-    void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
-    void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
-
-    std::string value;
-    while (!params_do_exit) {
-      if (value = util::read_file(getParamPath(key)); !value.empty()) {
-        break;
-      }
-      util::sleep_for(100);  // 0.1 s
-    }
-
-    std::signal(SIGINT, prev_handler_sigint);
-    std::signal(SIGTERM, prev_handler_sigterm);
-    return value;
-  }
-}
-
-std::map<std::string, std::string> Params::readAll() {
-  FileLock file_lock(params_path + "/.lock");
-  return util::read_files_in_dir(getParamPath());
-}
-
-void Params::clearAll(ParamKeyFlag key_flag) {
-  FileLock file_lock(params_path + "/.lock");
-
-  // 1) delete params of key_flag
-  // 2) delete files that are not defined in the keys.
-  if (DIR *d = opendir(getParamPath().c_str())) {
-    struct dirent *de = NULL;
-    while ((de = readdir(d))) {
-      if (de->d_type != DT_DIR) {
-        auto it = keys.find(de->d_name);
-        if (it == keys.end() || (it->second.flags & key_flag)) {
-          unlink(getParamPath(de->d_name).c_str());
-        }
-      }
-    }
-    closedir(d);
-  }
-
-  fsync_dir(getParamPath());
-}
-
-void Params::putNonBlocking(const std::string &key, const std::string &val) {
-   queue.push(std::make_pair(key, val));
-  // start thread on demand
-  if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
-    future = std::async(std::launch::async, &Params::asyncWriteThread, this);
-  }
-}
-
-void Params::asyncWriteThread() {
-  // TODO: write the latest one if a key has multiple values in the queue.
-  std::pair<std::string, std::string> p;
-  while (queue.try_pop(p, 0)) {
-    // Params::put is Thread-Safe
-    put(p.first, p.second);
-  }
-}

common/params.h

@@ -1,90 +0,0 @@
-#pragma once
-
-#include <future>
-#include <map>
-#include <optional>
-#include <string>
-#include <tuple>
-#include <utility>
-#include <vector>
-
-#include "common/queue.h"
-
-enum ParamKeyFlag {
-  PERSISTENT = 0x02,
-  CLEAR_ON_MANAGER_START = 0x04,
-  CLEAR_ON_ONROAD_TRANSITION = 0x08,
-  CLEAR_ON_OFFROAD_TRANSITION = 0x10,
-  DONT_LOG = 0x20,
-  DEVELOPMENT_ONLY = 0x40,
-  CLEAR_ON_IGNITION_ON = 0x80,
-  ALL = 0xFFFFFFFF
-};
-
-enum ParamKeyType {
-  STRING = 0, // must be utf-8 decodable
-  BOOL = 1,
-  INT = 2,
-  FLOAT = 3,
-  TIME = 4, // ISO 8601
-  JSON = 5,
-  BYTES = 6
-};
-
-struct ParamKeyAttributes {
-  uint32_t flags;
-  ParamKeyType type;
-  std::optional<std::string> default_value = std::nullopt;
-};
-
-class Params {
-public:
-  explicit Params(const std::string &path = {});
-  ~Params();
-  // Not copyable.
-  Params(const Params&) = delete;
-  Params& operator=(const Params&) = delete;
-
-  std::vector<std::string> allKeys() const;
-  bool checkKey(const std::string &key);
-  ParamKeyFlag getKeyFlag(const std::string &key);
-  ParamKeyType getKeyType(const std::string &key);
-  std::optional<std::string> getKeyDefaultValue(const std::string &key);
-  inline std::string getParamPath(const std::string &key = {}) {
-    return params_path + params_prefix + (key.empty() ? "" : "/" + key);
-  }
-
-  // Delete a value
-  int remove(const std::string &key);
-  void clearAll(ParamKeyFlag flag);
-
-  // helpers for reading values
-  std::string get(const std::string &key, bool block = false);
-  inline bool getBool(const std::string &key, bool block = false) {
-    return get(key, block) == "1";
-  }
-  std::map<std::string, std::string> readAll();
-
-  // helpers for writing values
-  int put(const char *key, const char *val, size_t value_size);
-  inline int put(const std::string &key, const std::string &val) {
-    return put(key.c_str(), val.data(), val.size());
-  }
-  inline int putBool(const std::string &key, bool val) {
-    return put(key.c_str(), val ? "1" : "0", 1);
-  }
-  void putNonBlocking(const std::string &key, const std::string &val);
-  inline void putBoolNonBlocking(const std::string &key, bool val) {
-    putNonBlocking(key, val ? "1" : "0");
-  }
-
-private:
-  void asyncWriteThread();
-
-  std::string params_path;
-  std::string params_prefix;
-
-  // for nonblocking write
-  std::future<void> future;
-  SafeQueue<std::pair<std::string, std::string>> queue;
-};

common/params.py

@@ -1,6 +1,5 @@
-from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
 assert Params
-assert ParamKeyFlag
 assert ParamKeyType
 assert UnknownKeyName
 

common/params_keys.h

@@ -1,133 +0,0 @@
-#pragma once
-
-#include <string>
-#include <unordered_map>
-
-#include "cereal/gen/cpp/log.capnp.h"
-
-inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
-    {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
-    {"AdbEnabled", {PERSISTENT, BOOL}},
-    {"AlwaysOnDM", {PERSISTENT, BOOL}},
-    {"ApiCache_Device", {PERSISTENT, STRING}},
-    {"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
-    {"AssistNowToken", {PERSISTENT, STRING}},
-    {"AthenadPid", {PERSISTENT, INT}},
-    {"AthenadUploadQueue", {PERSISTENT, JSON}},
-    {"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
-    {"BootCount", {PERSISTENT, INT}},
-    {"CalibrationParams", {PERSISTENT, BYTES}},
-    {"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
-    {"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
-    {"CarBatteryCapacity", {PERSISTENT, INT}},
-    {"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
-    {"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"CarParamsPersistent", {PERSISTENT, BYTES}},
-    {"CarParamsPrevRoute", {PERSISTENT, BYTES}},
-    {"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
-    {"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"CurrentBootlog", {PERSISTENT, STRING}},
-    {"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
-    {"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"DisablePowerDown", {PERSISTENT, BOOL}},
-    {"DisableUpdates", {PERSISTENT, BOOL}},
-    {"DisengageOnAccelerator", {PERSISTENT, BOOL, "0"}},
-    {"DongleId", {PERSISTENT, STRING}},
-    {"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
-    {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY, BOOL}},
-    {"ExperimentalMode", {PERSISTENT, BOOL}},
-    {"ExperimentalModeConfirmed", {PERSISTENT, BOOL}},
-    {"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"ForcePowerDown", {PERSISTENT, BOOL}},
-    {"GitBranch", {PERSISTENT, STRING}},
-    {"GitCommit", {PERSISTENT, STRING}},
-    {"GitCommitDate", {PERSISTENT, STRING}},
-    {"GitDiff", {PERSISTENT, STRING}},
-    {"GithubSshKeys", {PERSISTENT, STRING}},
-    {"GithubUsername", {PERSISTENT, STRING}},
-    {"GitRemote", {PERSISTENT, STRING}},
-    {"GsmApn", {PERSISTENT, STRING}},
-    {"GsmMetered", {PERSISTENT, BOOL, "1"}},
-    {"GsmRoaming", {PERSISTENT, BOOL}},
-    {"HardwareSerial", {PERSISTENT, STRING}},
-    {"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
-    {"InstallDate", {PERSISTENT, TIME}},
-    {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsEngaged", {PERSISTENT, BOOL}},
-    {"IsLdwEnabled", {PERSISTENT, BOOL}},
-    {"IsMetric", {PERSISTENT, BOOL}},
-    {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsOnroad", {PERSISTENT, BOOL}},
-    {"IsRhdDetected", {PERSISTENT, BOOL}},
-    {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LanguageSetting", {PERSISTENT, STRING, "en"}},
-    {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
-    {"LastGPSPosition", {PERSISTENT, STRING}},
-    {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
-    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
-    {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
-    {"LastUpdateTime", {PERSISTENT, TIME}},
-    {"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"LiveDelay", {PERSISTENT, BYTES}},
-    {"LiveParameters", {PERSISTENT, JSON}},
-    {"LiveParametersV2", {PERSISTENT, BYTES}},
-    {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
-    {"LocationFilterInitialState", {PERSISTENT, BYTES}},
-    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"LongitudinalPersonality", {PERSISTENT, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
-    {"NetworkMetered", {PERSISTENT, BOOL}},
-    {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
-    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
-    {"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
-    {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"OpenpilotEnabledToggle", {PERSISTENT, BOOL, "1"}},
-    {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"PrimeType", {PERSISTENT, INT}},
-    {"RecordAudio", {PERSISTENT, BOOL}},
-    {"RecordAudioFeedback", {PERSISTENT, BOOL, "0"}},
-    {"RecordFront", {PERSISTENT, BOOL}},
-    {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
-    {"SecOCKey", {PERSISTENT | DONT_LOG, STRING}},
-    {"ShowDebugInfo", {PERSISTENT, BOOL}},
-    {"RouteCount", {PERSISTENT, INT, "0"}},
-    {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"SshEnabled", {PERSISTENT, BOOL}},
-    {"UbloxAvailable", {PERSISTENT, BOOL}},
-    {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
-    {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
-    {"UpdaterAvailableBranches", {PERSISTENT, STRING}},
-    {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
-    {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
-    {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
-    {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
-    {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
-    {"Version", {PERSISTENT, STRING}},
-};

common/params_pyx.pyx

@@ -1,69 +1,45 @@
 # distutils: language = c++
 # cython: language_level = 3
-import builtins
-import datetime
-import json
 from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
-from libcpp.optional cimport optional
-
-from openpilot.common.swaglog import cloudlog
 
 cdef extern from "common/params.h":
-  cpdef enum ParamKeyFlag:
+  cpdef enum ParamKeyType:
     PERSISTENT
     CLEAR_ON_MANAGER_START
     CLEAR_ON_ONROAD_TRANSITION
     CLEAR_ON_OFFROAD_TRANSITION
     DEVELOPMENT_ONLY
-    CLEAR_ON_IGNITION_ON
+    FROGPILOT_CONTROLS
+    FROGPILOT_VEHICLES
+    FROGPILOT_VISUALS
+    FROGPILOT_OTHER
+    FROGPILOT_STORAGE
+    FROGPILOT_TRACKING
     ALL
 
-  cpdef enum ParamKeyType:
-    STRING
-    BOOL
-    INT
-    FLOAT
-    TIME
-    JSON
-    BYTES
-
   cdef cppclass c_Params "Params":
     c_Params(string) except + nogil
     string get(string, bool) nogil
     bool getBool(string, bool) nogil
+    int getInt(string, bool) nogil
+    float getFloat(string, bool) nogil
     int remove(string) nogil
     int put(string, string) nogil
     void putNonBlocking(string, string) nogil
     void putBoolNonBlocking(string, bool) nogil
+    void putIntNonBlocking(string, int) nogil
+    void putFloatNonBlocking(string, float) nogil
     int putBool(string, bool) nogil
+    int putInt(string, int) nogil
+    int putFloat(string, float) nogil
     bool checkKey(string) nogil
-    ParamKeyType getKeyType(string) nogil
-    optional[string] getKeyDefaultValue(string) nogil
     string getParamPath(string) nogil
-    void clearAll(ParamKeyFlag)
+    void clearAll(ParamKeyType)
     vector[string] allKeys()
+    ParamKeyType getKeyType(string) nogil
 
-PYTHON_2_CPP = {
-  (str, STRING): lambda v: v,
-  (builtins.bool, BOOL): lambda v: "1" if v else "0",
-  (int, INT): str,
-  (float, FLOAT): str,
-  (datetime.datetime, TIME): lambda v: v.isoformat(),
-  (dict, JSON): json.dumps,
-  (list, JSON): json.dumps,
-  (bytes, BYTES): lambda v: v,
-}
-CPP_2_PYTHON = {
-  STRING: lambda v: v.decode("utf-8"),
-  BOOL: lambda v: v == b"1",
-  INT: int,
-  FLOAT: float,
-  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
-  JSON: json.loads,
-  BYTES: lambda v: v,
-}
 
 def ensure_bytes(v):
   return v.encode() if isinstance(v, str) else v
@@ -73,22 +49,17 @@ class UnknownKeyName(Exception):
 
 cdef class Params:
   cdef c_Params* p
-  cdef str d
 
   def __cinit__(self, d=""):
     cdef string path = <string>d.encode()
     with nogil:
       self.p = new c_Params(path)
-    self.d = d
-
-  def __reduce__(self):
-    return (type(self), (self.d,))
 
   def __dealloc__(self):
     del self.p
 
-  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
-    self.p.clearAll(tx_flag)
+  def clear_all(self, tx_type=ParamKeyType.ALL):
+    self.p.clearAll(tx_type)
 
   def check_key(self, key):
     key = ensure_bytes(key)
@@ -96,38 +67,21 @@ cdef class Params:
       raise UnknownKeyName(key)
     return key
 
-  def python2cpp(self, proposed_type, expected_type, value, key):
-    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
-    if cast:
-      return cast(value)
-    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
-
-  def _cpp2python(self, t, value, default, key):
-    if value is None:
-      return None
-    try:
-      return CPP_2_PYTHON[t](value)
-    except (KeyError, TypeError, ValueError):
-      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
-      return self._cpp2python(t, default, None, key)
-
-  def get(self, key, bool block=False, bool return_default=False):
+  def get(self, key, bool block=False, encoding=None):
     cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    cdef optional[string] default = self.p.getKeyDefaultValue(k)
     cdef string val
     with nogil:
       val = self.p.get(k, block)
 
-    default_val = (default.value() if default.has_value() else None) if return_default else None
     if val == b"":
       if block:
         # If we got no value while running in blocked mode
         # it means we got an interrupt while waiting
         raise KeyboardInterrupt
       else:
-        return self._cpp2python(t, default_val, None, key)
-    return self._cpp2python(t, val, default_val, key)
+        return None
+
+    return val if encoding is None else val.decode(encoding)
 
   def get_bool(self, key, bool block=False):
     cdef string k = self.check_key(key)
@@ -136,10 +90,19 @@ cdef class Params:
       r = self.p.getBool(k, block)
     return r
 
-  def _put_cast(self, key, dat):
+  def get_int(self, key, bool block=False):
     cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
+    cdef int r
+    with nogil:
+      r = self.p.getInt(k, block)
+    return r
+
+  def get_float(self, key, bool block=False):
+    cdef string k = self.check_key(key)
+    cdef float r
+    with nogil:
+      r = self.p.getFloat(k, block)
+    return r
 
   def put(self, key, dat):
     """
@@ -149,7 +112,7 @@ cdef class Params:
     in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
+    cdef string dat_bytes = ensure_bytes(dat)
     with nogil:
       self.p.put(k, dat_bytes)
 
@@ -158,9 +121,19 @@ cdef class Params:
     with nogil:
       self.p.putBool(k, val)
 
+  def put_int(self, key, int val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putInt(k, val)
+
+  def put_float(self, key, float val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putFloat(k, val)
+
   def put_nonblocking(self, key, dat):
     cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
+    cdef string dat_bytes = ensure_bytes(dat)
     with nogil:
       self.p.putNonBlocking(k, dat_bytes)
 
@@ -169,6 +142,16 @@ cdef class Params:
     with nogil:
       self.p.putBoolNonBlocking(k, val)
 
+  def put_int_nonblocking(self, key, int val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putIntNonBlocking(k, val)
+
+  def put_float_nonblocking(self, key, float val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putFloatNonBlocking(k, val)
+
   def remove(self, key):
     cdef string k = self.check_key(key)
     with nogil:
@@ -178,19 +161,9 @@ cdef class Params:
     cdef string key_bytes = ensure_bytes(key)
     return self.p.getParamPath(key_bytes).decode("utf-8")
 
-  def get_type(self, key):
-    return self.p.getKeyType(self.check_key(key))
-
   def all_keys(self):
     return self.p.allKeys()
 
-  def get_default_value(self, key):
+  def get_key_type(self, key):
     cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    cdef optional[string] default = self.p.getKeyDefaultValue(k)
-    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
-
-  def cpp2python(self, key, value):
-    cdef string k = self.check_key(key)
-    cdef ParamKeyType t = self.p.getKeyType(k)
-    return self._cpp2python(t, value, None, key)
+    return self.p.getKeyType(k)