openpilot v0.8.13 release

This reverts commit f482354e4e2ce5d96f5b0c96f7ba923611dc199d.

.github/workflows/compile_frogpilot.yaml

@@ -1,277 +0,0 @@
-name: Compile FrogPilot
-
-on:
-  workflow_dispatch:
-    inputs:
-      runner:
-        description: "Select the runner"
-        required: true
-        default: "c3"
-        type: choice
-        options:
-          - c3
-          - c3x
-      not_vetted:
-        description: "This branch is not vetted"
-        required: false
-        default: "false"
-        type: boolean
-      publish_frogpilot:
-        description: "Push to FrogPilot"
-        required: false
-        default: "false"
-        type: boolean
-      publish_staging:
-        description: "Push to FrogPilot-Staging"
-        required: false
-        default: "false"
-        type: boolean
-      publish_testing:
-        description: "Push to FrogPilot-Testing"
-        required: false
-        default: "false"
-        type: boolean
-      publish_custom_branch:
-        description: "Push to custom branch:"
-        required: false
-        default: ""
-        type: string
-      update_translations:
-        description: "Update missing/outdated translations"
-        required: false
-        default: "false"
-        type: boolean
-      vet_existing_translations:
-        description: "Vet existing translations"
-        required: false
-        default: "false"
-        type: boolean
-
-env:
-  BASEDIR: "${{ github.workspace }}"
-  BUILD_DIR: /data/openpilot
-  OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
-
-jobs:
-  get_branch:
-    runs-on:
-      - self-hosted
-      - ${{ github.event.inputs.runner }}
-    outputs:
-      branch: ${{ steps.get_branch.outputs.branch }}
-      python_version: ${{ steps.get_python_version.outputs.python_version }}
-    steps:
-      - name: Determine Current Branch on Runner
-        id: get_branch
-        run: |
-          cd $BUILD_DIR
-
-          BRANCH=$(git rev-parse --abbrev-ref HEAD)
-          echo "branch=$BRANCH" >> $GITHUB_OUTPUT
-
-      - name: Get Python Version from Runner
-        id: get_python_version
-        run: |
-          PYTHON_VERSION=$(tr -d '[:space:]' < "$BUILD_DIR/.python-version")
-          echo "python_version=$PYTHON_VERSION" >> $GITHUB_OUTPUT
-
-  translate:
-    if: inputs.update_translations
-    needs: get_branch
-    runs-on: ubuntu-latest
-    steps:
-      - name: Configure Git Identity
-        run: |
-          git config --global user.name "James"
-          git config --global user.email "91348155+FrogAi@users.noreply.github.com"
-
-      - name: Checkout Required Files
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.get_branch.outputs.branch }}
-          sparse-checkout: |
-            frogpilot/ui/
-            selfdrive/controls/lib/alerts_offroad.json
-            selfdrive/ui/
-            selfdrive/ui/update_translations.py
-            selfdrive/ui/translations/
-            selfdrive/ui/translations/auto_translate.py
-
-      - name: Set Up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "${{ needs.get_branch.outputs.python_version }}"
-
-      - name: Install Dependencies
-        run: pip install requests
-
-      - name: Install Qt5 Tools
-        run: sudo apt update && sudo apt install -y qttools5-dev-tools
-
-      - name: Update Translations
-        run: python selfdrive/ui/update_translations.py --vanish
-
-      - name: Update Missing Translations
-        continue-on-error: true
-        run: python selfdrive/ui/translations/auto_translate.py --all-files
-        timeout-minutes: 300
-
-      - name: Vet Existing Translations
-        if: github.event.inputs.vet_existing_translations == 'true'
-        continue-on-error: true
-        run: python selfdrive/ui/translations/auto_translate.py --all-files --vet-translations
-        timeout-minutes: 300
-
-      - name: Commit and Push Translation Updates
-        run: |
-          if git diff --quiet selfdrive/ui/translations/*.ts; then
-            echo "No translation updates detected."
-            exit 0
-          fi
-
-          git fetch --unshallow origin ${{ needs.get_branch.outputs.branch }}
-          git checkout ${{ needs.get_branch.outputs.branch }}
-          git add selfdrive/ui/translations/*.ts
-          git commit --amend --no-edit
-          git push --force origin ${{ needs.get_branch.outputs.branch }}
-
-  build_and_push:
-    needs:
-      - get_branch
-      - translate
-    if: always()
-    runs-on:
-      - self-hosted
-      - ${{ github.event.inputs.runner }}
-    permissions:
-      contents: write
-    defaults:
-      run:
-        working-directory: ${{ env.BUILD_DIR }}
-    steps:
-      - name: Configure Git Identity
-        run: |
-          git config --global http.postBuffer 104857600
-
-          git config --global user.name "James"
-          git config --global user.email "91348155+FrogAi@users.noreply.github.com"
-
-      - name: Update Repository
-        run: |
-          git remote set-url origin https://${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com/FrogAi/FrogPilot.git
-
-          if [ "${{ github.event.inputs.update_translations }}" = "true" ]; then
-            git fetch origin ${{ needs.get_branch.outputs.branch }}
-            git reset --hard FETCH_HEAD
-          fi
-
-      - name: Take Ownership of Build
-        run: |
-          sudo chown -R $(whoami):$(whoami) .
-
-      - name: Finalize Build
-        run: |
-          rm -f .clang-tidy
-          rm -f .dockerignore
-          rm -f .editorconfig
-          rm -f .gitattributes
-          rm -f .gitmodules
-          rm -f .lfsconfig
-          rm -f .overlay_init
-          rm -f .pre-commit-config.yaml
-          rm -f .sconsign.dblite
-          rm -f codecov.yml
-          rm -f conftest.py
-          rm -f poetry.lock
-          rm -f pyproject.toml
-          rm -f teleoprtc
-          rm -f Dockerfile.openpilot
-          rm -f Dockerfile.openpilot_base
-          rm -f Jenkinsfile
-
-          rm -f panda/board/obj/.placeholder
-          rm -f panda/board/obj/bootstub.panda.elf
-          rm -f panda/board/obj/bootstub.panda_h7.elf
-          rm -f panda/board/obj/panda.bin
-          rm -f panda/board/obj/panda.elf
-          rm -f panda/board/obj/panda_h7.bin
-          rm -f panda/board/obj/panda_h7.elf
-          rm -f panda/board/obj/version
-
-          find . -name '*.a' -delete
-          find . -name '*.o' -delete
-          find . -name '*.onnx' -delete
-          find . -name '*.os' -delete
-          find . -name '*.pyc' -delete
-          find . -name 'moc_*' -delete
-
-          rm -rf .devcontainer/
-          rm -rf .vscode/
-          rm -rf body/
-          rm -rf opendbc/generator/
-          rm -rf release/
-          rm -rf scripts/
-          rm -rf site_scons/
-          rm -rf teleoprtc_repo/
-
-          find .github -mindepth 1 -maxdepth 1 ! -name 'workflows' -exec rm -rf {} +
-
-          find .github/workflows -mindepth 1 ! \( \
-            -type f \( \
-              -name 'compile_frogpilot.yaml' -o \
-              -name 'review_pull_request.yaml' -o \
-              -name 'schedule_update.yaml' -o \
-              -name 'update_pr_branch.yaml' -o \
-              -name 'update_release_branch.yaml' -o \
-              -name 'update_tinygrad.yaml' \
-            \) \
-          \) -exec rm -rf {} +
-
-          find panda/board/jungle -type f ! -name '__init__.py' -delete
-          find panda/board/jungle -type d -empty -delete
-
-          find third_party/ -name '*x86*' -exec rm -rf {} +
-          find third_party/ -name '*Darwin*' -exec rm -rf {} +
-
-          find tools/ -mindepth 1 -maxdepth 1 ! \( -name '__init__.py' -o -name 'bodyteleop' -o -name 'lib' -o -name 'scripts' \) -exec rm -rf {} +
-
-          find . -name 'SConstruct' -delete
-          find . -name 'SConscript' -delete
-
-          find . -type d \( -iname "debug" -o -iname "test" -o -iname "tests" \) -exec rm -rf {} +
-          find . -type d -empty ! -path "./.git*" -delete
-          find . -type d -name '__pycache__' -exec rm -rf {} +
-          find . -type f -regex '.*matlab.*\.md' -delete
-
-          touch prebuilt
-
-          if [ "${{ github.event.inputs.not_vetted }}" = "true" ]; then
-            touch not_vetted
-          fi
-
-      - name: Add the update_date file
-        if: github.event.inputs.publish_staging == 'true'
-        run: |
-          curl -fLsS https://raw.githubusercontent.com/FrogAi/FrogPilot/FrogPilot-Staging/.github/update_date -o .github/update_date || echo "No update_date found, skipping."
-
-      - name: Commit Build
-        run: |
-          git add -f .
-          git commit -m "Compile FrogPilot"
-          git push --force origin HEAD
-
-          if [ "${{ github.event.inputs.publish_frogpilot }}" = "true" ]; then
-            git push --force origin HEAD:"FrogPilot"
-          fi
-
-          if [ "${{ github.event.inputs.publish_staging }}" = "true" ]; then
-            git push --force origin HEAD:"FrogPilot-Staging"
-          fi
-
-          if [ "${{ github.event.inputs.publish_testing }}" = "true" ]; then
-            git push --force origin HEAD:"FrogPilot-Testing"
-          fi
-
-          if [ -n "${{ github.event.inputs.publish_custom_branch }}" ]; then
-            git push --force origin HEAD:"${{ github.event.inputs.publish_custom_branch }}"
-          fi

.github/workflows/review_pull_request.yaml

@@ -1,34 +0,0 @@
-name: Review Pull Request
-
-on:
-  pull_request_target:
-    types: [opened, reopened]
-
-env:
-  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-jobs:
-  pr_check:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Exit If PR Opened by FrogAi
-        if: ${{ github.actor == 'FrogAi' }}
-        run: |
-          echo PR opened or reopened by FrogAi. No action needed.
-          exit 0
-
-      - name: Close PR for Invalid Target Branch
-        if: ${{ github.base_ref != 'MAKE-PRS-HERE' }}
-        run: |
-          gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
-            -f body="Please submit your pull request to the \"MAKE-PRS-HERE\" branch."
-
-          gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }} \
-            -X PATCH \
-            -f state="closed"
-
-      - name: Acknowledge PR for Valid Target Branch
-        if: ${{ github.base_ref == 'MAKE-PRS-HERE' }}
-        run: |
-          gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
-            -f body="Thank you for your PR! If you're not already in the FrogPilot Discord, [feel free to join](https://discord.FrogPilot.download) and let me know you've opened a PR!"

.github/workflows/schedule_update.yaml

@@ -1,57 +0,0 @@
-name: Schedule FrogPilot Update
-
-on:
-  workflow_dispatch:
-    inputs:
-      scheduled_date:
-        description: "Enter the date to update the \"FrogPilot\" branch (YYYY-MM-DD)"
-        required: true
-
-env:
-  BRANCH: FrogPilot-Staging
-
-jobs:
-  schedule-update:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Configure Git Identity
-        run: |
-          git config --global user.name "James"
-          git config --global user.email "91348155+FrogAi@users.noreply.github.com"
-
-      - name: Checkout ${{ env.BRANCH }}
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ env.BRANCH }}
-          fetch-depth: 3
-
-      - name: Schedule Update for ${{ github.event.inputs.scheduled_date }}
-        run: |
-          echo "${{ github.event.inputs.scheduled_date }}" > .github/update_date
-          git add .github/update_date
-
-      - name: Get Target Commit (Second Most Recent)
-        id: get_target
-        run: |
-          TARGET_COMMIT=$(git rev-parse HEAD~1)
-
-          AUTHOR_DATE=$(git show -s --format=%aD "$TARGET_COMMIT")
-          COMMITTER_DATE=$(git show -s --format=%cD "$TARGET_COMMIT")
-
-          echo "AUTHOR_DATE=$AUTHOR_DATE" >> $GITHUB_ENV
-          echo "COMMITTER_DATE=$COMMITTER_DATE" >> $GITHUB_ENV
-          echo "TARGET_COMMIT=$TARGET_COMMIT" >> $GITHUB_ENV
-
-      - name: Create Fixup Commit for ${{ env.TARGET_COMMIT }}
-        run: git commit --fixup="${{ env.TARGET_COMMIT }}"
-
-      - name: Autosquash Fixup into Target Commit
-        run: |
-          GIT_SEQUENCE_EDITOR=: git rebase --autosquash -i HEAD~3
-
-      - name: Restore Timestamps on Final Two Commits
-        run: |
-          git rebase --exec "GIT_COMMITTER_DATE='${{ env.COMMITTER_DATE }}' git commit --amend --no-edit --date='${{ env.AUTHOR_DATE }}'" HEAD~2
-
-      - name: Push Updated ${{ env.BRANCH }} Branch
-        run: git push origin "${{ env.BRANCH }}" --force-with-lease

.github/workflows/update_pr_branch.yaml

@@ -1,85 +0,0 @@
-name: Update MAKE-PRS-HERE
-
-on:
-  push:
-    branches:
-      - FrogPilot-Staging
-
-env:
-  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-  SOURCE_BRANCH: FrogPilot-Staging
-  TARGET_BRANCH: MAKE-PRS-HERE
-
-jobs:
-  update_branch:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Configure Git Identity
-        run: |
-          git config --global user.name "James"
-          git config --global user.email "91348155+FrogAi@users.noreply.github.com"
-
-      - name: Checkout ${{ env.SOURCE_BRANCH }}
-        uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-          fetch-depth: 0
-
-          ref: ${{ env.SOURCE_BRANCH }}
-
-      - name: Find "Compile FrogPilot" Commit in ${{ env.SOURCE_BRANCH }}
-        id: find_parent
-        run: |
-          COMMIT=$(git rev-list HEAD -n 1 --grep="Compile FrogPilot")
-          [ -z "$COMMIT" ] && echo "Compile commit not found." >&2 && exit 1
-
-          PARENT=$(git rev-list --parents -n 1 "$COMMIT" | awk '{print $2}')
-          [ -z "$PARENT" ] && echo "Parent commit not found." >&2 && exit 1
-
-          echo "parent_commit=$PARENT" >> "$GITHUB_OUTPUT"
-          echo "compile_commit=$COMMIT" >> "$GITHUB_OUTPUT"
-
-      - name: Fetch and Checkout ${{ env.TARGET_BRANCH }}
-        run: |
-          git fetch origin "${{ env.TARGET_BRANCH }}"
-          git checkout "${{ env.TARGET_BRANCH }}"
-
-      - name: Clean ${{ env.TARGET_BRANCH }} and Apply Updates from ${{ env.SOURCE_BRANCH }}
-        run: |
-          git rm -r --ignore-unmatch .
-          git clean -fdx
-          git checkout "${{ steps.find_parent.outputs.parent_commit }}" -- .
-
-          COMPILE_COMMIT="${{ steps.find_parent.outputs.compile_commit }}"
-          SOURCE_HEAD=$(git rev-parse "origin/${{ env.SOURCE_BRANCH }}")
-
-          [ "$COMPILE_COMMIT" != "$SOURCE_HEAD" ] && git cherry-pick --no-commit "${COMPILE_COMMIT}".."$SOURCE_HEAD"
-
-          rm -f .github/update_date
-
-          git add --all
-
-      - name: Commit and Push to ${{ env.TARGET_BRANCH }}
-        run: |
-          TZ_VALUE="America/Phoenix"
-          day=$(TZ="$TZ_VALUE" date +"%-d")
-          month=$(TZ="$TZ_VALUE" date +"%B")
-          year=$(TZ="$TZ_VALUE" date +"%Y")
-
-          case $day in
-            11|12|13) s=th ;;
-            *) case $((day % 10)) in 1) s=st ;; 2) s=nd ;; 3) s=rd ;; *) s=th ;; esac ;;
-          esac
-
-          commit_message="${month} ${day}${s}, ${year} Update"
-
-          if git log -1 --pretty=%s | grep -q "$commit_message"; then
-            git commit --amend --no-edit
-          else
-            git commit -m "$commit_message"
-          fi
-
-          git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}
-          git push origin "${{ env.TARGET_BRANCH }}" --force

.github/workflows/update_release_branch.yaml

@@ -1,131 +0,0 @@
-name: Update FrogPilot Branch
-
-on:
-  schedule:
-    - cron: "0 18 * * 6"
-
-env:
-  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-  BRANCH_FROGPILOT: FrogPilot
-  BRANCH_PREVIOUS: FrogPilot-Previous
-  BRANCH_STAGING: FrogPilot-Staging
-
-  TZ: America/Phoenix
-
-  UPDATE_FILE: .github/update_date
-
-jobs:
-  check_update:
-    runs-on: ubuntu-latest
-    outputs:
-      update_due: ${{ steps.check_update.outputs.update_due }}
-      scheduled_date: ${{ steps.check_update.outputs.scheduled_date }}
-    steps:
-      - name: Download the "update_date" File
-        id: download_update
-        run: |
-          curl -fLsS "https://raw.githubusercontent.com/FrogAi/FrogPilot/${{ env.BRANCH_STAGING }}/${{ env.UPDATE_FILE }}" -o update_date || touch update_date_missing
-
-      - name: Check If Update Is Due
-        id: check_update
-        run: |
-          if [ -f update_date_missing ]; then
-            echo "update_due=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          SCHEDULED_DATE=$(cat update_date)
-          CURRENT_DATE=$(TZ="${{ env.TZ }}" date +%F)
-
-          if [ "$SCHEDULED_DATE" != "$CURRENT_DATE" ]; then
-            echo "update_due=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          echo "update_due=true" >> "$GITHUB_OUTPUT"
-          echo "scheduled_date=$SCHEDULED_DATE" >> "$GITHUB_OUTPUT"
-
-  update_branch:
-    needs: check_update
-    if: ${{ needs.check_update.outputs.update_due == 'true' }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Configure Git Identity
-        run: |
-          git config --global user.name "James"
-          git config --global user.email "91348155+FrogAi@users.noreply.github.com"
-
-      - name: Checkout ${{ env.BRANCH_STAGING }}
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: ${{ env.BRANCH_STAGING }}
-
-      - name: Authenticate with GITHUB_TOKEN
-        run: |
-          git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}
-
-      - name: Update README Date and Remove update file
-        run: |
-          DAY=$(TZ="${{ env.TZ }}" date +'%d' | sed 's/^0//')
-
-          case "$DAY" in
-            1|21|31) SUFFIX="st" ;;
-            2|22)    SUFFIX="nd" ;;
-            3|23)    SUFFIX="rd" ;;
-            *)       SUFFIX="th" ;;
-          esac
-
-          MONTH=$(TZ="${{ env.TZ }}" date +'%B')
-          YEAR=$(TZ="${{ env.TZ }}" date +'%Y')
-
-          DATE="${MONTH} ${DAY}${SUFFIX}, ${YEAR}"
-          DATE_ESCAPED=$(printf '%s' "$DATE" | sed -E 's/ /%20/g; s/,/%2C/g')
-
-          sed -i -E "s|(Last%20Updated-)[^-)]*|\1${DATE_ESCAPED}|g" README.md
-
-          git add README.md
-          git rm -f "${{ env.UPDATE_FILE }}"
-          git commit -m "Updated README date to ${DATE}"
-
-      - name: Squash Commits
-        run: |
-          COMMIT_MSG=$(git log -1 --pretty=%B HEAD~1)
-
-          git reset --soft HEAD~2
-          git commit -m "$COMMIT_MSG"
-
-      - name: Rewrite Commit Dates to Noon ${{ env.TZ }}
-        run: |
-          COMMIT_DATETIME="${{ needs.check_update.outputs.scheduled_date }} 12:00"
-          COMMIT_PHX=$(TZ="${{ env.TZ }}" date -d "$COMMIT_DATETIME" +"%Y-%m-%dT%H:%M:%S %z")
-
-          git filter-branch --env-filter "export GIT_AUTHOR_DATE='$COMMIT_PHX'; export GIT_COMMITTER_DATE='$COMMIT_PHX'" "${{ env.BRANCH_STAGING }}"
-
-      - name: Fetch ${{ env.BRANCH_PREVIOUS }} and ${{ env.BRANCH_FROGPILOT }}
-        run: |
-          git fetch origin ${{ env.BRANCH_PREVIOUS }} ${{ env.BRANCH_FROGPILOT }}
-
-      - name: Wait Until Noon ${{ env.TZ }}
-        run: |
-          NOW=$(TZ="${{ env.TZ }}" date +%s)
-          TARGET=$(TZ="${{ env.TZ }}" date -d "12:00" +%s)
-
-          [ "$NOW" -lt "$TARGET" ] && sleep $((TARGET - NOW))
-
-      - name: Push ${{ env.BRANCH_STAGING }}
-        run: |
-          git push origin "${{ env.BRANCH_STAGING }}" --force
-
-      - name: Reset ${{ env.BRANCH_PREVIOUS }} to Match ${{ env.BRANCH_FROGPILOT }}
-        run: |
-          git switch "${{ env.BRANCH_PREVIOUS }}"
-          git reset --hard "origin/${{ env.BRANCH_FROGPILOT }}"
-          git push origin "${{ env.BRANCH_PREVIOUS }}" --force
-
-      - name: Reset ${{ env.BRANCH_FROGPILOT }} to Match ${{ env.BRANCH_STAGING }}
-        run: |
-          git switch "${{ env.BRANCH_FROGPILOT }}"
-          git reset --hard "origin/${{ env.BRANCH_STAGING }}"
-          git push origin "${{ env.BRANCH_FROGPILOT }}" --force

.github/workflows/update_tinygrad.yaml

@@ -1,79 +0,0 @@
-name: Update Tinygrad
-
-on:
-  workflow_dispatch:
-    inputs:
-      runner:
-        description: "Select the runner"
-        required: true
-        default: "c3"
-        type: choice
-        options:
-          - c3
-          - c3x
-
-jobs:
-  update_tinygrad:
-    name: Update Tinygrad
-    runs-on:
-      - self-hosted
-      - ${{ github.event.inputs.runner }}
-
-    steps:
-      - name: Get version
-        id: get_version
-        run: |
-          VERSION=$(grep -oP '^VERSION\s*=\s*"\K[^"]+' /data/openpilot/frogpilot/assets/model_manager.py)
-          echo "VERSION=$VERSION"
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Clone GitLab repo
-        env:
-          GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
-        run: |
-          rm -rf /data/tmp/frogpilot_tinygrad
-          mkdir -p /data/tmp/frogpilot_tinygrad
-          cd /data/tmp/frogpilot_tinygrad
-
-          git clone --depth 1 --branch Tinygrad "https://oauth2:${GITLAB_TOKEN}@gitlab.com/FrogAi/FrogPilot-Resources.git"
-
-      - name: Create Tinygrad Archive
-        run: |
-          set -euo pipefail
-          cd /data/openpilot
-
-          ARCHIVE_DIR="/data/tmp/frogpilot_tinygrad/FrogPilot-Resources"
-          ARCHIVE_NAME="Tinygrad_${{ steps.get_version.outputs.version }}.tar.gz"
-
-          TMPDIR="$(mktemp -d)"
-          touch "$TMPDIR/SConscript"
-
-          tar -czf "$ARCHIVE_NAME" \
-            --exclude="*.a" \
-            --exclude="*.o" \
-            --exclude="*.onnx" \
-            --exclude="*__pycache__*" \
-            --exclude="*tests*" \
-            --exclude="frogpilot/tinygrad_modeld/SConscript" \
-            frogpilot/tinygrad_modeld tinygrad_repo \
-            -C "$TMPDIR" \
-            --transform 's|^SConscript$|frogpilot/tinygrad_modeld/SConscript|' \
-            SConscript
-
-          mv "$ARCHIVE_NAME" "$ARCHIVE_DIR/"
-          rm -rf "$TMPDIR"
-
-      - name: Push updated Tinygrad
-        run: |
-          cd /data/tmp/frogpilot_tinygrad/FrogPilot-Resources
-
-          git config user.name "James"
-          git config user.email "91348155+FrogAi@users.noreply.github.com"
-
-          git add Tinygrad_*.tar.gz
-          git commit -m "Updated Tinygrad: ${{ steps.get_version.outputs.version }}" || echo "No changes to commit"
-          git push origin Tinygrad
-
-      - name: Cleanup temporary files
-        run: |
-          rm -rf /data/tmp/frogpilot_tinygrad

.gitignore

@@ -1,6 +1,4 @@
 venv/
-.venv/
-.ci_cache
 .env
 .clang-format
 .DS_Store
@@ -10,6 +8,7 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
+.vscode*
 model2.png
 a.out
 .hypothesis
@@ -33,31 +32,36 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
-*.qm
 config.json
 clcache
 compile_commands.json
-compare_runtime*.html
 
 persist
-selfdrive/pandad/pandad
-cereal/services.h
-cereal/gen
+board/obj/
+selfdrive/boardd/boardd
 selfdrive/logcatd/logcatd
 selfdrive/mapd/default_speeds_by_region.json
-system/proclogd/proclogd
-selfdrive/ui/translations/alerts_generated.h
-selfdrive/ui/translations/tmp
+selfdrive/proclogd/proclogd
+selfdrive/ui/_ui
 selfdrive/test/longitudinal_maneuvers/out
-selfdrive/car/tests/cars_dump
-system/camerad/camerad
-system/camerad/test/ae_gray_test
+selfdrive/visiond/visiond
+selfdrive/loggerd/loggerd
+selfdrive/loggerd/bootlog
+selfdrive/sensord/_gpsd
+selfdrive/sensord/_sensord
+selfdrive/camerad/camerad
+selfdrive/camerad/test/ae_gray_test
 selfdrive/modeld/_modeld
 selfdrive/modeld/_dmonitoringmodeld
 /src/
 
+one
+openpilot
 notebooks
+xx
+yy
 hyperthneed
+panda_jungle
 provisioning
 
 .coverage*
@@ -72,14 +76,10 @@ cppcheck_report.txt
 comma*.sh
 
 selfdrive/modeld/thneed/compile
-selfdrive/modeld/models/*.thneed
-selfdrive/modeld/models/*.pkl
+models/*.thneed
 
 *.bz2
 
 build/
 
 !**/.gitkeep
-
-poetry.toml
-Pipfile

.python-version

@@ -1 +0,0 @@
-3.11.4

Jenkinsfile

@@ -0,0 +1,243 @@
+def phone(String ip, String step_label, String cmd) {
+  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
+    def ssh_cmd = """
+ssh -tt -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END'
+
+set -e
+
+export CI=1
+export TEST_DIR=${env.TEST_DIR}
+export SOURCE_DIR=${env.SOURCE_DIR}
+export GIT_BRANCH=${env.GIT_BRANCH}
+export GIT_COMMIT=${env.GIT_COMMIT}
+
+source ~/.bash_profile
+if [ -f /TICI ]; then
+  source /etc/profile
+fi
+
+ln -snf ${env.TEST_DIR} /data/pythonpath
+
+if [ -f /EON ]; then
+  echo \$\$ > /dev/cpuset/app/tasks || true
+  echo \$PPID > /dev/cpuset/app/tasks || true
+  mkdir -p /dev/shm
+  chmod 777 /dev/shm
+fi
+
+cd ${env.TEST_DIR} || true
+${cmd}
+exit 0
+
+END"""
+
+    sh script: ssh_cmd, label: step_label
+  }
+}
+
+def phone_steps(String device_type, steps) {
+  lock(resource: "", label: device_type, inversePrecedence: true, variable: 'device_ip', quantity: 1) {
+    timeout(time: 60, unit: 'MINUTES') {
+      phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),)
+      steps.each { item ->
+        phone(device_ip, item[0], item[1])
+      }
+    }
+  }
+}
+
+pipeline {
+  agent none
+  environment {
+    TEST_DIR = "/data/openpilot"
+    SOURCE_DIR = "/data/openpilot_source/"
+  }
+  options {
+      timeout(time: 4, unit: 'HOURS')
+  }
+
+  stages {
+    stage('build releases') {
+      when {
+        branch 'devel-staging'
+      }
+
+      parallel {
+        stage('release2') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          steps {
+            phone_steps("eon-build", [
+              ["build release2-staging & dashcam-staging", "PUSH=1 $SOURCE_DIR/release/build_release.sh"],
+            ])
+          }
+        }
+
+        stage('release3') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          steps {
+            phone_steps("tici", [
+              ["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release.sh"],
+            ])
+          }
+        }
+      }
+    }
+
+    stage('openpilot tests') {
+      when {
+        not {
+          anyOf {
+            branch 'master-ci'; branch 'devel'; branch 'devel-staging';
+            branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging';
+            branch 'release3'; branch 'release3-staging'; branch 'dashcam3'; branch 'dashcam3-staging';
+            branch 'testing-closet*'; branch 'hotfix-*'
+          }
+        }
+      }
+
+      stages {
+        stage('On-device Tests') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          stages {
+            stage('parallel tests') {
+              parallel {
+                stage('C2: build') {
+                  steps {
+                    phone_steps("eon-build", [
+                      ["build master-ci", "cd $SOURCE_DIR/release && EXTRA_FILES='tools/' ./build_devel.sh"],
+                      ["build openpilot", "cd selfdrive/manager && ./build.py"],
+                      ["test manager", "python selfdrive/manager/test/test_manager.py"],
+                      ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
+                      ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
+                    ])
+                  }
+                }
+
+                stage('C2: replay') {
+                  steps {
+                    phone_steps("eon2", [
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
+                    ])
+                  }
+                }
+
+                stage('C2: HW + Unit Tests') {
+                  steps {
+                    phone_steps("eon", [
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["test sounds", "python selfdrive/ui/tests/test_soundd.py"],
+                      ["test boardd loopback", "python selfdrive/boardd/tests/test_boardd_loopback.py"],
+                      ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
+                      ["test encoder", "python selfdrive/loggerd/tests/test_encoder.py"],
+                      ["test logcatd", "python selfdrive/logcatd/tests/test_logcatd_android.py"],
+                      ["test updater", "python selfdrive/hardware/eon/test_neos_updater.py"],
+                    ])
+                  }
+                }
+
+                /*
+                stage('Power Consumption Tests') {
+                  steps {
+                    lock(resource: "", label: "c2-zookeeper", inversePrecedence: true, variable: 'device_ip', quantity: 1) {
+                      timeout(time: 90, unit: 'MINUTES') {
+                        sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device"
+                        phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),)
+                        phone(device_ip, "build", "scons -j4 && sync")
+                        sh script: "/home/batman/tools/zookeeper/disable.py $device_ip", label: "turn off device"
+                        sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device"
+                        sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 3", label: "idle power consumption after boot"
+                        sh script: "/home/batman/tools/zookeeper/ignition.py 1", label: "go onroad"
+                        sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 10", label: "onroad power consumption"
+                        sh script: "/home/batman/tools/zookeeper/ignition.py 0", label: "go offroad"
+                        sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 2", label: "idle power consumption offroad"
+                      }
+                    }
+                  }
+                }
+                */
+
+                stage('C3: build') {
+                  environment {
+                    R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"
+                  }
+                  steps {
+                    phone_steps("tici", [
+                      ["build master-ci", "cd $SOURCE_DIR/release && EXTRA_FILES='tools/' ./build_devel.sh"],
+                      ["build openpilot", "cd selfdrive/manager && ./build.py"],
+                      ["test manager", "python selfdrive/manager/test/test_manager.py"],
+                      ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
+                      ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
+                    ])
+                  }
+                }
+
+                stage('C3: HW + Unit Tests') {
+                  steps {
+                    phone_steps("tici2", [
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["test boardd loopback", "python selfdrive/boardd/tests/test_boardd_loopback.py"],
+                      ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
+                      ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python selfdrive/loggerd/tests/test_encoder.py"],
+                    ])
+                  }
+                }
+
+                stage('C2: camerad') {
+                  steps {
+                    phone_steps("eon-party", [
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["test camerad", "python selfdrive/camerad/test/test_camerad.py"],
+                      ["test exposure", "python selfdrive/camerad/test/test_exposure.py"],
+                    ])
+                  }
+                }
+
+                stage('C3: camerad') {
+                  steps {
+                    phone_steps("tici-party", [
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["test camerad", "python selfdrive/camerad/test/test_camerad.py"],
+                      ["test exposure", "python selfdrive/camerad/test/test_exposure.py"],
+                    ])
+                  }
+                }
+
+                stage('C3: replay') {
+                  steps {
+                    phone_steps("tici-party", [
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
+                    ])
+                  }
+                }
+
+              }
+            }
+
+            stage('Push master-ci') {
+              when {
+                branch 'master'
+              }
+              steps {
+                phone_steps("eon-build", [
+                  ["push devel", "cd $SOURCE_DIR/release && PUSH='master-ci' ./build_devel.sh"],
+                ])
+              }
+            }
+
+          }
+
+          post {
+            always {
+              cleanWs()
+            }
+          }
+
+        }
+
+      }
+    }
+  }
+}
+

README.md

@@ -1,252 +1,142 @@
-<div align="center" style="text-align: center;">
+![](https://i.imgur.com/b0ZyIx5.jpg)
 
-<h1>openpilot</h1>
+Table of Contents
+=======================
 
-<p>
-  <b>openpilot is an operating system for robotics.</b>
-  <br>
-  Currently, it upgrades the driver assistance system in 300+ supported cars.
-</p>
+* [What is openpilot?](#what-is-openpilot)
+* [Running in a car](#running-in-a-car)
+* [Running on PC](#running-on-pc)
+* [Community and Contributing](#community-and-contributing)
+* [User Data and comma Account](#user-data-and-comma-account)
+* [Safety and Testing](#safety-and-testing)
+* [Directory Structure](#directory-structure)
+* [Licensing](#licensing)
 
-<h3>
-  <a href="https://docs.comma.ai">Docs</a>
-  <span> · </span>
-  <a href="https://docs.comma.ai/contributing/roadmap/">Roadmap</a>
-  <span> · </span>
-  <a href="https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md">Contribute</a>
-  <span> · </span>
-  <a href="https://discord.comma.ai">Community</a>
-  <span> · </span>
-  <a href="https://comma.ai/shop">Try it on a comma 3X</a>
-</h3>
+---
 
-Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
+What is openpilot?
+------
 
-[![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg)](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
-[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
-[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
-
-</div>
+[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW) and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models and model years](docs/CARS.md). In addition, while openpilot is engaged, a camera based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](docs/INTEGRATION.md) and [limitations](docs/LIMITATIONS.md).
 
 <table>
   <tr>
-    <td><a href="https://youtu.be/NmBfgOanCyk" title="Video By Greer Viau"><img src="https://github.com/commaai/openpilot/assets/8762862/2f7112ae-f748-4f39-b617-fabd689c3772"></a></td>
-    <td><a href="https://youtu.be/VHKyqZ7t8Gw" title="Video By Logan LeGrand"><img src="https://github.com/commaai/openpilot/assets/8762862/92351544-2833-40d7-9e0b-7ef7ae37ec4c"></a></td>
-    <td><a href="https://youtu.be/SUIZYzxtMQs" title="A drive to Taco Bell"><img src="https://github.com/commaai/openpilot/assets/8762862/05ceefc5-2628-439c-a9b2-89ce77dc6f63"></a></td>
+    <td><a href="https://youtu.be/NmBfgOanCyk" title="Video By Greer Viau"><img src="https://i.imgur.com/1w8c6d2.jpg"></a></td>
+    <td><a href="https://youtu.be/VHKyqZ7t8Gw" title="Video By Logan LeGrand"><img src="https://i.imgur.com/LnBucik.jpg"></a></td>
+    <td><a href="https://youtu.be/VxiR4iyBruo" title="Video By Charlie Kim"><img src="https://i.imgur.com/4Qoy48c.jpg"></a></td>
+    <td><a href="https://youtu.be/-IkImTe1NYE" title="Video By Aragon"><img src="https://i.imgur.com/04VNzPf.jpg"></a></td>
+  </tr>
+  <tr>
+    <td><a href="https://youtu.be/iIUICQkdwFQ" title="Video By Logan LeGrand"><img src="https://i.imgur.com/b1LHQTy.jpg"></a></td>
+    <td><a href="https://youtu.be/XOsa0FsVIsg" title="Video By PinoyDrives"><img src="https://i.imgur.com/6FG0Bd8.jpg"></a></td>
+    <td><a href="https://youtu.be/bCwcJ98R_Xw" title="Video By JS"><img src="https://i.imgur.com/zO18CbW.jpg"></a></td>
+    <td><a href="https://youtu.be/BQ0tF3MTyyc" title="Video By Tsai-Fi"><img src="https://i.imgur.com/eZzelq3.jpg"></a></td>
   </tr>
 </table>
 
 
-Using openpilot in a car
+Running in a car
 ------
 
-To use openpilot in a car, you need four things:
-1. **Supported Device:** a comma 3/3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x).
-2. **Software:** The setup procedure for the comma 3/3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
-3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
-4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3/3X to your car.
+To use openpilot in a car, you need four things
+* This software. It's free and available right here.
+* One of [the 150+ supported cars](docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, and more. If your car is not supported, but has adaptive cruise control and lane keeping assist, it's likely able to run openpilot.
+* A supported device to run this software. This can be a [comma two](https://comma.ai/shop/products/two), [comma three](https://comma.ai/shop/products/three), or if you like to experiment, a [Ubuntu computer with webcams](https://github.com/commaai/openpilot/tree/master/tools/webcam).
+* A way to connect to your car. With a comma two or three, you need only a [car harness](https://comma.ai/shop/products/car-harness). With an EON Gold or PC, you also need a [black panda](https://comma.ai/shop/products/panda).
 
-We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
+We have detailed instructions for [how to install the device in a car](https://comma.ai/setup).
 
+Running on PC
 ------
 
-<div align="center" style="text-align: center;">
+All of openpilot's services can run as normal on a PC, even without special hardware or a car. To develop or experiment with openpilot you can run openpilot on recorded or simulated data.
 
-<h1>FrogPilot 🐸</h1>
+With openpilot's tools you can plot logs, replay drives and watch the full-res camera streams. See [the tools README](tools/README.md) for more information.
 
-[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/FrogAi/FrogPilot)
-[![Discord](https://img.shields.io/discord/1137853399715549214?label=Discord)](https://discord.frogpilot.download)
-[![Last Updated](https://img.shields.io/badge/Last%20Updated-August%209th%2C%202025-brightgreen)](https://github.com/FrogAi/FrogPilot/releases/latest)
-[![Wiki](https://img.shields.io/badge/Wiki-FrogPilot-blue?logo=wiki)](https://frogpilot.wiki.gg/)
+You can also run openpilot in simulation [with the CARLA simulator](tools/sim/README.md). This allows openpilot to drive around a virtual car on your Ubuntu machine. The whole setup should only take a few minutes, but does require a decent GPU.
 
-</div>
 
+Community and Contributing
 ------
 
-**FrogPilot** is a custom frog-themed fork of openpilot that embraces a collaborative, community-driven approach to push the project forward. It delivers bleeding-edge features and experimental improvements far ahead of official releases. As an unofficial and highly experimental version of openpilot, **FrogPilot** should always be used with caution!
+openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot). Bug fixes and new car ports are encouraged. Check out [the contributing docs](docs/CONTRIBUTING.md).
 
-openpilot vs **FrogPilot**
+Documentation related to openpilot development can be found on [docs.comma.ai](https://docs.comma.ai). Information about running openpilot (e.g. FAQ, fingerprinting, troubleshooting, custom forks, community hardware) should go on the [wiki](https://github.com/commaai/openpilot/wiki).
+
+You can add support for your car by following guides we have written for [Brand](https://blog.comma.ai/how-to-write-a-car-port-for-openpilot/) and [Model](https://blog.comma.ai/openpilot-port-guide-for-toyota-models/) ports. Generally, a car with adaptive cruise control and lane keep assist is a good candidate. [Join our Discord](https://discord.comma.ai) to discuss car ports: most car makes have a dedicated channel.
+
+Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs/).
+
+And [follow us on Twitter](https://twitter.com/comma_ai).
+
+User Data and comma Account
 ------
 
-#### Community
-| Feature | openpilot | **FrogPilot** |
-|---------|:---------:|:---------:|
-| A Welcoming Community | ❌ | ✅ |
-| Erich / Primary Moderators / 🦇 | ✅ | ❌ |
+By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
 
-#### Core Features
-| Feature | openpilot | **FrogPilot** |
-|---------|:---------:|:---------:|
-| Always On Lateral (Steering) | ❌ | ✅ |
-| Blind Spot Integration | ✅ | ✅ |
-| Conditional Experimental Mode | ❌ | ✅ |
-| Custom Themes | ❌ | ✅ |
-| Driver Monitoring | ✅ | ✅ |
-| Driving Model Selector | ❌ | ✅ |
-| Holiday Themes | ❌ | ✅ |
-| Speed Limit Support | ❌ | ✅ |
-| Weather Detection | ❌ | ✅ |
+openpilot is open source software: the user is free to disable data collection if they wish to do so.
 
-#### Device & Hardware
-| Feature | openpilot | **FrogPilot** |
-|---------|:---------:|:---------:|
-| Advanced Volume Controller | ❌ | ✅ |
-| Automatic Version Backups | ❌ | ✅ |
-| C3 Support | ❌ | ✅ |
-| comma Pedal Support | ❌ | ✅ |
-| High Quality Recordings | ❌ | ✅ |
-| SDSU Support | ❌ | ✅ |
-| ZSS Support | ❌ | ✅ |
+openpilot logs the road facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
+The driver facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
 
-#### Gas/Brake
-| Feature | openpilot | **FrogPilot** |
-|---------|:---------:|:---------:|
-| Adaptive Cruise Control (ACC) | ✅ | ✅ |
-| Advanced Live Tuning | ❌ | ✅ |
-| Custom Following Distances | ❌ | ✅ |
-| Faster Human-Like Acceleration | ❌ | ✅ |
-| Human-Like Speed Control in Curves | ❌ | ✅ |
-| Smoother Human-Like Braking | ❌ | ✅ |
+By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
 
-#### Steering
-| Feature | openpilot | **FrogPilot** |
-|---------|:---------:|:---------:|
-| Advanced Live Tuning | ❌ | ✅ |
-| Automatic Lane Changes | ❌ | ✅ |
-| Increased Steering Torque* | ❌ | ✅ |
-| Lane Centering (LKAS) | ✅ | ✅ |
-| Lane Change Assist | ✅ | ✅ |
+Safety and Testing
+----
 
-*Select vehicles only
+* openpilot observes ISO26262 guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
+* openpilot has software in the loop [tests](.github/workflows/selfdrive_tests.yaml) that run on every commit.
+* The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
+* panda has software in the loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
+* Internally, we have a hardware in the loop Jenkins test suite that builds and unit tests the various processes.
+* panda has additional hardware in the loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
+* We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
 
-And much much more!
+Directory Structure
+------
+    .
+    ├── cereal              # The messaging spec and libs used for all logs
+    ├── common              # Library like functionality we've developed here
+    ├── docs                # Documentation
+    ├── opendbc             # Files showing how to interpret data from cars
+    ├── panda               # Code used to communicate on CAN
+    ├── third_party         # External libraries
+    ├── pyextra             # Extra python packages
+    └── selfdrive           # Code needed to drive the car
+        ├── assets          # Fonts, images, and sounds for UI
+        ├── athena          # Allows communication with the app
+        ├── boardd          # Daemon to talk to the board
+        ├── camerad         # Driver to capture images from the camera sensors
+        ├── car             # Car specific code to read states and control actuators
+        ├── common          # Shared C/C++ code for the daemons
+        ├── controls        # Planning and controls
+        ├── debug           # Tools to help you debug and do car ports
+        ├── locationd       # Precise localization and vehicle parameter estimation
+        ├── logcatd         # Android logcat as a service
+        ├── loggerd         # Logger and uploader of car data
+        ├── modeld          # Driving and monitoring model runners
+        ├── proclogd        # Logs information from proc
+        ├── sensord         # IMU interface code
+        ├── test            # Unit tests, system tests, and a car simulator
+        └── ui              # The UI
 
-🌟 Highlight Features
+Licensing
 ------
 
-### 🚗 Always On Lateral (AOL)
+openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
-With **"Always On Lateral"**, lane-centering stays active whenever cruise control is on, even when you press the accelerator or brake. This means steering assist won't cut out during manual speed adjustments giving you continuous support through curves, traffic, or mountain roads!
+Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
+
+**THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
+YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
+NO WARRANTY EXPRESSED OR IMPLIED.**
 
 ---
 
-### 🧠 Conditional Experimental Mode (CEM)
+<img src="https://d1qb2nb5cznatu.cloudfront.net/startups/i/1061157-bc7e9bf3b246ece7322e6ffe653f6af8-medium_jpg.jpg?buster=1458363130" width="75"></img> <img src="https://cdn-images-1.medium.com/max/1600/1*C87EjxGeMPrkTuVRVWVg4w.png" width="225"></img>
 
-**["Experimental Mode"](https://blog.comma.ai/090release/#experimental-mode)** lets openpilot drive at the speed it thinks a human would to allow slowing for curves, stopping at stoplights/stop signs, and adapting to traffic. This makes it powerful in complex scenarios, but it's still, well, "experimental" and less predictable than **"Chill Mode"**. But **"Conditional Experimental Mode"** gives you the best of both worlds by automatically switching between **"Chill Mode"** for steady cruising and **"Experimental Mode"** for more advanced situations to help fully automate your driving experience!
-
-**"Conditional Experimental Mode"** switches into **"Experimental Mode"** when conditions like these are met:
-- Approaching curves and turns
-- Detecting slower or stopped lead vehicles
-- Driving below a set speed
-- Predicting an upcoming stop (e.g. stoplight or stop sign)
-
-Once conditions clear it returns to **"Chill Mode"** for stability and predictability.
-
-**Note: Stay attentive as "Experimental Mode" is an alpha feature and mistakes are expected!**
-
----
-
-### 🎭 Driving Personalities
-
-With **"Driving Personalities"**, you choose how the vehicle behaves with four adjustable profiles:
-
-- **Traffic:** Catered towards stop-and-go traffic by minimizing gaps and delays  
-- **Aggressive:** Aimed to provide tighter following distances and quicker reactions  
-- **Standard:** Useful for a balanced, all-purpose driving  
-- **Relaxed:** A smoother driving experience with larger following distance gaps  
-
-Each profile can be fine-tuned to change the desired following distance, acceleration, and braking style letting you shape **FrogPilot**'s behavior to match your own driving preferences! Profiles can be switched instantly using the following distance button on the steering wheel, while **"Traffic Mode"** can be enabled by simply holding down the following distance button.
-
----
-
-### 📏 Speed Limit Controller (SLC)
-
-With **"Speed Limit Controller"**, **FrogPilot** automatically adapts to the road's posted speed using information from downloaded **["OpenStreetMap"](https://www.openstreetmap.org)** maps, online **["Mapbox"](https://www.mapbox.com)** data, and the vehicle's dashboard (if supported).
-
-Offsets let you fine-tune how closely **FrogPilot** follows posted limits across different speed ranges allowing you to cruise slightly above or below for a more natural driving experience. If no speed limit is available, you can choose whether **FrogPilot** drives at the set speed, falls back to the last known speed limit, or uses **"Experimental Mode"** to estimate one with the driving model.
-
-Maps can be downloaded directly in settings and updated automatically on a schedule ensuring your device always has the latest speed limits!
-
-**Note: Speed limits are only as accurate as the available speed limit data. Always stay attentive and adjust your speed when necessary!**
-
----
-
-### 🎨 Themes
-
-With **"Themes"**, you can personalize **FrogPilot**'s driving screen to make it uniquely yours! Choose from:
-
-- **Color Schemes**
-- **Icon Packs**
-- **Sound Packs**
-- **Turn Signal Animations**
-- **Steering Wheel Icons**
-
-Enjoy pre-existing **FrogPilot** and seasonal holiday themes, or you can create your own with the **"Theme Maker"** and even share them with the community! For extra fun, enable features like the Mario Kart–style **"Rainbow Path"** or **"Random Events"** that add playful visual effects while you drive!
-
----
-
-And lots more! From safety enhancements to personalization options, **FrogPilot** continues to evolve with features that put you in control. Check it out today for yourself!
-
----
-
-🔧 Branches
-------
-| Branch                     | Install&nbsp;URL          | Description                                            | Recommended&nbsp;For     |
-|----------------------------|---------------------------|--------------------------------------------------------|--------------------------|
-| FrogPilot                  | frogpilot.download        | The main release branch.                               | Everyone                 |
-| FrogPilot&#8209;Staging    | staging.frogpilot.download| Beta branch with upcoming features. Expect bugs!       | Early&nbsp;Adopters      |
-| FrogPilot&#8209;Testing    | testing.frogpilot.download| Alpha branch with bleeding-edge features. Breaks often!| Advanced&nbsp;Testers    |
-| FrogPilot&#8209;Development| No :)                     | Active development branch. Don't use!                  | **FrogPilot**&nbsp;Developers|
-| MAKE&#8209;PRS&#8209;HERE  | No :)                     | Workspace for pull requests. Don't use!                | Contributors             |
-
-🧰 How to Install
-------
-
-The easiest way to install **FrogPilot** is by entering this URL on the installation screen:
-
-```
-frogpilot.download
-```
-
-**DO NOT** install the **FrogPilot-Development** branch. I'm constantly breaking things on there, so unless you don't want to use **FrogPilot**, **NEVER** install it!
-
-![](https://i.imgur.com/LTCqRqB.png)
-
-🐞 Bug Reports / Feature Requests
-------
-
-If you run into bugs, issues, or have ideas for new features, please post about it on the **[FrogPilot Discord](https://discord.gg/frogpilot)**! Feedback helps improve **FrogPilot** and create a better experience for everyone!
-
-To report a bug, please post it in [**#bug-reports**](https://discord.com/channels/1137853399715549214/1162100167110053888).  
-To request a feature, please post it in [**#feature-requests**](https://discord.com/channels/1137853399715549214/1160318669839147259).  
-
-Please include as much detail as possible! Photos, videos, log files, or anything that can help explain the issue or idea are very helpful!
-
-I'll do my best to respond promptly, but not every request can be addressed right away. Your feedback is always appreciated and helps make **FrogPilot** the best it can be!
-
-📋 Credits
-------
-
-* [Aidenir](https://github.com/Aidenir)
-* [AlexandreSato](https://github.com/AlexandreSato)
-* [cfranyota](https://github.com/cfranyota)
-* [cydia2020](https://github.com/cydia2020)
-* [dragonpilot-community](https://github.com/dragonpilot-community)
-* [ErichMoraga](https://github.com/ErichMoraga)
-* [garrettpall](https://github.com/garrettpall)
-* [jakethesnake420](https://github.com/jakethesnake420)
-* [jyoung8607](https://github.com/jyoung8607)
-* [mike8643](https://github.com/mike8643)
-* [neokii](https://github.com/neokii)
-* [OPGM](https://github.com/opgm)
-* [OPKR](https://github.com/openpilotkr)
-* [pfeiferj](https://github.com/pfeiferj)
-* [realfast](https://github.com/realfast)
-* [syncword](https://github.com/syncword)
-* [twilsonco](https://github.com/twilsonco)
-
-Star History
-------
-
-[![Star History Chart](https://api.star-history.com/svg?repos=FrogAi/FrogPilot&type=Date)](https://www.star-history.com/#FrogAi/FrogPilot&Date)
+[![openpilot tests](https://github.com/commaai/openpilot/workflows/openpilot%20tests/badge.svg?event=push)](https://github.com/commaai/openpilot/actions)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/alerts/)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:python)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:cpp)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)

RELEASES.md

@@ -1,243 +1,7 @@
-Version 0.9.7 (2024-06-13)
-========================
-* New driving model
-  * Inputs the past curvature for smoother and more accurate lateral control
-  * Simplified neural network architecture in the model's last layers
-  * Minor fixes to desire augmentation and weight decay
-* New driver monitoring model
-  * Improved end-to-end bit for phone detection
-* Adjust driving personality with the follow distance button
-* Support for hybrid variants of supported Ford models
-* Fingerprinting without the OBD-II port on all cars
-* Improved fuzzy fingerprinting for Ford and Volkswagen
-
-Version 0.9.6 (2024-02-27)
-========================
-* New driving model
-  * Vision model trained on more data
-  * Improved driving performance
-  * Directly outputs curvature for lateral control
-* New driver monitoring model
-  * Trained on larger dataset
-* Model path UI
-  * Shows where driving model wants to be
-  * Shows what model is seeing more clearly, but more jittery
-* AGNOS 9
-* comma body streaming and controls over WebRTC
-* Improved fuzzy fingerprinting for many makes and models
-* Alpha longitudinal support for new Toyota models
-* Chevrolet Equinox 2019-22 support thanks to JasonJShuler and nworb-cire!
-* Dodge Durango 2020-21 support
-* Hyundai Staria 2023 support thanks to sunnyhaibin!
-* Kia Niro Plug-in Hybrid 2022 support thanks to sunnyhaibin!
-* Lexus LC 2024 support thanks to nelsonjchen!
-* Toyota RAV4 2023-24 support
-* Toyota RAV4 Hybrid 2023-24 support
-
-Version 0.9.5 (2023-11-17)
-========================
-* New driving model
-  * Improved navigate on openpilot performance using navigation instructions as an additional model input
-  * Do lateral planning inside the model
-  * New vision transformer architecture
-* Cadillac Escalade ESV 2019 support thanks to twilsonco!
-* Hyundai Azera 2022 support thanks to sunnyhaibin!
-* Hyundai Azera Hybrid 2020 support thanks to chanhojung and haram-KONA!
-* Hyundai Custin 2023 support thanks to sunnyhaibin and Saber422!
-* Hyundai Ioniq 6 2023 support thanks to sunnyhaibin and alamo3!
-* Hyundai Kona Electric 2023 (Korean version) support thanks to sunnyhaibin and haram-KONA!
-* Kia K8 Hybrid (with HDA II) 2023 support thanks to sunnyhaibin!
-* Kia Optima Hybrid 2019 support
-* Kia Sorento Hybrid 2023 support thanks to sunnyhaibin!
-* Lexus GS F 2016 support thanks to snyperifle!
-* Lexus IS 2023 support thanks to L3R5!
-
-Version 0.9.4 (2023-07-27)
-========================
-* comma 3X support
-* Navigate on openpilot in Experimental mode
-  * When navigation has a destination, openpilot will input the map information into the model, which provides useful context to help the model understand the scene
-  * When navigating on openpilot, openpilot will keep left or right appropriately at forks and exits
-  * When navigating on openpilot, lane change behavior is unchanged and still activated by the driver
-  * When navigate on openpilot is active, the path on the map is green
-* UI updates
-  * Navigation settings moved to home screen and map
-  * Border color always shows engagement status. Blue means disengaged, green means engaged, and grey means engaged with human overriding
-  * Alerts are shown inside the border. Black means info, orange means warning, and red means critical alert
-* Bookmarked segments are preserved on the device's storage
-* Ford Focus 2018 support
-* Kia Carnival 2023 support thanks to sunnyhaibin!
-
-Version 0.9.3 (2023-06-29)
-========================
-* New driving model
-  * Improved height estimation and added height tracking in liveCalibration
-  * Model inputs refactor
-* New driving personality setting
-  * Three settings: aggressive, standard, and relaxed
-  * Standard is recommended and the default
-  * In aggressive mode, lead follow distance is shorter and acceleration response is quicker
-  * In relaxed mode, lead follow distance is longer
-* Improved fuzzy fingerprinting for Hyundai, Kia, and Genesis
-* Improved thermal management logic
-
-Version 0.9.2 (2023-05-22)
-========================
-* New driving model
-  * Reduced turn diving
-  * Trained on a new dataset
-* UI updates
-  * New experimental mode visualization
-  * Draw MPC path instead of model-predicted path
-* AGNOS 7
-  * Faster boot time
-  * Fixes rare no sounds bug
-  * Fixes bootsplash bug at extreme temperatures
-* Buick LaCrosse 2017-19 support thanks to koch-cf!
-* Chevrolet Trailblazer 2021-22 support thanks to TurboCE!
-* Ford Bronco Sport 2021-22 support
-* Ford Escape 2020-22 support
-* Ford Explorer 2020-22 support
-* Ford Kuga 2020-22 support
-* Ford Maverick 2022-23 support
-* Genesis GV80 2023 support thanks to JWingate80!
-* Honda HR-V 2023 support thanks to AlexandreSato and galegozi!
-* Kia Niro EV 2023 support thanks to JosselinLecocq!
-* Lexus ES 2017-18 support
-* Lincoln Aviator 2021 support
-* Škoda Fabia 2022-23 support thanks to jyoung8607!
-
-
-Version 0.9.1 (2023-02-28)
-========================
-* New driving model
-  * 30% improved height estimation resulting in better driving performance for tall cars
-* Driver monitoring: removed timer resetting on user interaction if distracted
-* UI updates
-  * Adjust alert volume using ambient noise level
-  * Driver monitoring icon shows driver's head pose
-  * German translation thanks to Vrabetz and CzokNorris!
-* Cadillac Escalade 2017 support thanks to rickygilleland!
-* Chevrolet Bolt EV 2022-23 support thanks to JasonJShuler!
-* Genesis GV60 2023 support thanks to sunnyhaibin!
-* Hyundai Tucson 2022-23 support
-* Kia K5 Hybrid 2020 support thanks to sunnyhaibin!
-* Kia Niro Hybrid 2023 support thanks to sunnyhaibin!
-* Kia Sorento 2022-23 support thanks to sunnyhaibin!
-* Kia Sorento Plug-in Hybrid 2022 support thanks to sunnyhaibin!
-* Toyota C-HR 2021 support thanks to eFiniLan!
-* Toyota C-HR Hybrid 2022 support thanks to Korben00!
-* Volkswagen Crafter and MAN TGE 2017-23 support thanks to jyoung8607!
-
-Version 0.9.0 (2022-11-21)
-========================
-* New driving model
-  * Internal feature space information content increased tenfold during training to ~700 bits, which makes the model dramatically more accurate
-  * Less reliance on previous frames makes model more reactive and snappy
-  * Trained in new reprojective simulator
-  * Trained in 36 hours from scratch, compared to one week for previous releases
-  * Training now simulates both lateral and longitudinal behavior, which allows openpilot to slow down for turns, stop at traffic lights, and more in experimental mode
-* Experimental driving mode
-  * End-to-end longitudinal control
-  * Stops for traffic lights and stop signs
-  * Slows down for turns
-  * openpilot defaults to chill mode, enable experimental mode in settings
-* Driver monitoring updates
-  * New bigger model with added end-to-end distracted trigger
-  * Reduced false positives during driver calibration
-* Self-tuning torque controller: learns parameters live for each car
-* Torque controller used on all Toyota, Lexus, Hyundai, Kia, and Genesis models
-* UI updates
-  * Matched speeds shown on car's dash
-  * Multi-language in navigation
-  * Improved update experience
-  * Border turns grey while overriding steering
-  * Bookmark events while driving; view them in comma connect
-  * New onroad visualization for experimental mode
-* tools: new and improved cabana thanks to deanlee!
-* Experimental longitudinal support for Volkswagen, CAN-FD Hyundai, and new GM models
-* Genesis GV70 2022-23 support thanks to zunichky and sunnyhaibin!
-* Hyundai Santa Cruz 2021-22 support thanks to sunnyhaibin!
-* Kia Sportage 2023 support thanks to sunnyhaibin!
-* Kia Sportage Hybrid 2023 support thanks to sunnyhaibin!
-* Kia Stinger 2022 support thanks to sunnyhaibin!
-
-Version 0.8.16 (2022-08-26)
-========================
-* New driving model
-  * Reduced turn cutting
-* Auto-detect right hand drive setting with driver monitoring model
-* Improved fan controller for comma three
-* New translations
-  * Japanese thanks to cydia2020!
-  * Brazilian Portuguese thanks to AlexandreSato!
-* Chevrolet Bolt EUV 2022-23 support thanks to JasonJShuler!
-* Chevrolet Silverado 1500 2020-21 support thanks to JasonJShuler!
-* GMC Sierra 1500 2020-21 support thanks to JasonJShuler!
-* Hyundai Ioniq 5 2022 support thanks to sunnyhaibin!
-* Hyundai Kona Electric 2022 support thanks to sunnyhaibin!
-* Hyundai Tucson Hybrid 2022 support thanks to sunnyhaibin!
-* Subaru Legacy 2020-22 support thanks to martinl!
-* Subaru Outback 2020-22 support
-
-Version 0.8.15 (2022-07-20)
-========================
-* New driving model
-  * Path planning uses end-to-end output instead of lane lines at all times
-  * Reduced ping pong
-  * Improved lane centering
-* New lateral controller based on physical wheel torque model
-  * Much smoother control that's consistent across the speed range
-  * Effective feedforward that uses road roll
-  * Simplified tuning, all car-specific parameters can be derived from data
-  * Used on select Toyota and Hyundai models at first
-  * Significantly improved control on TSS-P Prius
-* New driver monitoring model
-  * Bigger model, covering full interior view from driver camera
-  * Works with a wider variety of mounting angles
-  * 3x more unique comma three training data than previous
-* Navigation improvements
-  * Speed limits shown while navigating
-  * Faster position fix by using raw GPS measurements
-* UI updates
-  * Multilanguage support for settings and home screen
-  * New font
-  * Refreshed max speed design
-  * More consistent camera view perspective across cars
-* Reduced power usage: device runs cooler and fan spins less
-* AGNOS 5
-  * Support VSCode remote SSH target
-  * Support for delta updates to reduce data usage on future OS updates
-* Chrysler ECU firmware fingerprinting thanks to realfast!
-* Honda Civic 2022 support
-* Hyundai Tucson 2021 support thanks to bluesforte!
-* Kia EV6 2022 support
-* Lexus NX Hybrid 2020 support thanks to AlexandreSato!
-* Ram 1500 2019-21 support thanks to realfast!
-
-Version 0.8.14 (2022-06-01)
-========================
- * New driving model
-   * Bigger model, using both of comma three's road-facing cameras
-   * Better at cut-in detection and tight turns
- * New driver monitoring model
-   * Tweaked network structure to improve output resolution for DSP
-   * Fixed bug in quantization aware training to reduce quantizing errors
-   * Resulted in 7x less MSE and no more random biases at runtime
- * Added toggle to disable disengaging on the accelerator pedal
- * comma body support
- * Audi RS3 support thanks to jyoung8607!
- * Hyundai Ioniq Plug-in Hybrid 2019 support thanks to sunnyhaibin!
- * Hyundai Tucson Diesel 2019 support thanks to sunnyhaibin!
- * Toyota Alphard Hybrid 2021 support
- * Toyota Avalon Hybrid 2022 support
- * Toyota RAV4 2022 support
- * Toyota RAV4 Hybrid 2022 support
-
 Version 0.8.13 (2022-02-18)
 ========================
  * Improved driver monitoring
-   * Re-tuned driver pose learner for relaxed driving positions
+   * Retuned driver pose learner for relaxed driving positions
    * Added reliance on driving model to be more scene adaptive
    * Matched strictness between comma two and comma three
  * Improved performance in turns by compensating for the road bank angle
@@ -383,7 +147,7 @@ Version 0.8.4 (2021-05-17)
  * Delay controls start until system is ready
  * Fuzzy car identification, enabled with Community Features toggle
  * Localizer optimized for increased precision and less CPU usage
- * Re-tuned lateral control to be more aggressive when model is confident
+ * Retuned lateral control to be more aggressive when model is confident
  * Toyota Mirai 2021 support
  * Lexus NX 300 2020 support thanks to goesreallyfast!
  * Volkswagen Atlas 2018-19 support thanks to jyoung8607!
@@ -548,7 +312,7 @@ Version 0.7 (2019-12-13)
  * Improve GM longitudinal control: proper computations for 15Hz radar
  * Move GM port, Toyota with DSU removed, comma pedal in community features; toggle switch required
  * Remove upload over cellular toggle: only upload qlog and qcamera files if not on wifi
- * Refactor Panda code towards ISO26262 and SIL2 compliance
+ * Refactor Panda code towards ISO26262 and SIL2 compliancy
  * Forward stock FCW for Honda Nidec
  * Volkswagen port now standard: comma Harness intercepts stock camera
 
@@ -639,7 +403,7 @@ Version 0.5.13 (2019-05-31)
  * Reduce CPU utilization by 20% and improve stability
  * Temporarily remove mapd functionalities to improve stability
  * Add openpilot record-only mode for unsupported cars
- * Synchronize controlsd to pandad to reduce latency
+ * Synchronize controlsd to boardd to reduce latency
  * Remove panda support for Subaru giraffe
 
 Version 0.5.12 (2019-05-16)
@@ -975,7 +739,7 @@ Version 0.2.8  (2017-02-27)
 Version 0.2.7  (2017-02-08)
 ===========================
  * Better performance and pictures at night
- * Fix ptr alignment issue in pandad
+ * Fix ptr alignment issue in boardd
  * Fix brake error light, fix crash if too cold
 
 Version 0.2.6  (2017-01-31)
@@ -1007,7 +771,7 @@ Version 0.2.2  (2017-01-10)
 Version 0.2.1  (2016-12-14)
 ===========================
  * Performance improvements, removal of more numpy
- * Fix pandad process priority
+ * Fix boardd process priority
  * Make counter timer reset on use of steering wheel
 
 Version 0.2  (2016-12-12)

SConstruct

@@ -0,0 +1,445 @@
+import os
+import shutil
+import subprocess
+import sys
+import sysconfig
+import platform
+import numpy as np
+
+TICI = os.path.isfile('/TICI')
+Decider('MD5-timestamp')
+
+AddOption('--test',
+          action='store_true',
+          help='build test files')
+
+AddOption('--extras',
+          action='store_true',
+          help='build misc extras, like setup and installer files')
+
+AddOption('--kaitai',
+          action='store_true',
+          help='Regenerate kaitai struct parsers')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--compile_db',
+          action='store_true',
+          help='build clang compilation database')
+
+AddOption('--snpe',
+          action='store_true',
+          help='use SNPE on PC')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+AddOption('--no-thneed',
+          action='store_true',
+          dest='no_thneed',
+          help='avoid using thneed')
+
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
+
+if arch == "aarch64" and TICI:
+  arch = "larch64"
+
+USE_WEBCAM = os.getenv("USE_WEBCAM") is not None
+
+lenv = {
+  "PATH": os.environ['PATH'],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath,
+
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath,
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer",
+}
+
+rpath = lenv["LD_LIBRARY_PATH"].copy()
+
+if arch == "aarch64" or arch == "larch64":
+  lenv["LD_LIBRARY_PATH"] += ['/data/data/com.termux/files/usr/lib']
+
+  if arch == "aarch64":
+    # android
+    lenv["ANDROID_DATA"] = os.environ['ANDROID_DATA']
+    lenv["ANDROID_ROOT"] = os.environ['ANDROID_ROOT']
+
+  cpppath = [
+    "#third_party/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/local/lib",
+    "/usr/lib",
+    "/system/vendor/lib64",
+    "/system/comma/usr/lib",
+    f"#third_party/acados/{arch}/lib",
+  ]
+
+  if arch == "larch64":
+    libpath += [
+      "#third_party/snpe/larch64",
+      "#third_party/libyuv/larch64/lib",
+      "/usr/lib/aarch64-linux-gnu"
+    ]
+    cpppath += [
+      "#selfdrive/camerad/include",
+    ]
+    cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+    cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+    rpath += ["/usr/local/lib"]
+  else:
+    rpath = []
+    libpath += [
+      "#third_party/snpe/aarch64",
+      "#third_party/libyuv/lib",
+      "/system/vendor/lib64"
+    ]
+    cflags = ["-DQCOM", "-D_USING_LIBCXX", "-mcpu=cortex-a57"]
+    cxxflags = ["-DQCOM", "-D_USING_LIBCXX", "-mcpu=cortex-a57"]
+else:
+  cflags = []
+  cxxflags = []
+  cpppath = []
+
+  if arch == "Darwin":
+    yuv_dir = "mac" if real_arch != "arm64" else "mac_arm64"
+    libpath = [
+      f"#third_party/libyuv/{yuv_dir}/lib",
+      "/usr/local/lib",
+      "/opt/homebrew/lib",
+      "/usr/local/Homebrew/Library",
+      "/usr/local/opt/openssl/lib",
+      "/opt/homebrew/opt/openssl/lib",
+      "/usr/local/Cellar",
+      f"#third_party/acados/{arch}/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += [
+      "/opt/homebrew/include",
+      "/usr/local/include",
+      "/usr/local/opt/openssl/include",
+      "/opt/homebrew/opt/openssl/include"
+    ]
+  else:
+    libpath = [
+      "#third_party/acados/x86_64/lib",
+      "#third_party/snpe/x86_64-linux-clang",
+      "#third_party/libyuv/x64/lib",
+      "#third_party/mapbox-gl-native-qt/x86_64",
+      "#cereal",
+      "#selfdrive/common",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+  rpath += [
+    Dir("#third_party/snpe/x86_64-linux-clang").abspath,
+    Dir("#cereal").abspath,
+    Dir("#selfdrive/common").abspath
+  ]
+
+if GetOption('asan'):
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
+else:
+  ccflags = []
+  ldflags = []
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
+
+# Enable swaglog include in submodules
+cflags += ["-DSWAGLOG"]
+cxxflags += ["-DSWAGLOG"]
+
+env = Environment(
+  ENV=lenv,
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Wunused",
+    "-Werror",
+    "-Wshadow",
+    "-Wno-unknown-warning-option",
+    "-Wno-deprecated-register",
+    "-Wno-register",
+    "-Wno-inconsistent-missing-override",
+    "-Wno-c99-designator",
+    "-Wno-reorder-init-list",
+    "-Wno-error=unused-but-set-variable",
+  ] + cflags + ccflags,
+
+  CPPPATH=cpppath + [
+    "#",
+    "#third_party/acados/include",
+    "#third_party/acados/include/blasfeo/include",
+    "#third_party/acados/include/hpipm/include",
+    "#third_party/catch2/include",
+    "#third_party/bzip2",
+    "#third_party/libyuv/include",
+    "#third_party/openmax/include",
+    "#third_party/json11",
+    "#third_party/curl/include",
+    "#third_party/libgralloc/include",
+    "#third_party/android_frameworks_native/include",
+    "#third_party/android_hardware_libhardware/include",
+    "#third_party/android_system_core/include",
+    "#third_party/linux/include",
+    "#third_party/snpe/include",
+    "#third_party/mapbox-gl-native-qt/include",
+    "#third_party/qrcode",
+    "#third_party",
+    "#cereal",
+    "#opendbc/can",
+  ],
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
+  LIBPATH=libpath + [
+    "#cereal",
+    "#third_party",
+    "#opendbc/can",
+    "#selfdrive/boardd",
+    "#selfdrive/common",
+  ],
+  CYTHONCFILESUFFIX=".cpp",
+  COMPILATIONDB_USE_ABSPATH=True,
+  tools=["default", "cython", "compilation_db"],
+)
+
+if arch == "Darwin":
+  env['RPATHPREFIX'] = "-rpath "
+
+if GetOption('compile_db'):
+  env.CompilationDatabase('compile_commands.json')
+
+# Setup cache dir
+cache_dir = '/data/scons_cache' if TICI else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
+
+node_interval = 5
+node_count = 0
+def progress_function(node):
+  global node_count
+  node_count += node_interval
+  sys.stderr.write("progress: %d\n" % node_count)
+
+if os.environ.get('SCONS_PROGRESS'):
+  Progress(progress_function, interval=node_interval)
+
+SHARED = False
+
+def abspath(x):
+  if arch == 'aarch64':
+    pth = os.path.join("/data/pythonpath", x[0].path)
+    env.Depends(pth, x)
+    return File(pth)
+  else:
+    # rpath works elsewhere
+    return x[0].path.rsplit("/", 1)[1][:-3]
+
+# Cython build enviroment
+py_include = sysconfig.get_paths()['include']
+envCython = env.Clone()
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+
+envCython["LIBS"] = []
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+elif arch == "aarch64":
+  envCython["LINKFLAGS"] = ["-shared"]
+  envCython["LIBS"] = [os.path.basename(py_include)]
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+Export('envCython')
+
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning"]
+if arch != "aarch64":
+  qt_modules += ["DBus"]
+
+qt_libs = []
+if arch == "Darwin":
+  if real_arch == "arm64":
+    qt_env['QTDIR'] = "/opt/homebrew/opt/qt@5"
+  else:
+    qt_env['QTDIR'] = "/usr/local/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
+elif arch == "aarch64":
+  qt_env['QTDIR'] = "/system/comma/usr"
+  qt_dirs = [
+    f"/system/comma/usr/include/qt",
+  ]
+  qt_dirs += [f"/system/comma/usr/include/qt/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  qt_libs += ['EGL', 'GLESv3', 'c++_shared']
+else:
+  qt_env['QTDIR'] = "/usr"
+  qt_dirs = [
+    f"/usr/include/{real_arch}-linux-gnu/qt5",
+    f"/usr/include/{real_arch}-linux-gnu/qt5/QtGui/5.12.8/QtGui",
+  ]
+  qt_dirs += [f"/usr/include/{real_arch}-linux-gnu/qt5/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+
+qt_env.Tool('qt')
+qt_env['CPPPATH'] += qt_dirs + ["#selfdrive/ui/qt/"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_QUICK_LIB",
+  "-DQT_QUICKWIDGETS_LIB",
+  "-DQT_QML_LIB",
+  "-DQT_CORE_LIB",
+  "-DQT_MESSAGELOGCONTEXT",
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui']
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+
+Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM')
+
+SConscript(['selfdrive/common/SConscript'])
+Import('_common', '_gpucommon', '_gpu_libs')
+
+if SHARED:
+  common, gpucommon = abspath(common), abspath(gpucommon)
+else:
+  common = [_common, 'json11']
+  gpucommon = [_gpucommon] + _gpu_libs
+
+Export('common', 'gpucommon')
+
+# cereal and messaging are shared with the system
+SConscript(['cereal/SConscript'])
+if SHARED:
+  cereal = abspath([File('cereal/libcereal_shared.so')])
+  messaging = abspath([File('cereal/libmessaging_shared.so')])
+else:
+  cereal = [File('#cereal/libcereal.a')]
+  messaging = [File('#cereal/libmessaging.a')]
+  visionipc = [File('#cereal/libvisionipc.a')]
+
+Export('cereal', 'messaging', 'visionipc')
+
+# Build rednose library and ekf models
+
+rednose_config = {
+  'generated_folder': '#selfdrive/locationd/models/generated',
+  'to_build': {
+    'live': ('#selfdrive/locationd/models/live_kf.py', True, ['live_kf_constants.h']),
+    'car': ('#selfdrive/locationd/models/car_kf.py', True, []),
+  },
+}
+
+if arch not in ["aarch64", "larch64"]:
+  rednose_config['to_build'].update({
+    'gnss': ('#selfdrive/locationd/models/gnss_kf.py', True, []),
+    'loc_4': ('#selfdrive/locationd/models/loc_kf.py', True, []),
+    'pos_computer_4': ('#rednose/helpers/lst_sq_computer.py', False, []),
+    'pos_computer_5': ('#rednose/helpers/lst_sq_computer.py', False, []),
+    'feature_handler_5': ('#rednose/helpers/feature_handler.py', False, []),
+    'lane': ('#xx/pipeline/lib/ekf/lane_kf.py', True, []),
+  })
+
+Export('rednose_config')
+SConscript(['rednose/SConscript'])
+
+# Build openpilot
+
+SConscript(['cereal/SConscript'])
+SConscript(['panda/board/SConscript'])
+SConscript(['opendbc/can/SConscript'])
+
+SConscript(['third_party/SConscript'])
+
+SConscript(['common/SConscript'])
+SConscript(['common/kalman/SConscript'])
+SConscript(['common/transformations/SConscript'])
+
+SConscript(['selfdrive/camerad/SConscript'])
+SConscript(['selfdrive/modeld/SConscript'])
+
+SConscript(['selfdrive/controls/lib/cluster/SConscript'])
+SConscript(['selfdrive/controls/lib/lateral_mpc_lib/SConscript'])
+SConscript(['selfdrive/controls/lib/longitudinal_mpc_lib/SConscript'])
+
+SConscript(['selfdrive/boardd/SConscript'])
+SConscript(['selfdrive/proclogd/SConscript'])
+SConscript(['selfdrive/clocksd/SConscript'])
+
+SConscript(['selfdrive/loggerd/SConscript'])
+
+SConscript(['selfdrive/locationd/SConscript'])
+SConscript(['selfdrive/sensord/SConscript'])
+SConscript(['selfdrive/ui/SConscript'])
+
+if arch != "Darwin":
+  SConscript(['selfdrive/logcatd/SConscript'])
+
+if GetOption('test'):
+  SConscript('panda/tests/safety/SConscript')
+
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

SECURITY.md

@@ -1,5 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-Suspected vulnerabilities can be reported to both `adeeb@comma.ai` and `security@comma.ai`.

cereal/.gitignore

@@ -0,0 +1,21 @@
+gen
+node_modules
+package-lock.json
+*.tmp
+*.pyc
+__pycache__
+.*.swp
+.*.swo
+*.os
+*.o
+*.a
+
+test_runner
+
+libmessaging.*
+libmessaging_shared.*
+services.h
+.sconsign.dblite
+libcereal_shared.*
+.mypy_cache/
+catch2/

cereal/README.md

@@ -1,60 +0,0 @@
-# What is cereal? [![cereal tests](https://github.com/commaai/cereal/workflows/tests/badge.svg?event=push)](https://github.com/commaai/cereal/actions) [![codecov](https://codecov.io/gh/commaai/cereal/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/cereal)
-
-cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers.
-
-Imagine this use case:
-* A sensor process reads gyro measurements directly from an IMU and publishes a `sensorEvents` packet
-* A calibration process subscribes to the `sensorEvents` packet to use the IMU
-* A localization process subscribes to the `sensorEvents` packet to use the IMU also
-
-
-## Messaging Spec
-
-You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
-
-All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
-
-### Best Practices
-
-- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
-- In the context of the message they are in, field names should be completely unambiguous.
-- All values should be easy to plot and be human-readable with minimal parsing.
-
-### Maintaining backwards-compatibility
-
-When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
-be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
-things are not. Read more details [here](https://capnproto.org/language.html).
-
-### Custom forks
-
-Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
-spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
-then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
-[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
-fork will remain backwards-compatible with all versions of mainline cereal/openpilot and your fork.**
-
-## Pub Sub Backends
-
-cereal supports two backends, one based on [zmq](https://zeromq.org/) and another called [msgq](messaging/msgq.cc), a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel.
-
-Example
----
-```python
-import cereal.messaging as messaging
-
-# in subscriber
-sm = messaging.SubMaster(['sensorEvents'])
-while 1:
-  sm.update()
-  print(sm['sensorEvents'])
-
-```
-
-```python
-# in publisher
-pm = messaging.PubMaster(['sensorEvents'])
-dat = messaging.new_message('sensorEvents', size=1)
-dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
-pm.send('sensorEvents', dat)
-```

cereal/SConscript

@@ -0,0 +1,72 @@
+Import('env', 'envCython', 'arch', 'common')
+
+import shutil
+
+cereal_dir = Dir('.')
+gen_dir = Dir('gen')
+messaging_dir = Dir('messaging')
+
+# Build cereal
+
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp']
+env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
+env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
+            schema_files,
+            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
+
+# TODO: remove non shared cereal and messaging
+cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files])
+
+env.Library('cereal', cereal_objects)
+env.SharedLibrary('cereal_shared', cereal_objects)
+
+# Build messaging
+
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+
+messaging_objects = env.SharedObject([
+  'messaging/messaging.cc',
+  'messaging/impl_zmq.cc',
+  'messaging/impl_msgq.cc',
+  'messaging/msgq.cc',
+  'messaging/socketmaster.cc',
+])
+
+messaging_lib = env.Library('messaging', messaging_objects)
+Depends('messaging/impl_zmq.cc', services_h)
+
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq', common])
+Depends('messaging/bridge.cc', services_h)
+
+envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging_lib, "zmq", common])
+
+
+# Build Vision IPC
+vipc_sources = [
+  'visionipc/ipc.cc',
+  'visionipc/visionipc_server.cc',
+  'visionipc/visionipc_client.cc',
+  'visionipc/visionbuf.cc',
+]
+
+if arch in ["aarch64", "larch64"]:
+  vipc_sources += ['visionipc/visionbuf_ion.cc']
+else:
+  vipc_sources += ['visionipc/visionbuf_cl.cc']
+
+vipc_objects = env.SharedObject(vipc_sources)
+vipc = env.Library('visionipc', vipc_objects)
+
+
+libs = envCython["LIBS"]+["OpenCL", "zmq", vipc, messaging_lib, common]
+if arch == "aarch64":
+  libs += ["adreno_utils"]
+if arch == "Darwin":
+  del libs[libs.index('OpenCL')]
+  envCython['FRAMEWORKS'] += ['OpenCL']
+envCython.Program('visionipc/visionipc_pyx.so', 'visionipc/visionipc_pyx.pyx', LIBS=libs)
+
+
+if GetOption('test'):
+  env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib, common])
+  env.Program('visionipc/test_runner', ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], LIBS=[vipc, messaging_lib, 'zmq', 'pthread', 'OpenCL', common])

cereal/__init__.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 import os
 import capnp
 
@@ -6,4 +7,3 @@ capnp.remove_import_hook()
 
 log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
 car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
-custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -17,12 +17,11 @@ struct CarEvent @0x9b1657f34caf3ad3 {
   immediateDisable @6 :Bool;
   preEnable @7 :Bool;
   permanent @8 :Bool; # alerts presented regardless of openpilot state
-  overrideLateral @10 :Bool;
-  overrideLongitudinal @9 :Bool;
 
   enum EventName @0xbaa8c5d505f727de {
     canError @0;
     steerUnavailable @1;
+    brakeUnavailable @2;
     wrongGear @4;
     doorOpen @5;
     seatbeltNotLatched @6;
@@ -32,25 +31,23 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     reverseGear @10;
     buttonCancel @11;
     buttonEnable @12;
-    pedalPressed @13;  # exits active state
-    preEnableStandstill @73;  # added during pre-enable state with brake
-    gasPressedOverride @108;  # added when user is pressing gas with no disengage on gas
-    steerOverride @114;
+    pedalPressed @13;
     cruiseDisabled @14;
     speedTooLow @17;
     outOfSpace @18;
     overheat @19;
     calibrationIncomplete @20;
     calibrationInvalid @21;
-    calibrationRecalibrating @117;
     controlsMismatch @22;
     pcmEnable @23;
     pcmDisable @24;
+    noTarget @25;
     radarFault @26;
     brakeHold @28;
     parkBrake @29;
     manualRestart @30;
     lowSpeedLockout @31;
+    plannerError @32;
     joystickDebug @34;
     steerTempUnavailableSilent @35;
     resumeRequired @36;
@@ -62,10 +59,10 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     driverUnresponsive @45;
     belowSteerSpeed @46;
     lowBattery @48;
+    vehicleModelInvalid @50;
     accFaulted @51;
     sensorDataInvalid @52;
     commIssue @53;
-    commIssueAvgFreq @109;
     tooDistracted @54;
     posenetInvalid @55;
     soundsUnavailable @56;
@@ -80,11 +77,11 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     speedTooHigh @70;
     laneChangeBlocked @71;
     relayMalfunction @72;
+    gasPressed @73;
     stockFcw @74;
     startup @75;
     startupNoCar @76;
     startupNoControl @77;
-    startupNoSecOcKey @121;
     startupMaster @78;
     startupNoFw @104;
     fcw @79;
@@ -96,7 +93,7 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     deviceFalling @90;
     fanMalfunction @91;
     cameraMalfunction @92;
-    cameraFrameRate @110;
+    gpsMalfunction @94;
     processNotRunning @95;
     dashcamMode @96;
     controlsInitializing @98;
@@ -104,19 +101,10 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     roadCameraError @100;
     driverCameraError @101;
     wideRoadCameraError @102;
+    localizerMalfunction @103;
     highCpuUsage @105;
     cruiseMismatch @106;
     lkasDisabled @107;
-    canBusMissing @111;
-    controlsdLagging @112;
-    resumeBlocked @113;
-    steerTimeLimit @115;
-    vehicleSensorsInvalid @116;
-    locationdTemporaryError @103;
-    locationdPermanentError @118;
-    paramsdTemporaryError @50;
-    paramsdPermanentError @119;
-    actuatorsApiUnavailable @120;
 
     radarCanErrorDEPRECATED @15;
     communityFeatureDisallowedDEPRECATED @62;
@@ -140,10 +128,6 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     modelLagWarningDEPRECATED @93;
     startupOneplusDEPRECATED @82;
     startupFuzzyFingerprintDEPRECATED @97;
-    noTargetDEPRECATED @25;
-    brakeUnavailableDEPRECATED @2;
-    plannerErrorDEPRECATED @32;
-    gpsMalfunctionDEPRECATED @94;
   }
 }
 
@@ -153,17 +137,10 @@ struct CarEvent @0x9b1657f34caf3ad3 {
 struct CarState {
   events @13 :List(CarEvent);
 
-  # CAN health
-  canValid @26 :Bool;       # invalid counter/checksums
-  canTimeout @40 :Bool;     # CAN bus dropped out
-  canErrorCounter @48 :UInt32;
-
   # car speed
-  vEgo @1 :Float32;          # best estimate of speed
-  aEgo @16 :Float32;         # best estimate of acceleration
-  vEgoRaw @17 :Float32;      # unfiltered speed from CAN sensors
-  vEgoCluster @44 :Float32;  # best estimate of speed shown on car's instrument cluster, used for UI
-
+  vEgo @1 :Float32;         # best estimate of speed
+  aEgo @16 :Float32;        # best estimate of acceleration
+  vEgoRaw @17 :Float32;     # unfiltered speed from CAN sensors
   yawRate @22 :Float32;     # best estimate of yaw rate
   standstill @18 :Bool;
   wheelSpeeds @2 :WheelSpeeds;
@@ -172,13 +149,9 @@ struct CarState {
   gas @3 :Float32;        # this is user pedal only
   gasPressed @4 :Bool;    # this is user pedal only
 
-  engineRpm @46 :Float32;
-
   # brake pedal, 0.0-1.0
   brake @5 :Float32;      # this is user pedal only
   brakePressed @6 :Bool;  # this is user pedal only
-  regenBraking @45 :Bool; # this is user pedal only
-  parkingBrake @39 :Bool;
   brakeHoldActive @38 :Bool;
 
   # steering wheel
@@ -188,13 +161,12 @@ struct CarState {
   steeringTorque @8 :Float32;      # TODO: standardize units
   steeringTorqueEps @27 :Float32;  # TODO: standardize units
   steeringPressed @9 :Bool;        # if the user is using the steering wheel
-  steerFaultTemporary @35 :Bool;   # temporary EPS fault
-  steerFaultPermanent @36 :Bool;   # permanent EPS fault
+  steeringRateLimited @29 :Bool;   # if the torque is limited by the rate limiter
+  steerWarning @35 :Bool;          # temporary steer unavailble
+  steerError @36 :Bool;            # permanent steer error
   stockAeb @30 :Bool;
   stockFcw @31 :Bool;
   espDisabled @32 :Bool;
-  accFaulted @42 :Bool;
-  carFaultedNonCritical @47 :Bool;  # some ECU is faulted, but car remains controllable
 
   # cruise state
   cruiseState @10 :CruiseState;
@@ -211,20 +183,18 @@ struct CarState {
   # lock info
   doorOpen @24 :Bool;
   seatbeltUnlatched @25 :Bool;
+  canValid @26 :Bool;
 
   # clutch (manual transmission only)
   clutchPressed @28 :Bool;
 
+  # which packets this state came from
+  canMonoTimes @12: List(UInt64);
+
   # blindspot sensors
   leftBlindspot @33 :Bool; # Is there something blocking the left lane change
   rightBlindspot @34 :Bool; # Is there something blocking the right lane change
 
-  fuelGauge @41 :Float32; # battery or fuel tank level from 0.0 to 1.0
-  charging @43 :Bool;
-
-  # process meta
-  cumLagMs @50 :Float32;
-
   struct WheelSpeeds {
     # optional wheel speeds
     fl @0 :Float32;
@@ -236,7 +206,6 @@ struct CarState {
   struct CruiseState {
     enabled @0 :Bool;
     speed @1 :Float32;
-    speedCluster @6 :Float32;  # Set speed as shown on instrument cluster
     available @2 :Bool;
     speedOffset @3 :Float32;
     standstill @4 :Bool;
@@ -277,12 +246,8 @@ struct CarState {
     }
   }
 
-  # deprecated
   errorsDEPRECATED @0 :List(CarEvent.EventName);
   brakeLightsDEPRECATED @19 :Bool;
-  steeringRateLimitedDEPRECATED @29 :Bool;
-  canMonoTimesDEPRECATED @12: List(UInt64);
-  canRcvTimeoutDEPRECATED @49 :Bool;
 }
 
 # ******* radar state @ 20hz *******
@@ -291,6 +256,9 @@ struct RadarData @0x888ad6581cf0aacb {
   errors @0 :List(Error);
   points @1 :List(RadarPoint);
 
+  # which packets this state came from
+  canMonoTimes @2 :List(UInt64);
+
   enum Error {
     canError @0;
     fault @1;
@@ -314,9 +282,6 @@ struct RadarData @0x888ad6581cf0aacb {
     # some radars flag measurements VS estimates
     measured @6 :Bool;
   }
-
-  # deprecated
-  canMonoTimesDEPRECATED @2 :List(UInt64);
 }
 
 # ******* car controls @ 100hz *******
@@ -324,20 +289,18 @@ struct RadarData @0x888ad6581cf0aacb {
 struct CarControl {
   # must be true for any actuator commands to work
   enabled @0 :Bool;
-  latActive @11: Bool;
-  longActive @12: Bool;
+  active @7 :Bool;
 
   # Actuator commands as computed by controlsd
   actuators @6 :Actuators;
 
-  # moved to CarOutput
-  actuatorsOutputDEPRECATED @10 :Actuators;
+  # Any car specific rate limits or quirks applied by
+  # the CarController are reflected in actuatorsOutput
+  # and matches what is sent to the car
+  actuatorsOutput @10 :Actuators;
 
-  leftBlinker @15: Bool;
-  rightBlinker @16: Bool;
-
-  orientationNED @13 :List(Float32);
-  angularVelocity @14 :List(Float32);
+  roll @8 :Float32;
+  pitch @9 :Float32;
 
   cruiseControl @4 :CruiseControl;
   hudControl @5 :HUDControl;
@@ -348,12 +311,8 @@ struct CarControl {
     brake @1: Float32;
     # range from -1.0 - 1.0
     steer @2: Float32;
-    # value sent over can to the car
-    steerOutputCan @8: Float32;
     steeringAngleDeg @3: Float32;
 
-    curvature @7: Float32;
-
     speed @6: Float32; # m/s
     accel @4: Float32; # m/s^2
     longControlState @5: LongControlState;
@@ -362,16 +321,17 @@ struct CarControl {
       off @0;
       pid @1;
       stopping @2;
-      starting @3;
+
+      startingDEPRECATED @3;
     }
+
   }
 
   struct CruiseControl {
     cancel @0: Bool;
-    resume @1: Bool;
-    override @4: Bool;
-    speedOverrideDEPRECATED @2: Float32;
-    accelOverrideDEPRECATED @3: Float32;
+    override @1: Bool;
+    speedOverride @2: Float32;
+    accelOverride @3: Float32;
   }
 
   struct HUDControl {
@@ -385,7 +345,6 @@ struct CarControl {
     leftLaneVisible @7: Bool;
     rightLaneDepart @8: Bool;
     leftLaneDepart @9: Bool;
-    leadDistanceBars @10: Int8;  # 1-3: 1 is closest, 3 is farthest. some ports may utilize 2-4 bars instead
 
     enum VisualAlert {
       # these are the choices from the Honda
@@ -419,16 +378,6 @@ struct CarControl {
   gasDEPRECATED @1 :Float32;
   brakeDEPRECATED @2 :Float32;
   steeringTorqueDEPRECATED @3 :Float32;
-  activeDEPRECATED @7 :Bool;
-  rollDEPRECATED @8 :Float32;
-  pitchDEPRECATED @9 :Float32;
-}
-
-struct CarOutput {
-  # Any car specific rate limits or quirks applied by
-  # the CarController are reflected in actuatorsOutput
-  # and matches what is sent to the car
-  actuatorsOutput @0 :CarControl.Actuators;
 }
 
 # ****** car param ******
@@ -438,22 +387,25 @@ struct CarParams {
   carFingerprint @1 :Text;
   fuzzyFingerprint @55 :Bool;
 
-  notCar @66 :Bool;  # flag for non-car robotics platforms
-
+  enableGasInterceptor @2 :Bool;
   pcmCruise @3 :Bool;        # is openpilot's state tied to the PCM's cruise state?
   enableDsu @5 :Bool;        # driving support unit
+  enableApgs @6 :Bool;       # advanced parking guidance system
   enableBsm @56 :Bool;       # blind spot monitoring
   flags @64 :UInt32;         # flags for car specific quirks
-  experimentalLongitudinalAvailable @71 :Bool;
 
   minEnableSpeed @7 :Float32;
   minSteerSpeed @8 :Float32;
+  maxSteeringAngleDeg @54 :Float32;
   safetyConfigs @62 :List(SafetyConfig);
-  alternativeExperience @65 :Int16;      # panda flag for features like no disengage on gas
+  unsafeMode @65 :Int16;
 
-  # Car docs fields
-  maxLateralAccel @68 :Float32;
-  autoResumeSng @69 :Bool;               # describes whether car can resume from a stop automatically
+  steerMaxBP @11 :List(Float32);
+  steerMaxV @12 :List(Float32);
+  gasMaxBPDEPRECATED @13 :List(Float32);
+  gasMaxVDEPRECATED @14 :List(Float32);
+  brakeMaxBPDEPRECATED @15 :List(Float32);
+  brakeMaxVDEPRECATED @16 :List(Float32);
 
   # things about the car in the manual
   mass @17 :Float32;            # [kg] curb weight: all fluids no cargo
@@ -464,7 +416,6 @@ struct CarParams {
 
   # things we can derive
   rotationalInertia @22 :Float32;    # [kg*m2] body rotational inertia
-  tireStiffnessFactor @72 :Float32;  # scaling factor used in calculating tireStiffness[Front,Rear]
   tireStiffnessFront @23 :Float32;   # [N/rad] front tire coeff of stiff
   tireStiffnessRear @24 :Float32;    # [N/rad] rear tire coeff of stiff
 
@@ -472,9 +423,8 @@ struct CarParams {
   lateralParams @48 :LateralParams;
   lateralTuning :union {
     pid @26 :LateralPIDTuning;
-    indiDEPRECATED @27 :LateralINDITuning;
-    lqrDEPRECATED @40 :LateralLQRTuning;
-    torque @67 :LateralTorqueTuning;
+    indi @27 :LateralINDITuning;
+    lqr @40 :LateralLQRTuning;
   }
 
   steerLimitAlert @28 :Bool;
@@ -482,20 +432,20 @@ struct CarParams {
 
   vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state
   vEgoStarting @59 :Float32; # Speed at which the car goes into starting state
-  stoppingControl @31 :Bool; # Does the car allow full control even at lows speeds when stopping
+  directAccelControl @30 :Bool; # Does the car have direct accel control or just gas/brake
+  stoppingControl @31 :Bool; # Does the car allows full control even at lows speeds when stopping
+  stopAccel @60 :Float32; # Required acceleraton to keep vehicle stationary
+  steerRateCost @33 :Float32; # Lateral MPC cost on steering rate
   steerControlType @34 :SteerControlType;
-  radarUnavailable @35 :Bool; # True when radar objects aren't visible on CAN or aren't parsed out
-  stopAccel @60 :Float32; # Required acceleration to keep vehicle stationary
+  radarOffCan @35 :Bool; # True when radar objects aren't visible on CAN
   stoppingDecelRate @52 :Float32; # m/s^2/s while trying to stop
-  startAccel @32 :Float32; # Required acceleration to get car moving
-  startingState @70 :Bool; # Does this car make use of special starting state
 
   steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds
-  longitudinalActuatorDelay @58 :Float32; # Gas/Brake actuator delay in seconds
+  longitudinalActuatorDelayLowerBound @61 :Float32; # Gas/Brake actuator delay in seconds, lower bound
+  longitudinalActuatorDelayUpperBound @58 :Float32; # Gas/Brake actuator delay in seconds, upper bound
   openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control?
   carVin @38 :Text; # VIN number queried during fingerprinting
   dashcamOnly @41: Bool;
-  passive @73: Bool;   # is openpilot in control?
   transmissionType @43 :TransmissionType;
   carFw @44 :List(CarFw);
 
@@ -505,14 +455,9 @@ struct CarParams {
 
   wheelSpeedFactor @63 :Float32; # Multiplier on wheels speeds to computer actual speeds
 
-  secOcRequired @74 :Bool;  # Car requires SecOC message authentication to operate
-  secOcKeyAvailable @75 :Bool;  # Stored SecOC key loaded from params
-
   struct SafetyConfig {
     safetyModel @0 :SafetyModel;
-    safetyParam @3 :UInt16;
-    safetyParamDEPRECATED @1 :Int16;
-    safetyParam2DEPRECATED @2 :UInt32;
+    safetyParam @1 :Int16;
   }
 
   struct LateralParams {
@@ -528,23 +473,11 @@ struct CarParams {
     kf @4 :Float32;
   }
 
-  struct LateralTorqueTuning {
-    useSteeringAngle @0 :Bool;
-    kp @1 :Float32;
-    ki @2 :Float32;
-    friction @3 :Float32;
-    kf @4 :Float32;
-    steeringAngleDeadzoneDeg @5 :Float32;
-    latAccelFactor @6 :Float32;
-    latAccelOffset @7 :Float32;
-  }
-
   struct LongitudinalPIDTuning {
     kpBP @0 :List(Float32);
     kpV @1 :List(Float32);
     kiBP @2 :List(Float32);
     kiV @3 :List(Float32);
-    kf @6 :Float32;
     deadzoneBP @4 :List(Float32);
     deadzoneV @5 :List(Float32);
   }
@@ -602,23 +535,15 @@ struct CarParams {
     noOutput @19;  # like silent but without silent CAN TXs
     hondaBosch @20;
     volkswagenPq @21;
-    subaruPreglobal @22;  # pre-Global platform
+    subaruLegacy @22;  # pre-Global platform
     hyundaiLegacy @23;
     hyundaiCommunity @24;
-    volkswagenMlb @25;
-    hongqi @26;
-    body @27;
-    hyundaiCanfd @28;
-    volkswagenMqbEvo @29;
-    chryslerCusw @30;
-    psa @31;
+    stellantis @25;
   }
 
   enum SteerControlType {
     torque @0;
     angle @1;
-
-    curvatureDEPRECATED @2;
   }
 
   enum TransmissionType {
@@ -632,47 +557,32 @@ struct CarParams {
   struct CarFw {
     ecu @0 :Ecu;
     fwVersion @1 :Data;
-    address @2 :UInt32;
-    subAddress @3 :UInt8;
-    responseAddress @4 :UInt32;
-    request @5 :List(Data);
-    brand @6 :Text;
-    bus @7 :UInt8;
-    logging @8 :Bool;
-    obdMultiplexing @9 :Bool;
+    address @2: UInt32;
+    subAddress @3: UInt8;
   }
 
   enum Ecu {
     eps @0;
-    abs @1;
+    esp @1;
     fwdRadar @2;
     fwdCamera @3;
     engine @4;
     unknown @5;
     transmission @8; # Transmission Control Module
-    hybrid @18; # hybrid control unit, e.g. Chrysler's HCP, Honda's IMA Control Unit, Toyota's hybrid control computer
     srs @9; # airbag
     gateway @10; # can gateway
     hud @11; # heads up display
     combinationMeter @12; # instrument cluster
-    electricBrakeBooster @15;
-    shiftByWire @16;
-    adas @19;
-    cornerRadar @21;
-    hvac @20;
-    parkingAdas @7;  # parking assist system ECU, e.g. Toyota's IPAS, Hyundai's RSPA, etc.
-    epb @22;  # electronic parking brake
-    telematics @23;
-    body @24;  # body control module
 
     # Toyota only
     dsu @6;
+    apgs @7;
 
     # Honda only
     vsa @13; # Vehicle Stability Assist
     programmedFuelInjection @14;
-
-    debug @17;
+    electricBrakeBooster @15;
+    shiftByWire @16;
   }
 
   enum FingerprintSource {
@@ -686,25 +596,14 @@ struct CarParams {
     gateway @1;    # Integration at vehicle's CAN gateway
   }
 
-  enableGasInterceptor @2 :Bool;
   enableCameraDEPRECATED @4 :Bool;
-  enableApgsDEPRECATED @6 :Bool;
-  steerRateCostDEPRECATED @33 :Float32;
   isPandaBlackDEPRECATED @39 :Bool;
   hasStockCameraDEPRECATED @57 :Bool;
   safetyParamDEPRECATED @10 :Int16;
   safetyModelDEPRECATED @9 :SafetyModel;
   safetyModelPassiveDEPRECATED @42 :SafetyModel = silent;
   minSpeedCanDEPRECATED @51 :Float32;
+  startAccelDEPRECATED @32 :Float32;
   communityFeatureDEPRECATED @46: Bool;
   startingAccelRateDEPRECATED @53 :Float32;
-  steerMaxBPDEPRECATED @11 :List(Float32);
-  steerMaxVDEPRECATED @12 :List(Float32);
-  gasMaxBPDEPRECATED @13 :List(Float32);
-  gasMaxVDEPRECATED @14 :List(Float32);
-  brakeMaxBPDEPRECATED @15 :List(Float32);
-  brakeMaxVDEPRECATED @16 :List(Float32);
-  directAccelControlDEPRECATED @30 :Bool;
-  maxSteeringAngleDegDEPRECATED @54 :Float32;
-  longitudinalActuatorDelayLowerBoundDEPRECATEDDEPRECATED @61 :Float32;
 }

cereal/custom.capnp

@@ -1,249 +0,0 @@
-using Cxx = import "./include/c++.capnp";
-$Cxx.namespace("cereal");
-
-using Car = import "car.capnp";
-
-@0xb526ba661d550a59;
-
-# custom.capnp: a home for empty structs reserved for custom forks
-# These structs are guaranteed to remain reserved and empty in mainline
-# cereal, so use these if you want custom events in your fork.
-
-# you can rename the struct, but don't change the identifier
-struct FrogPilotCarControl {
-  hudControl @0 :HUDControl;
-
-  struct HUDControl {
-    audibleAlert @0 :AudibleAlert;
-
-    enum AudibleAlert {
-      none @0;
-
-      engage @1;
-      disengage @2;
-      refuse @3;
-
-      warningSoft @4;
-      warningImmediate @5;
-
-      prompt @6;
-      promptRepeat @7;
-      promptDistracted @8;
-
-      # Random Events
-      angry @9;
-      continued @10;
-      dejaVu @11;
-      doc @12;
-      fart @13;
-      firefox @14;
-      goat @15;
-      hal9000 @16;
-      mail @17;
-      nessie @18;
-      noice @19;
-      startup @20;
-      thisIsFine @21;
-      uwu @22;
-    }
-  }
-}
-
-struct FrogPilotCarEvent @0x81c2f05a394cf4af {
-  name @0 :EventName;
-
-  enable @1 :Bool;
-  noEntry @2 :Bool;
-  warning @3 :Bool;
-  userDisable @4 :Bool;
-  softDisable @5 :Bool;
-  immediateDisable @6 :Bool;
-  preEnable @7 :Bool;
-  permanent @8 :Bool;
-  overrideLateral @10 :Bool;
-  overrideLongitudinal @9 :Bool;
-
-  enum EventName @0xaedffd8f31e7b55d {
-    blockUser @0;
-    customStartupAlert @1;
-    forcingStop @2;
-    goatSteerSaturated @3;
-    greenLight @4;
-    holidayActive @5;
-    laneChangeBlockedLoud @6;
-    leadDeparting @7;
-    noLaneAvailable @8;
-    openpilotCrashed @9;
-    pedalInterceptorNoBrake @10;
-    speedLimitChanged @11;
-    torqueNNLoad @12;
-    trafficModeActive @13;
-    trafficModeInactive @14;
-    turningLeft @15;
-    turningRight @16;
-
-    # Random Events
-    accel30 @17;
-    accel35 @18;
-    accel40 @19;
-    dejaVuCurve @20;
-    firefoxSteerSaturated @21;
-    hal9000 @22;
-    openpilotCrashedRandomEvent @23;
-    thisIsFineSteerSaturated @24;
-    toBeContinued @25;
-    vCruise69 @26;
-    yourFrogTriedToKillMe @27;
-    youveGotMail @28;
-  }
-}
-
-struct FrogPilotCarParams @0xf35cc4560bbf6ec2 {
-  canUsePedal @0 :Bool;
-  canUseSDSU @1 :Bool;
-  fpFlags @2 :UInt32;
-  isHDA2 @3 :Bool;
-  openpilotLongitudinalControlDisabled @4 :Bool;
-  safetyConfigs @5 :List(SafetyConfig);
-
-  lateralTuning :union {
-    pid @6 :Car.CarParams.LateralPIDTuning;
-    torque @7 :Car.CarParams.LateralTorqueTuning;
-  }
-
-  struct SafetyConfig {
-    safetyParam @0 :UInt16;
-  }
-}
-
-struct FrogPilotCarState @0xda96579883444c35 {
-  accelPressed @0 :Bool;
-  alwaysOnLateralAllowed @1 :Bool;
-  alwaysOnLateralEnabled @2 :Bool;
-  brakeLights @3 :Bool;
-  dashboardSpeedLimit @4 :Float32;
-  decelPressed @5 :Bool;
-  distancePressed @6 :Bool;
-  distanceLongPressed @7 :Bool;
-  distanceVeryLongPressed @8 :Bool;
-  ecoGear @9 :Bool;
-  forceCoast @10 :Bool;
-  pauseLateral @11 :Bool;
-  pauseLongitudinal @12 :Bool;
-  sportGear @13 :Bool;
-  trafficModeEnabled @14 :Bool;
-
-  struct ButtonEvent {
-    enum Type {
-      lkas @0;
-    }
-  }
-}
-
-struct FrogPilotControlsState @0x80ae746ee2596b11 {
-  alertStatus @0 :AlertStatus;
-  alertText1 @1 :Text;
-  alertText2 @2 :Text;
-  alertSize @3 :AlertSize;
-  alertBlinkingRate @4 :Float32;
-  alertType @5 :Text;
-  alertSound @6 :Car.CarControl.HUDControl.AudibleAlert;
-
-  enum AlertSize {
-    none @0;    # don't display the alert
-    small @1;   # small box
-    mid @2;     # mid screen
-    full @3;    # full screen
-  }
-
-  enum AlertStatus {
-    normal @0;       # low priority alert for user's convenience
-    userPrompt @1;   # mid priority alert that might require user intervention
-    critical @2;     # high priority alert that needs immediate user intervention
-    frogpilot @3;    # FrogPilot startup alert
-  }
-}
-
-struct FrogPilotDeviceState @0xa5cd762cd951a455 {
-  freeSpace @0 :Int16;
-  usedSpace @1 :Int16;
-}
-
-struct FrogPilotModelDataV2 @0xf98d843bfd7004a3 {
-  turnDirection @0 :TurnDirection;
-
-  enum TurnDirection {
-    none @0;
-    turnLeft @1;
-    turnRight @2;
-  }
-}
-
-struct FrogPilotNavigation @0xf416ec09499d9d19 {
-  approachingIntersection @0 :Bool;
-  approachingTurn @1 :Bool;
-  navigationSpeedLimit @2 :Float32;
-}
-
-struct FrogPilotPlan @0xa1680744031fdb2d {
-  accelerationJerk @0 :Float32;
-  accelerationJerkStock @1 :Float32;
-  cscControllingSpeed @2 :Bool;
-  cscSpeed @3 :Float32;
-  cscTraining @4 :Bool;
-  dangerJerk @5 :Float32;
-  desiredFollowDistance @6 :Int64;
-  experimentalMode @7 :Bool;
-  forcingStop @8 :Bool;
-  forcingStopLength @9 :Float32;
-  frogpilotEvents @10 :List(FrogPilotCarEvent);
-  increasedStoppedDistance @11 :Float32;
-  lateralCheck @12 :Bool;
-  laneWidthLeft @13 :Float32;
-  laneWidthRight @14 :Float32;
-  maxAcceleration @15 :Float32;
-  minAcceleration @16 :Float32;
-  redLight @17 :Bool;
-  roadCurvature @18 :Float32;
-  slcMapSpeedLimit @19 :Float32;
-  slcMapboxSpeedLimit @20 :Float32;
-  slcNextSpeedLimit @21 :Float32;
-  slcOverriddenSpeed @22 :Float32;
-  slcSpeedLimit @23 :Float32;
-  slcSpeedLimitOffset @24 :Float32;
-  slcSpeedLimitSource @25 :Text;
-  speedJerk @26 :Float32;
-  speedJerkStock @27 :Float32;
-  speedLimitChanged @28 :Bool;
-  tFollow @29 :Float32;
-  themeUpdated @30 :Bool;
-  togglesUpdated @31 :Bool;
-  trackingLead @32 :Bool;
-  unconfirmedSlcSpeedLimit @33 :Float32;
-  vCruise @34 :Float32;
-}
-
-struct FrogPilotRadarState @0xcb9fd56c7057593a {
-  leadLeft @0 :LeadData;
-  leadRight @1 :LeadData;
-
-  struct LeadData {
-    dRel @0 :Float32;
-    yRel @1 :Float32;
-    vRel @2 :Float32;
-    aRel @3 :Float32;
-    vLead @4 :Float32;
-    dPath @6 :Float32;
-    vLat @7 :Float32;
-    vLeadK @8 :Float32;
-    aLeadK @9 :Float32;
-    fcw @10 :Bool;
-    status @11 :Bool;
-    aLeadTau @12 :Float32;
-    modelProb @13 :Float32;
-    radar @14 :Bool;
-    radarTrackId @15 :Int32 = -1;
-
-    aLeadDEPRECATED @5 :Float32;
-  }
-}

cereal/legacy.capnp

@@ -559,6 +559,293 @@ struct AndroidGnss @0xdfdf30d03fc485bd {
   }
 }
 
+struct QcomGnss @0xde94674b07ae51c1 {
+  logTs @0 :UInt64;
+  union {
+    measurementReport @1 :MeasurementReport;
+    clockReport @2 :ClockReport;
+    drMeasurementReport @3 :DrMeasurementReport;
+    drSvPoly @4 :DrSvPolyReport;
+    rawLog @5 :Data;
+  }
+
+  enum MeasurementSource @0xd71a12b6faada7ee {
+    gps @0;
+    glonass @1;
+    beidou @2;
+  }
+
+  enum SVObservationState @0xe81e829a0d6c83e9 {
+    idle @0;
+    search @1;
+    searchVerify @2;
+    bitEdge @3;
+    trackVerify @4;
+    track @5;
+    restart @6;
+    dpo @7;
+    glo10msBe @8;
+    glo10msAt @9;
+  }
+
+  struct MeasurementStatus @0xe501010e1bcae83b {
+    subMillisecondIsValid @0 :Bool;
+    subBitTimeIsKnown @1 :Bool;
+    satelliteTimeIsKnown @2 :Bool;
+    bitEdgeConfirmedFromSignal @3 :Bool;
+    measuredVelocity @4 :Bool;
+    fineOrCoarseVelocity @5 :Bool;
+    lockPointValid @6 :Bool;
+    lockPointPositive @7 :Bool;
+    lastUpdateFromDifference @8 :Bool;
+    lastUpdateFromVelocityDifference @9 :Bool;
+    strongIndicationOfCrossCorelation @10 :Bool;
+    tentativeMeasurement @11 :Bool;
+    measurementNotUsable @12 :Bool;
+    sirCheckIsNeeded @13 :Bool;
+    probationMode @14 :Bool;
+
+    glonassMeanderBitEdgeValid @15 :Bool;
+    glonassTimeMarkValid @16 :Bool;
+
+    gpsRoundRobinRxDiversity @17 :Bool;
+    gpsRxDiversity @18 :Bool;
+    gpsLowBandwidthRxDiversityCombined @19 :Bool;
+    gpsHighBandwidthNu4 @20 :Bool;
+    gpsHighBandwidthNu8 @21 :Bool;
+    gpsHighBandwidthUniform @22 :Bool;
+    multipathIndicator @23 :Bool;
+
+    imdJammingIndicator @24 :Bool;
+    lteB13TxJammingIndicator @25 :Bool;
+    freshMeasurementIndicator @26 :Bool;
+
+    multipathEstimateIsValid @27 :Bool;
+    directionIsValid @28 :Bool;
+  }
+
+  struct MeasurementReport @0xf580d7d86b7b8692 {
+    source @0 :MeasurementSource;
+
+    fCount @1 :UInt32;
+
+    gpsWeek @2 :UInt16;
+    glonassCycleNumber @3 :UInt8;
+    glonassNumberOfDays @4 :UInt16;
+
+    milliseconds @5 :UInt32;
+    timeBias @6 :Float32;
+    clockTimeUncertainty @7 :Float32;
+    clockFrequencyBias @8 :Float32;
+    clockFrequencyUncertainty @9 :Float32;
+
+    sv @10 :List(SV);
+
+    struct SV @0xf10c595ae7bb2c27 {
+      svId @0 :UInt8;
+      observationState @2 :SVObservationState;
+      observations @3 :UInt8;
+      goodObservations @4 :UInt8;
+      gpsParityErrorCount @5 :UInt16;
+      glonassFrequencyIndex @1 :Int8;
+      glonassHemmingErrorCount @6 :UInt8;
+      filterStages @7 :UInt8;
+      carrierNoise @8 :UInt16;
+      latency @9 :Int16;
+      predetectInterval @10 :UInt8;
+      postdetections @11 :UInt16;
+
+      unfilteredMeasurementIntegral @12 :UInt32;
+      unfilteredMeasurementFraction @13 :Float32;
+      unfilteredTimeUncertainty @14 :Float32;
+      unfilteredSpeed @15 :Float32;
+      unfilteredSpeedUncertainty @16 :Float32;
+      measurementStatus @17 :MeasurementStatus;
+      multipathEstimate @18 :UInt32;
+      azimuth @19 :Float32;
+      elevation @20 :Float32;
+      carrierPhaseCyclesIntegral @21 :Int32;
+      carrierPhaseCyclesFraction @22 :UInt16;
+      fineSpeed @23 :Float32;
+      fineSpeedUncertainty @24 :Float32;
+      cycleSlipCount @25 :UInt8;
+    }
+
+  }
+
+  struct ClockReport @0xca965e4add8f4f0b {
+    hasFCount @0 :Bool;
+    fCount @1 :UInt32;
+
+    hasGpsWeek @2 :Bool;
+    gpsWeek @3 :UInt16;
+    hasGpsMilliseconds @4 :Bool;
+    gpsMilliseconds @5 :UInt32;
+    gpsTimeBias @6 :Float32;
+    gpsClockTimeUncertainty @7 :Float32;
+    gpsClockSource @8 :UInt8;
+
+    hasGlonassYear @9 :Bool;
+    glonassYear @10 :UInt8;
+    hasGlonassDay @11 :Bool;
+    glonassDay @12 :UInt16;
+    hasGlonassMilliseconds @13 :Bool;
+    glonassMilliseconds @14 :UInt32;
+    glonassTimeBias @15 :Float32;
+    glonassClockTimeUncertainty @16 :Float32;
+    glonassClockSource @17 :UInt8;
+
+    bdsWeek @18 :UInt16;
+    bdsMilliseconds @19 :UInt32;
+    bdsTimeBias @20 :Float32;
+    bdsClockTimeUncertainty @21 :Float32;
+    bdsClockSource @22 :UInt8;
+
+    galWeek @23 :UInt16;
+    galMilliseconds @24 :UInt32;
+    galTimeBias @25 :Float32;
+    galClockTimeUncertainty @26 :Float32;
+    galClockSource @27 :UInt8;
+
+    clockFrequencyBias @28 :Float32;
+    clockFrequencyUncertainty @29 :Float32;
+    frequencySource @30 :UInt8;
+    gpsLeapSeconds @31 :UInt8;
+    gpsLeapSecondsUncertainty @32 :UInt8;
+    gpsLeapSecondsSource @33 :UInt8;
+
+    gpsToGlonassTimeBiasMilliseconds @34 :Float32;
+    gpsToGlonassTimeBiasMillisecondsUncertainty @35 :Float32;
+    gpsToBdsTimeBiasMilliseconds @36 :Float32;
+    gpsToBdsTimeBiasMillisecondsUncertainty @37 :Float32;
+    bdsToGloTimeBiasMilliseconds @38 :Float32;
+    bdsToGloTimeBiasMillisecondsUncertainty @39 :Float32;
+    gpsToGalTimeBiasMilliseconds @40 :Float32;
+    gpsToGalTimeBiasMillisecondsUncertainty @41 :Float32;
+    galToGloTimeBiasMilliseconds @42 :Float32;
+    galToGloTimeBiasMillisecondsUncertainty @43 :Float32;
+    galToBdsTimeBiasMilliseconds @44 :Float32;
+    galToBdsTimeBiasMillisecondsUncertainty @45 :Float32;
+
+    hasRtcTime @46 :Bool;
+    systemRtcTime @47 :UInt32;
+    fCountOffset @48 :UInt32;
+    lpmRtcCount @49 :UInt32;
+    clockResets @50 :UInt32;
+  }
+
+  struct DrMeasurementReport @0x8053c39445c6c75c {
+
+    reason @0 :UInt8;
+    seqNum @1 :UInt8;
+    seqMax @2 :UInt8;
+    rfLoss @3 :UInt16;
+
+    systemRtcValid @4 :Bool;
+    fCount @5 :UInt32;
+    clockResets @6 :UInt32;
+    systemRtcTime @7 :UInt64;
+
+    gpsLeapSeconds @8 :UInt8;
+    gpsLeapSecondsUncertainty @9 :UInt8;
+    gpsToGlonassTimeBiasMilliseconds @10 :Float32;
+    gpsToGlonassTimeBiasMillisecondsUncertainty @11 :Float32;
+
+    gpsWeek @12 :UInt16;
+    gpsMilliseconds @13 :UInt32;
+    gpsTimeBiasMs @14 :UInt32;
+    gpsClockTimeUncertaintyMs @15 :UInt32;
+    gpsClockSource @16 :UInt8;
+
+    glonassClockSource @17 :UInt8;
+    glonassYear @18 :UInt8;
+    glonassDay @19 :UInt16;
+    glonassMilliseconds @20 :UInt32;
+    glonassTimeBias @21 :Float32;
+    glonassClockTimeUncertainty @22 :Float32;
+
+    clockFrequencyBias @23 :Float32;
+    clockFrequencyUncertainty @24 :Float32;
+    frequencySource @25 :UInt8;
+
+    source @26 :MeasurementSource;
+
+    sv @27 :List(SV);
+
+    struct SV @0xf08b81df8cbf459c {
+      svId @0 :UInt8;
+      glonassFrequencyIndex @1 :Int8;
+      observationState @2 :SVObservationState;
+      observations @3 :UInt8;
+      goodObservations @4 :UInt8;
+      filterStages @5 :UInt8;
+      predetectInterval @6 :UInt8;
+      cycleSlipCount @7 :UInt8;
+      postdetections @8 :UInt16;
+
+      measurementStatus @9 :MeasurementStatus;
+
+      carrierNoise @10 :UInt16;
+      rfLoss @11 :UInt16;
+      latency @12 :Int16;
+
+      filteredMeasurementFraction @13 :Float32;
+      filteredMeasurementIntegral @14 :UInt32;
+      filteredTimeUncertainty @15 :Float32;
+      filteredSpeed @16 :Float32;
+      filteredSpeedUncertainty @17 :Float32;
+
+      unfilteredMeasurementFraction @18 :Float32;
+      unfilteredMeasurementIntegral @19 :UInt32;
+      unfilteredTimeUncertainty @20 :Float32;
+      unfilteredSpeed @21 :Float32;
+      unfilteredSpeedUncertainty @22 :Float32;
+
+      multipathEstimate @23 :UInt32;
+      azimuth @24 :Float32;
+      elevation @25 :Float32;
+      dopplerAcceleration @26 :Float32;
+      fineSpeed @27 :Float32;
+      fineSpeedUncertainty @28 :Float32;
+
+      carrierPhase @29 :Float64;
+      fCount @30 :UInt32;
+
+      parityErrorCount @31 :UInt16;
+      goodParity @32 :Bool;
+    }
+  }
+
+  struct DrSvPolyReport @0xb1fb80811a673270 {
+    svId @0 :UInt16;
+    frequencyIndex @1 :Int8;
+
+    hasPosition @2 :Bool;
+    hasIono @3 :Bool;
+    hasTropo @4 :Bool;
+    hasElevation @5 :Bool;
+    polyFromXtra @6 :Bool;
+    hasSbasIono @7 :Bool;
+
+    iode @8 :UInt16;
+    t0 @9 :Float64;
+    xyz0 @10 :List(Float64);
+    xyzN @11 :List(Float64);
+    other @12 :List(Float32);
+
+    positionUncertainty @13 :Float32;
+    ionoDelay @14 :Float32;
+    ionoDot @15 :Float32;
+    sbasIonoDelay @16 :Float32;
+    sbasIonoDot @17 :Float32;
+    tropoDelay @18 :Float32;
+    elevation @19 :Float32;
+    elevationDot @20 :Float32;
+    elevationUncertainty @21 :Float32;
+    velocityCoeff @22 :List(Float64);
+  }
+}
+
 struct LidarPts @0xe3d6685d4e9d8f7a {
   r @0 :List(UInt16);        # uint16   m*500.0
   theta @1 :List(UInt16);    # uint16 deg*100.0

cereal/logger/logger.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#ifdef SWAGLOG
+#include "selfdrive/common/swaglog.h"
+#else
+
+#define CLOUDLOG_DEBUG 10
+#define CLOUDLOG_INFO 20
+#define CLOUDLOG_WARNING 30
+#define CLOUDLOG_ERROR 40
+#define CLOUDLOG_CRITICAL 50
+
+#define cloudlog(lvl, fmt, ...) printf(fmt "\n", ## __VA_ARGS__)
+
+#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
+#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
+#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
+#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
+
+#endif

cereal/maptile.capnp

@@ -1,49 +0,0 @@
-using Cxx = import "./include/c++.capnp";
-$Cxx.namespace("cereal");
-
-@0xa086df597ef5d7a0;
-
-# Geometry
-struct Point {
-  x @0: Float64;
-  y @1: Float64;
-  z @2: Float64;
-}
-
-struct PolyLine {
-  points @0: List(Point);
-}
-
-# Map features
-struct Lane {
-  id @0 :Text;
-
-  leftBoundary @1 :LaneBoundary;
-  rightBoundary @2 :LaneBoundary;
-
-  leftAdjacentId @3 :Text;
-  rightAdjacentId @4 :Text;
-
-  inboundIds @5 :List(Text);
-  outboundIds @6 :List(Text);
-
-  struct LaneBoundary {
-    polyLine @0 :PolyLine;
-    startHeading @1 :Float32; # WRT north
-  }
-}
-
-# Map tiles
-struct TileSummary {
-  version @0 :Text;
-  updatedAt @1 :UInt64; # Millis since epoch
-
-  level @2 :UInt8;
-  x @3 :UInt16;
-  y @4 :UInt16;
-}
-
-struct MapTile {
-  summary @0 :TileSummary;
-  lanes @1 :List(Lane);
-}

cereal/messaging/__init__.py

@@ -1,34 +1,39 @@
-# must be built with scons
-from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-from msgq.ipc_pyx import MultiplePublishersError, IpcError
-from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw, context
-
+# must be build with scons
+from .messaging_pyx import Context, Poller, SubSocket, PubSocket  # pylint: disable=no-name-in-module, import-error
+from .messaging_pyx import MultiplePublishersError, MessagingError  # pylint: disable=no-name-in-module, import-error
 import os
 import capnp
-import time
 
-from typing import Optional, List, Union, Dict, Deque
+from typing import Optional, List, Union
 from collections import deque
 
 from cereal import log
-from cereal.services import SERVICE_LIST
+from cereal.services import service_list
+
+assert MultiplePublishersError
+assert MessagingError
 
 NO_TRAVERSAL_LIMIT = 2**64-1
+AVG_FREQ_HISTORY = 100
+SIMULATION = "SIMULATION" in os.environ
 
+# sec_since_boot is faster, but allow to run standalone too
+try:
+  from common.realtime import sec_since_boot
+except ImportError:
+  import time
+  sec_since_boot = time.time
+  print("Warning, using python time.time() instead of faster sec_since_boot")
+
+context = Context()
 
 def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
-  with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
-    return msg
+  return log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT)
 
-
-def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
-  args = {
-    'valid': False,
-    'logMonoTime': int(time.monotonic() * 1e9),
-    **kwargs
-  }
-  dat = log.Event.new_message(**args)
+def new_message(service: Optional[str] = None, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder:
+  dat = log.Event.new_message()
+  dat.logMonoTime = int(sec_since_boot() * 1e9)
+  dat.valid = True
   if service is not None:
     if size is None:
       dat.init(service)
@@ -36,49 +41,91 @@ def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) ->
       dat.init(service, size)
   return dat
 
+def pub_sock(endpoint: str) -> PubSocket:
+  sock = PubSocket()
+  sock.connect(context, endpoint)
+  return sock
+
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
+  sock = SubSocket()
+  sock.connect(context, endpoint, addr.encode('utf8'), conflate)
+
+  if timeout is not None:
+    sock.setTimeout(timeout)
+
+  if poller is not None:
+    poller.registerSocket(sock)
+  return sock
+
+
+def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]:
+  """Receive all message currently available on the queue"""
+  ret: List[bytes] = []
+  while 1:
+    if wait_for_one and len(ret) == 0:
+      dat = sock.receive()
+    else:
+      dat = sock.receive(non_blocking=True)
+
+    if dat is None:
+      break
+
+    ret.append(dat)
+
+  return ret
 
 def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
   """Receive all message currently available on the queue"""
-  msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
-  return [log_from_bytes(m) for m in msgs]
+  ret: List[capnp.lib.capnp._DynamicStructReader] = []
+  while 1:
+    if wait_for_one and len(ret) == 0:
+      dat = sock.receive()
+    else:
+      dat = sock.receive(non_blocking=True)
+
+    if dat is None:  # Timeout hit
+      break
+
+    dat = log_from_bytes(dat)
+    ret.append(dat)
+
+  return ret
 
 
 # TODO: print when we drop packets?
-def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+def recv_sock(sock: SubSocket, wait: bool = False) -> Union[None, capnp.lib.capnp._DynamicStructReader]:
   """Same as drain sock, but only returns latest message. Consider using conflate instead."""
   dat = None
 
   while 1:
     if wait and dat is None:
-      recv = sock.receive()
+      rcv = sock.receive()
     else:
-      recv = sock.receive(non_blocking=True)
+      rcv = sock.receive(non_blocking=True)
 
-    if recv is None:  # Timeout hit
+    if rcv is None:  # Timeout hit
       break
 
-    dat = recv
+    dat = rcv
 
   if dat is not None:
     dat = log_from_bytes(dat)
 
   return dat
 
-
-def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+def recv_one(sock: SubSocket) -> Union[None, capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive()
   if dat is not None:
     dat = log_from_bytes(dat)
   return dat
 
-
-def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+def recv_one_or_none(sock: SubSocket) -> Union[None, capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive(non_blocking=True)
   if dat is not None:
     dat = log_from_bytes(dat)
   return dat
 
-
 def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
   """Keep receiving until we get a message"""
   while True:
@@ -86,77 +133,48 @@ def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
     if dat is not None:
       return log_from_bytes(dat)
 
-
-class SubMaster:
-  def __init__(self, services: List[str], poll: Optional[str] = None,
+class SubMaster():
+  def __init__(self, services: List[str], poll: Optional[List[str]] = None,
                ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
-               ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
+               addr: str = "127.0.0.1"):
     self.frame = -1
-    self.seen = {s: False for s in services}
     self.updated = {s: False for s in services}
-    self.recv_time = {s: 0. for s in services}
-    self.recv_frame = {s: 0 for s in services}
+    self.rcv_time = {s: 0. for s in services}
+    self.rcv_frame = {s: 0 for s in services}
     self.alive = {s: False for s in services}
-    self.freq_ok = {s: False for s in services}
-    self.recv_dts: Dict[str, Deque[float]] = {}
+    self.recv_dts = {s: deque([0.0] * AVG_FREQ_HISTORY, maxlen=AVG_FREQ_HISTORY) for s in services}
     self.sock = {}
+    self.freq = {}
     self.data = {}
     self.valid = {}
     self.logMonoTime = {}
 
-    self.max_freq = {}
-    self.min_freq = {}
-
     self.poller = Poller()
-    polled_services = set([poll, ] if poll is not None else services)
-    self.non_polled_services = set(services) - polled_services
+    self.non_polled_services = [s for s in services if poll is not None and
+                                len(poll) and s not in poll]
 
     self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
     self.ignore_alive = [] if ignore_alive is None else ignore_alive
-    self.ignore_valid = [] if ignore_valid is None else ignore_valid
-
-    self.simulation = bool(int(os.getenv("SIMULATION", "0")))
-
-    # if freq and poll aren't specified, assume the max to be conservative
-    assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used."
-    self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services])
 
     for s in services:
-      p = self.poller if s not in self.non_polled_services else None
-      self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
+      if addr is not None:
+        p = self.poller if s not in self.non_polled_services else None
+        self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
+      self.freq[s] = service_list[s].frequency
 
       try:
         data = new_message(s)
-      except capnp.lib.capnp.KjException:
+      except capnp.lib.capnp.KjException:  # pylint: disable=c-extension-no-member
         data = new_message(s, 0) # lists
 
-      self.data[s] = getattr(data.as_reader(), s)
+      self.data[s] = getattr(data, s)
       self.logMonoTime[s] = 0
-      self.valid[s] = True  # FIXME: this should default to False
-
-      freq = max(min([SERVICE_LIST[s].frequency, self.update_freq]), 1.)
-      if s == poll:
-        max_freq = freq
-        min_freq = freq
-      else:
-        max_freq = min(freq, self.update_freq)
-        if SERVICE_LIST[s].frequency >= 2*self.update_freq:
-          min_freq = self.update_freq
-        elif self.update_freq >= 2*SERVICE_LIST[s].frequency:
-          min_freq = freq
-        else:
-          min_freq = min(freq, freq / 2.)
-      self.max_freq[s] = max_freq*1.2
-      self.min_freq[s] = min_freq*0.8
-      self.recv_dts[s] = deque(maxlen=int(10*freq))
+      self.valid[s] = data.valid
 
   def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
     return self.data[s]
 
-  def _check_avg_freq(self, s: str) -> bool:
-    return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive)
-
-  def update(self, timeout: int = 100) -> None:
+  def update(self, timeout: int = 1000) -> None:
     msgs = []
     for sock in self.poller.poll(timeout):
       msgs.append(recv_one_or_none(sock))
@@ -164,7 +182,7 @@ class SubMaster:
     # non-blocking receive for non-polled sockets
     for s in self.non_polled_services:
       msgs.append(recv_one_or_none(self.sock[s]))
-    self.update_msgs(time.monotonic(), msgs)
+    self.update_msgs(sec_since_boot(), msgs)
 
   def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1
@@ -174,63 +192,51 @@ class SubMaster:
         continue
 
       s = msg.which()
-      self.seen[s] = True
       self.updated[s] = True
 
-      if self.recv_time[s] > 1e-5:
-        self.recv_dts[s].append(cur_time - self.recv_time[s])
-      self.recv_time[s] = cur_time
-      self.recv_frame[s] = self.frame
+      if self.rcv_time[s] > 1e-5 and self.freq[s] > 1e-5 and (s not in self.non_polled_services) \
+        and (s not in self.ignore_average_freq):
+        self.recv_dts[s].append(cur_time - self.rcv_time[s])
+
+      self.rcv_time[s] = cur_time
+      self.rcv_frame[s] = self.frame
       self.data[s] = getattr(msg, s)
       self.logMonoTime[s] = msg.logMonoTime
       self.valid[s] = msg.valid
 
-    for s in self.data:
-      if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation:
-        # alive if delay is within 10x the expected frequency
-        self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency)
+      if SIMULATION:
+        self.alive[s] = True
 
-        # check average frequency; slow to fall, quick to recover
-        dts = self.recv_dts[s]
-        assert dts.maxlen is not None
-        recent_dts = list(dts)[-int(dts.maxlen / 10):]
-        try:
-          avg_freq = 1 / (sum(dts) / len(dts))
-          avg_freq_recent = 1 / (sum(recent_dts) / len(recent_dts))
-        except ZeroDivisionError:
-          avg_freq = 0
-          avg_freq_recent = 0
+    if not SIMULATION:
+      for s in self.data:
+        # arbitrary small number to avoid float comparison. If freq is 0, we can skip the check
+        if self.freq[s] > 1e-5:
+          # alive if delay is within 10x the expected frequency
+          self.alive[s] = (cur_time - self.rcv_time[s]) < (10. / self.freq[s])
 
-        avg_freq_ok = self.min_freq[s] <= avg_freq <= self.max_freq[s]
-        recent_freq_ok = self.min_freq[s] <= avg_freq_recent <= self.max_freq[s]
-        self.freq_ok[s] = avg_freq_ok or recent_freq_ok
-      else:
-        self.freq_ok[s] = True
-        if self.simulation:
-          self.alive[s] = self.seen[s] # alive is defined as seen when simulation flag set
+          # alive if average frequency is higher than 90% of expected frequency
+          avg_dt = sum(self.recv_dts[s]) / AVG_FREQ_HISTORY
+          expected_dt = 1 / (self.freq[s] * 0.90)
+          self.alive[s] = self.alive[s] and (avg_dt < expected_dt)
         else:
           self.alive[s] = True
 
-  def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
-    if service_list is None:
-      service_list = list(self.sock.keys())
+  def all_alive(self, service_list=None) -> bool:
+    if service_list is None:  # check all
+      service_list = self.alive.keys()
     return all(self.alive[s] for s in service_list if s not in self.ignore_alive)
 
-  def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
-    if service_list is None:
-      service_list = list(self.sock.keys())
-    return all(self.freq_ok[s] for s in service_list if self._check_avg_freq(s))
+  def all_valid(self, service_list=None) -> bool:
+    if service_list is None:  # check all
+      service_list = self.valid.keys()
+    return all(self.valid[s] for s in service_list)
 
-  def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
-    if service_list is None:
-      service_list = list(self.sock.keys())
-    return all(self.valid[s] for s in service_list if s not in self.ignore_valid)
+  def all_alive_and_valid(self, service_list=None) -> bool:
+    if service_list is None:  # check all
+      service_list = self.alive.keys()
+    return self.all_alive(service_list=service_list) and self.all_valid(service_list=service_list)
 
-  def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
-    return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
-
-
-class PubMaster:
+class PubMaster():
   def __init__(self, services: List[str]):
     self.sock = {}
     for s in services:
@@ -241,12 +247,5 @@ class PubMaster:
       dat = dat.to_bytes()
     self.sock[s].send(dat)
 
-  def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
-    for _ in range(int(timeout*(1./dt))):
-      if self.sock[s].all_readers_updated():
-        return True
-      time.sleep(dt)
-    return False
-
   def all_readers_updated(self, s: str) -> bool:
-    return self.sock[s].all_readers_updated()  # type: ignore
+    return self.sock[s].all_readers_updated()

cereal/messaging/bridge.cc

@@ -7,14 +7,9 @@
 
 typedef void (*sighandler_t)(int sig);
 
-#include "cereal/services.h"
-#include "msgq/impl_msgq.h"
-#include "msgq/impl_zmq.h"
-
-std::atomic<bool> do_exit = false;
-static void set_do_exit(int sig) {
-  do_exit = true;
-}
+#include "impl_msgq.h"
+#include "impl_zmq.h"
+#include "services.h"
 
 void sigpipe_handler(int sig) {
   assert(sig == SIGPIPE);
@@ -24,7 +19,7 @@ void sigpipe_handler(int sig) {
 static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
   std::vector<std::string> service_list;
   for (const auto& it : services) {
-    std::string name = it.second.name;
+    std::string name = it.name;
     bool in_whitelist = whitelist_str.find(name) != std::string::npos;
     if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
       continue;
@@ -36,8 +31,6 @@ static std::vector<std::string> get_services(std::string whitelist_str, bool zmq
 
 int main(int argc, char** argv) {
   signal(SIGPIPE, (sighandler_t)sigpipe_handler);
-  signal(SIGINT, (sighandler_t)set_do_exit);
-  signal(SIGTERM, (sighandler_t)set_do_exit);
 
   bool zmq_to_msgq = argc > 2;
   std::string ip = zmq_to_msgq ? argv[1] : "127.0.0.1";
@@ -57,7 +50,7 @@ int main(int argc, char** argv) {
   }
 
   std::map<SubSocket*, PubSocket*> sub2pub;
-  for (auto endpoint : get_services(whitelist_str, zmq_to_msgq)) {
+  for (auto endpoint: get_services(whitelist_str, zmq_to_msgq)) {
     PubSocket * pub_sock;
     SubSocket * sub_sock;
     if (zmq_to_msgq) {
@@ -74,18 +67,12 @@ int main(int argc, char** argv) {
     sub2pub[sub_sock] = pub_sock;
   }
 
-  while (!do_exit) {
+  while (true) {
     for (auto sub_sock : poller->poll(100)) {
       Message * msg = sub_sock->receive();
       if (msg == NULL) continue;
-      int ret;
-      do {
-        ret = sub2pub[sub_sock]->sendMessage(msg);
-      } while (ret == -1 && errno == EINTR && !do_exit);
-      assert(ret >= 0 || do_exit);
+      sub2pub[sub_sock]->sendMessage(msg);
       delete msg;
-
-      if (do_exit) break;
     }
   }
   return 0;

cereal/messaging/impl_msgq.cc

@@ -5,7 +5,8 @@
 #include <csignal>
 #include <cerrno>
 
-#include "msgq/impl_msgq.h"
+#include "services.h"
+#include "impl_msgq.h"
 
 
 volatile sig_atomic_t msgq_do_exit = 0;
@@ -15,6 +16,25 @@ void sig_handler(int signal) {
   msgq_do_exit = 1;
 }
 
+static bool service_exists(std::string path){
+  for (const auto& it : services) {
+    if (it.name == path) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static size_t get_size(std::string endpoint){
+  size_t sz = DEFAULT_SEGMENT_SIZE;
+
+  if (endpoint == "roadCameraState" || endpoint == "driverCameraState" || endpoint == "wideRoadCameraState"){
+    sz *= 10;
+  }
+
+  return sz;
+}
+
 
 MSGQContext::MSGQContext() {
 }
@@ -53,8 +73,12 @@ int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string a
   assert(context);
   assert(address == "127.0.0.1");
 
+  if (check_endpoint && !service_exists(std::string(endpoint))){
+    std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl;
+  }
+
   q = new msgq_queue_t;
-  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  int r = msgq_new_queue(q, endpoint.c_str(), get_size(endpoint));
   if (r != 0){
     return r;
   }
@@ -141,13 +165,12 @@ MSGQSubSocket::~MSGQSubSocket(){
 int MSGQPubSocket::connect(Context *context, std::string endpoint, bool check_endpoint){
   assert(context);
 
-  // TODO
-  //if (check_endpoint && !service_exists(std::string(endpoint))){
-  //  std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl;
-  //}
+  if (check_endpoint && !service_exists(std::string(endpoint))){
+    std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl;
+  }
 
   q = new msgq_queue_t;
-  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  int r = msgq_new_queue(q, endpoint.c_str(), get_size(endpoint));
   if (r != 0){
     return r;
   }

cereal/messaging/impl_msgq.h

@@ -1,10 +1,8 @@
 #pragma once
-
+#include "messaging.h"
+#include "msgq.h"
+#include <zmq.h>
 #include <string>
-#include <vector>
-
-#include "msgq/ipc.h"
-#include "msgq/msgq.h"
 
 #define MAX_POLLERS 128
 
@@ -63,5 +61,5 @@ private:
 public:
   void registerSocket(SubSocket *socket);
   std::vector<SubSocket*> poll(int timeout);
-  ~MSGQPoller(){}
+  ~MSGQPoller(){};
 };

cereal/messaging/impl_zmq.cc

@@ -3,18 +3,24 @@
 #include <iostream>
 #include <cstdlib>
 #include <cerrno>
-#include <unistd.h>
 
-#include "msgq/impl_zmq.h"
+#include <zmq.h>
+
+#include "services.h"
+#include "impl_zmq.h"
 
-//FIXME: This is a hack to get the port number from the socket name, might have collisions
 static int get_port(std::string endpoint) {
-    std::hash<std::string> hasher;
-    size_t hash_value = hasher(endpoint);
-    int start_port = 8023;
-    int max_port = 65535;
-    int port = start_port + (hash_value % (max_port - start_port));
-    return port;
+  int port = -1;
+  for (const auto& it : services) {
+    std::string name = it.name;
+    if (name == endpoint) {
+      port = it.port;
+      break;
+    }
+  }
+
+  assert(port >= 0);
+  return port;
 }
 
 ZMQContext::ZMQContext() {
@@ -64,7 +70,6 @@ int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string ad
   int reconnect_ivl = 500;
   zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
 
-
   full_endpoint = "tcp://" + address + ":";
   if (check_endpoint){
     full_endpoint += std::to_string(get_port(endpoint));
@@ -115,19 +120,14 @@ int ZMQPubSocket::connect(Context *context, std::string endpoint, bool check_end
     full_endpoint += endpoint;
   }
 
-  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same
-  pid = getpid();
-
   return zmq_bind(sock, full_endpoint.c_str());
 }
 
-int ZMQPubSocket::sendMessage(Message *message) {
-  assert(pid == getpid());
+int ZMQPubSocket::sendMessage(Message *message){
   return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
 }
 
-int ZMQPubSocket::send(char *data, size_t size) {
-  assert(pid == getpid());
+int ZMQPubSocket::send(char *data, size_t size){
   return zmq_send(sock, data, size, ZMQ_DONTWAIT);
 }
 

cereal/messaging/impl_zmq.h

@@ -1,10 +1,7 @@
 #pragma once
-
+#include "messaging.h"
 #include <zmq.h>
 #include <string>
-#include <vector>
-
-#include "msgq/ipc.h"
 
 #define MAX_POLLERS 128
 
@@ -46,7 +43,6 @@ class ZMQPubSocket : public PubSocket {
 private:
   void * sock;
   std::string full_endpoint;
-  int pid = -1;
 public:
   int connect(Context *context, std::string endpoint, bool check_endpoint=true);
   int sendMessage(Message *message);
@@ -64,5 +60,5 @@ private:
 public:
   void registerSocket(SubSocket *socket);
   std::vector<SubSocket*> poll(int timeout);
-  ~ZMQPoller(){}
+  ~ZMQPoller(){};
 };

cereal/messaging/messaging.cc

@@ -0,0 +1,103 @@
+#include "messaging.h"
+#include "impl_zmq.h"
+#include "impl_msgq.h"
+
+#ifdef __APPLE__
+const bool MUST_USE_ZMQ = true;
+#else
+const bool MUST_USE_ZMQ = false;
+#endif
+
+bool messaging_use_zmq(){
+  return std::getenv("ZMQ") || MUST_USE_ZMQ;
+}
+
+Context * Context::create(){
+  Context * c;
+  if (messaging_use_zmq()){
+    c = new ZMQContext();
+  } else {
+    c = new MSGQContext();
+  }
+  return c;
+}
+
+SubSocket * SubSocket::create(){
+  SubSocket * s;
+  if (messaging_use_zmq()){
+    s = new ZMQSubSocket();
+  } else {
+    s = new MSGQSubSocket();
+  }
+  return s;
+}
+
+SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
+  SubSocket *s = SubSocket::create();
+  int r = s->connect(context, endpoint, address, conflate, check_endpoint);
+
+  if (r == 0) {
+    return s;
+  } else {
+    delete s;
+    return NULL;
+  }
+}
+
+PubSocket * PubSocket::create(){
+  PubSocket * s;
+  if (messaging_use_zmq()){
+    s = new ZMQPubSocket();
+  } else {
+    s = new MSGQPubSocket();
+  }
+  return s;
+}
+
+PubSocket * PubSocket::create(Context * context, std::string endpoint, bool check_endpoint){
+  PubSocket *s = PubSocket::create();
+  int r = s->connect(context, endpoint, check_endpoint);
+
+  if (r == 0) {
+    return s;
+  } else {
+    delete s;
+    return NULL;
+  }
+}
+
+Poller * Poller::create(){
+  Poller * p;
+  if (messaging_use_zmq()){
+    p = new ZMQPoller();
+  } else {
+    p = new MSGQPoller();
+  }
+  return p;
+}
+
+Poller * Poller::create(std::vector<SubSocket*> sockets){
+  Poller * p = Poller::create();
+
+  for (auto s : sockets){
+    p->registerSocket(s);
+  }
+  return p;
+}
+
+extern "C" Context * messaging_context_create() {
+  return Context::create();
+}
+
+extern "C" SubSocket * messaging_subsocket_create(Context* context, const char* endpoint) {
+  return SubSocket::create(context, std::string(endpoint));
+}
+
+extern "C" PubSocket * messaging_pubsocket_create(Context* context, const char* endpoint) {
+  return PubSocket::create(context, std::string(endpoint));
+}
+
+extern "C" Poller * messaging_poller_create(SubSocket** sockets, int size) {
+  std::vector<SubSocket*> socketsVec(sockets, sockets + size);
+  return Poller::create(socketsVec);
+}

cereal/messaging/messaging.h

@@ -1,16 +1,10 @@
 #pragma once
-
 #include <cstddef>
 #include <map>
 #include <string>
 #include <vector>
-#include <utility>
-#include <time.h>
-
 #include <capnp/serialize.h>
-
-#include "cereal/gen/cpp/log.capnp.h"
-#include "msgq/ipc.h"
+#include "../gen/cpp/log.capnp.h"
 
 #ifdef __APPLE__
 #define CLOCK_BOOTTIME CLOCK_MONOTONIC
@@ -18,10 +12,61 @@
 
 #define MSG_MULTIPLE_PUBLISHERS 100
 
+bool messaging_use_zmq();
+
+class Context {
+public:
+  virtual void * getRawContext() = 0;
+  static Context * create();
+  virtual ~Context(){};
+};
+
+class Message {
+public:
+  virtual void init(size_t size) = 0;
+  virtual void init(char * data, size_t size) = 0;
+  virtual void close() = 0;
+  virtual size_t getSize() = 0;
+  virtual char * getData() = 0;
+  virtual ~Message(){};
+};
+
+
+class SubSocket {
+public:
+  virtual int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true) = 0;
+  virtual void setTimeout(int timeout) = 0;
+  virtual Message *receive(bool non_blocking=false) = 0;
+  virtual void * getRawSocket() = 0;
+  static SubSocket * create();
+  static SubSocket * create(Context * context, std::string endpoint, std::string address="127.0.0.1", bool conflate=false, bool check_endpoint=true);
+  virtual ~SubSocket(){};
+};
+
+class PubSocket {
+public:
+  virtual int connect(Context *context, std::string endpoint, bool check_endpoint=true) = 0;
+  virtual int sendMessage(Message *message) = 0;
+  virtual int send(char *data, size_t size) = 0;
+  virtual bool all_readers_updated() = 0;
+  static PubSocket * create();
+  static PubSocket * create(Context * context, std::string endpoint, bool check_endpoint=true);
+  static PubSocket * create(Context * context, std::string endpoint, int port, bool check_endpoint=true);
+  virtual ~PubSocket(){};
+};
+
+class Poller {
+public:
+  virtual void registerSocket(SubSocket *socket) = 0;
+  virtual std::vector<SubSocket*> poll(int timeout) = 0;
+  static Poller * create();
+  static Poller * create(std::vector<SubSocket*> sockets);
+  virtual ~Poller(){};
+};
 
 class SubMaster {
 public:
-  SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll = {},
+  SubMaster(const std::vector<const char *> &service_list,
             const char *address = nullptr, const std::vector<const char *> &ignore_alive = {});
   void update(int timeout = 1000);
   void update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages);
@@ -66,18 +111,6 @@ public:
     return heapArray_.asBytes();
   }
 
-  size_t getSerializedSize() {
-    return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
-  }
-
-  int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
-    size_t serialized_size = getSerializedSize();
-    if (serialized_size > buffer_size) { return -1; }
-    kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
-    capnp::writeMessage(out, *this);
-    return serialized_size;
-  }
-
 private:
   kj::Array<capnp::word> heapArray_;
 };

cereal/messaging/messaging.pxd

@@ -0,0 +1,40 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp cimport bool
+
+
+cdef extern from "messaging.h":
+  cdef cppclass Context:
+    @staticmethod
+    Context * create()
+
+  cdef cppclass Message:
+    void init(size_t)
+    void init(char *, size_t)
+    void close()
+    size_t getSize()
+    char *getData()
+
+  cdef cppclass SubSocket:
+    @staticmethod
+    SubSocket * create()
+    int connect(Context *, string, string, bool)
+    Message * receive(bool)
+    void setTimeout(int)
+
+  cdef cppclass PubSocket:
+    @staticmethod
+    PubSocket * create()
+    int connect(Context *, string)
+    int sendMessage(Message *)
+    int send(char *, size_t)
+    bool all_readers_updated()
+
+  cdef cppclass Poller:
+    @staticmethod
+    Poller * create()
+    void registerSocket(SubSocket *)
+    vector[SubSocket*] poll(int) nogil

cereal/messaging/messaging_pyx.pyx

@@ -0,0 +1,154 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+from libcpp.string cimport string
+from libcpp cimport bool
+from libc cimport errno
+
+
+from .messaging cimport Context as cppContext
+from .messaging cimport SubSocket as cppSubSocket
+from .messaging cimport PubSocket as cppPubSocket
+from .messaging cimport Poller as cppPoller
+from .messaging cimport Message as cppMessage
+
+
+class MessagingError(Exception):
+  pass
+
+
+class MultiplePublishersError(MessagingError):
+  pass
+
+
+cdef class Context:
+  cdef cppContext * context
+
+  def __cinit__(self):
+    self.context = cppContext.create()
+
+  def term(self):
+    del self.context
+    self.context = NULL
+
+  def __dealloc__(self):
+    pass
+    # Deleting the context will hang if sockets are still active
+    # TODO: Figure out a way to make sure the context is closed last
+    # del self.context
+
+
+cdef class Poller:
+  cdef cppPoller * poller
+  cdef list sub_sockets
+
+  def __cinit__(self):
+    self.sub_sockets = []
+    self.poller = cppPoller.create()
+
+  def __dealloc__(self):
+    del self.poller
+
+  def registerSocket(self, SubSocket socket):
+    self.sub_sockets.append(socket)
+    self.poller.registerSocket(socket.socket)
+
+  def poll(self, timeout):
+    sockets = []
+    cdef int t = timeout
+
+    with nogil:
+      result = self.poller.poll(t)
+
+    for s in result:
+      socket = SubSocket()
+      socket.setPtr(s)
+      sockets.append(socket)
+
+    return sockets
+
+cdef class SubSocket:
+  cdef cppSubSocket * socket
+  cdef bool is_owner
+
+  def __cinit__(self):
+    self.socket = cppSubSocket.create()
+    self.is_owner = True
+
+    if self.socket == NULL:
+      raise MessagingError
+
+  def __dealloc__(self):
+    if self.is_owner:
+      del self.socket
+
+  cdef setPtr(self, cppSubSocket * ptr):
+    if self.is_owner:
+      del self.socket
+
+    self.is_owner = False
+    self.socket = ptr
+
+  def connect(self, Context context, string endpoint, string address=b"127.0.0.1", bool conflate=False):
+    r = self.socket.connect(context.context, endpoint, address, conflate)
+
+    if r != 0:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise MessagingError
+
+  def setTimeout(self, int timeout):
+    self.socket.setTimeout(timeout)
+
+  def receive(self, bool non_blocking=False):
+    msg = self.socket.receive(non_blocking)
+
+    if msg == NULL:
+      # If a blocking read returns no message check errno if SIGINT was caught in the C++ code
+      if errno.errno == errno.EINTR:
+        print("SIGINT received, exiting")
+        sys.exit(1)
+
+      return None
+    else:
+      sz = msg.getSize()
+      m = msg.getData()[:sz]
+      del msg
+
+      return m
+
+
+cdef class PubSocket:
+  cdef cppPubSocket * socket
+
+  def __cinit__(self):
+    self.socket = cppPubSocket.create()
+    if self.socket == NULL:
+      raise MessagingError
+
+  def __dealloc__(self):
+    del self.socket
+
+  def connect(self, Context context, string endpoint):
+    r = self.socket.connect(context.context, endpoint)
+
+    if r != 0:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise MessagingError
+
+  def send(self, bytes data):
+    length = len(data)
+    r = self.socket.send(<char*>data, length)
+
+    if r != length:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise MessagingError
+
+  def all_readers_updated(self):
+    return self.socket.all_readers_updated()

cereal/messaging/msgq.cc

@@ -9,8 +9,6 @@
 #include <cstdlib>
 #include <csignal>
 #include <random>
-#include <string>
-#include <limits>
 
 #include <poll.h>
 #include <sys/ioctl.h>
@@ -23,7 +21,7 @@
 
 #include <stdio.h>
 
-#include "msgq/msgq.h"
+#include "msgq.h"
 
 void sigusr2_handler(int signal) {
   assert(signal == SIGUSR2);
@@ -31,7 +29,7 @@ void sigusr2_handler(int signal) {
 
 uint64_t msgq_get_uid(void){
   std::random_device rd("/dev/urandom");
-  std::uniform_int_distribution<uint64_t> distribution(0, std::numeric_limits<uint32_t>::max());
+  std::uniform_int_distribution<uint64_t> distribution(0,std::numeric_limits<uint32_t>::max());
 
   #ifdef __APPLE__
     // TODO: this doesn't work
@@ -77,28 +75,29 @@ void msgq_reset_reader(msgq_queue_t * q){
 
 void msgq_wait_for_subscriber(msgq_queue_t *q){
   while (*q->num_readers == 0){
-    // wait for subscriber
+    ;
   }
 
   return;
 }
 
+
 int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){
   assert(size < 0xFFFFFFFF); // Buffer must be smaller than 2^32 bytes
   std::signal(SIGUSR2, sigusr2_handler);
 
-  std::string full_path = "/dev/shm/";
-  const char* prefix = std::getenv("OPENPILOT_PREFIX");
-  if (prefix) {
-    full_path += std::string(prefix) + "/";
-  }
-  full_path += path;
+  const char * prefix = "/dev/shm/";
+  char * full_path = new char[strlen(path) + strlen(prefix) + 1];
+  strcpy(full_path, prefix);
+  strcat(full_path, path);
 
-  auto fd = open(full_path.c_str(), O_RDWR | O_CREAT, 0664);
+  auto fd = open(full_path, O_RDWR | O_CREAT, 0664);
   if (fd < 0) {
     std::cout << "Warning, could not open: " << full_path << std::endl;
+    delete[] full_path;
     return -1;
   }
+  delete[] full_path;
 
   int rc = ftruncate(fd, size + sizeof(msgq_header_t));
   if (rc < 0){
@@ -180,7 +179,7 @@ void msgq_init_subscriber(msgq_queue_t * q) {
 
     // No more slots available. Reset all subscribers to kick out inactive ones
     if (new_num_readers > NUM_READERS){
-      //std::cout << "Warning, evicting all subscribers!" << std::endl;
+      std::cout << "Warning, evicting all subscribers!" << std::endl;
       *q->num_readers = 0;
 
       for (size_t i = 0; i < NUM_READERS; i++){
@@ -308,7 +307,7 @@ int msgq_msg_ready(msgq_queue_t * q){
   assert(id >= 0); // Make sure subscriber is initialized
 
   if (q->read_uid_local != *q->read_uids[id]){
-    //std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
+    std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
     msgq_init_subscriber(q);
     goto start;
   }
@@ -321,11 +320,9 @@ int msgq_msg_ready(msgq_queue_t * q){
 
   uint32_t read_cycles, read_pointer;
   UNPACK64(read_cycles, read_pointer, *q->read_pointers[id]);
-  UNUSED(read_cycles);
 
   uint32_t write_cycles, write_pointer;
   UNPACK64(write_cycles, write_pointer, *q->write_pointer);
-  UNUSED(write_cycles);
 
   // Check if new message is available
   return (read_pointer != write_pointer);
@@ -337,7 +334,7 @@ int msgq_msg_recv(msgq_msg_t * msg, msgq_queue_t * q){
   assert(id >= 0); // Make sure subscriber is initialized
 
   if (q->read_uid_local != *q->read_uids[id]){
-    //std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
+    std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
     msgq_init_subscriber(q);
     goto start;
   }
@@ -353,7 +350,6 @@ int msgq_msg_recv(msgq_msg_t * msg, msgq_queue_t * q){
 
   uint32_t write_cycles, write_pointer;
   UNPACK64(write_cycles, write_pointer, *q->write_pointer);
-  UNUSED(write_cycles);
 
   char * p = q->data + read_pointer;
 

cereal/messaging/msgq.h

@@ -1,17 +1,15 @@
 #pragma once
-
 #include <cstdint>
 #include <cstring>
 #include <string>
 #include <atomic>
 
 #define DEFAULT_SEGMENT_SIZE (10 * 1024 * 1024)
-#define NUM_READERS 15
+#define NUM_READERS 10
 #define ALIGN(n) ((n + (8 - 1)) & -8)
 
-#define UNUSED(x) (void)x
 #define UNPACK64(higher, lower, input) do {uint64_t tmp = input; higher = tmp >> 32; lower = tmp & 0xFFFFFFFF;} while (0)
-#define PACK64(output, higher, lower) output = ((uint64_t)higher << 32) | ((uint64_t)lower & 0xFFFFFFFF)
+#define PACK64(output, higher, lower) output = ((uint64_t)higher << 32 ) | ((uint64_t)lower & 0xFFFFFFFF)
 
 struct  msgq_header_t {
   uint64_t num_readers;

cereal/messaging/socketmaster.cc

@@ -4,9 +4,8 @@
 #include <string>
 #include <mutex>
 
-#include "cereal/services.h"
-#include "cereal/messaging/messaging.h"
-
+#include "services.h"
+#include "messaging.h"
 
 const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
 
@@ -16,6 +15,13 @@ static inline uint64_t nanos_since_boot() {
   return t.tv_sec * 1000000000ULL + t.tv_nsec;
 }
 
+static const service *get_service(const char *name) {
+  for (const auto &it : services) {
+    if (strcmp(it.name, name) == 0) return &it;
+  }
+  return nullptr;
+}
+
 static inline bool inList(const std::vector<const char *> &list, const char *value) {
   for (auto &v : list) {
     if (strcmp(value, v) == 0) return true;
@@ -25,7 +31,7 @@ static inline bool inList(const std::vector<const char *> &list, const char *val
 
 class MessageContext {
 public:
-  MessageContext() : ctx_(nullptr) {}
+  MessageContext() : ctx_(nullptr) {};
   ~MessageContext() { delete ctx_; }
   inline Context *context() {
     std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
@@ -45,30 +51,26 @@ struct SubMaster::SubMessage {
   bool updated = false, alive = false, valid = true, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
-  bool is_polled = false;
   capnp::FlatArrayMessageReader *msg_reader = nullptr;
   AlignedBuffer aligned_buf;
   cereal::Event::Reader event;
 };
 
-SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
-                     const char *address, const std::vector<const char *> &ignore_alive) {
+SubMaster::SubMaster(const std::vector<const char *> &service_list, const char *address,
+                     const std::vector<const char *> &ignore_alive) {
   poller_ = Poller::create();
   for (auto name : service_list) {
-    assert(services.count(std::string(name)) > 0);
-
-    service serv = services.at(std::string(name));
+    const service *serv = get_service(name);
+    assert(serv != nullptr);
     SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
-    bool is_polled = inList(poll, name) || poll.empty();
-    if (is_polled) poller_->registerSocket(socket);
+    poller_->registerSocket(socket);
     SubMessage *m = new SubMessage{
       .name = name,
       .socket = socket,
-      .freq = serv.frequency,
+      .freq = serv->frequency,
       .ignore_alive = inList(ignore_alive, name),
-      .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
-      .is_polled = is_polled};
+      .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader))};
     m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
     messages_[socket] = m;
     services_[name] = m;
@@ -79,14 +81,6 @@ void SubMaster::update(int timeout) {
   for (auto &kv : messages_) kv.second->updated = false;
 
   auto sockets = poller_->poll(timeout);
-
-  // add non-polled sockets for non-blocking receive
-  for (auto &kv : messages_) {
-    SubMessage *m = kv.second;
-    SubSocket *s = kv.first;
-    if (!m->is_polled) sockets.push_back(s);
-  }
-
   uint64_t current_time = nanos_since_boot();
 
   std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
@@ -111,7 +105,7 @@ void SubMaster::update(int timeout) {
 void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
   if (++frame == UINT64_MAX) frame = 1;
 
-  for (auto &kv : messages) {
+  for(auto &kv : messages) {
     auto m_find = services_.find(kv.first);
     if (m_find == services_.end()){
       continue;
@@ -179,7 +173,7 @@ uint64_t SubMaster::rcv_time(const char *name) const {
 
 cereal::Event::Reader &SubMaster::operator[](const char *name) const {
   return services_.at(name)->event;
-}
+};
 
 SubMaster::~SubMaster() {
   delete poller_;
@@ -194,7 +188,7 @@ SubMaster::~SubMaster() {
 
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
-    assert(services.count(name) > 0);
+    assert(get_service(name) != nullptr);
     PubSocket *socket = PubSocket::create(message_context.context(), name);
     assert(socket);
     sockets_[name] = socket;

cereal/services.h

@@ -1,98 +0,0 @@
-/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */
-#ifndef __SERVICES_H
-#define __SERVICES_H
-#include <map>
-#include <string>
-struct service { std::string name; bool should_log; int frequency; int decimation; };
-static std::map<std::string, service> services = {
-  { "gyroscope", {"gyroscope", true, 104, 104}},
-  { "gyroscope2", {"gyroscope2", true, 100, 100}},
-  { "accelerometer", {"accelerometer", true, 104, 104}},
-  { "accelerometer2", {"accelerometer2", true, 100, 100}},
-  { "magnetometer", {"magnetometer", true, 25, 25}},
-  { "lightSensor", {"lightSensor", true, 100, 100}},
-  { "temperatureSensor", {"temperatureSensor", true, 2, 200}},
-  { "temperatureSensor2", {"temperatureSensor2", true, 2, 200}},
-  { "gpsNMEA", {"gpsNMEA", true, 9, -1}},
-  { "deviceState", {"deviceState", true, 2, 1}},
-  { "can", {"can", true, 100, 1223}},
-  { "controlsState", {"controlsState", true, 100, 10}},
-  { "pandaStates", {"pandaStates", true, 10, 1}},
-  { "peripheralState", {"peripheralState", true, 2, 1}},
-  { "radarState", {"radarState", true, 20, 5}},
-  { "roadEncodeIdx", {"roadEncodeIdx", false, 20, 1}},
-  { "liveTracks", {"liveTracks", true, 20, -1}},
-  { "sendcan", {"sendcan", true, 100, 139}},
-  { "logMessage", {"logMessage", true, 0, -1}},
-  { "errorLogMessage", {"errorLogMessage", true, 0, 1}},
-  { "liveCalibration", {"liveCalibration", true, 4, 4}},
-  { "liveTorqueParameters", {"liveTorqueParameters", true, 4, 1}},
-  { "liveDelay", {"liveDelay", true, 4, 1}},
-  { "androidLog", {"androidLog", true, 0, -1}},
-  { "carState", {"carState", true, 100, 10}},
-  { "carControl", {"carControl", true, 100, 10}},
-  { "carOutput", {"carOutput", true, 100, 10}},
-  { "longitudinalPlan", {"longitudinalPlan", true, 20, 5}},
-  { "procLog", {"procLog", true, 0, 15}},
-  { "gpsLocationExternal", {"gpsLocationExternal", true, 10, 10}},
-  { "gpsLocation", {"gpsLocation", true, 1, 1}},
-  { "ubloxGnss", {"ubloxGnss", true, 10, -1}},
-  { "qcomGnss", {"qcomGnss", true, 2, -1}},
-  { "gnssMeasurements", {"gnssMeasurements", true, 10, 10}},
-  { "clocks", {"clocks", true, 0, 1}},
-  { "ubloxRaw", {"ubloxRaw", true, 20, -1}},
-  { "livePose", {"livePose", true, 20, 4}},
-  { "liveLocationKalman", {"liveLocationKalman", true, 20, -1}},
-  { "liveParameters", {"liveParameters", true, 20, 5}},
-  { "cameraOdometry", {"cameraOdometry", true, 20, 5}},
-  { "thumbnail", {"thumbnail", true, 0, 1}},
-  { "onroadEvents", {"onroadEvents", true, 1, 1}},
-  { "carParams", {"carParams", true, 0, 1}},
-  { "roadCameraState", {"roadCameraState", true, 20, 20}},
-  { "driverCameraState", {"driverCameraState", true, 20, 20}},
-  { "driverEncodeIdx", {"driverEncodeIdx", false, 20, 1}},
-  { "driverStateV2", {"driverStateV2", true, 20, 10}},
-  { "driverMonitoringState", {"driverMonitoringState", true, 20, 10}},
-  { "wideRoadEncodeIdx", {"wideRoadEncodeIdx", false, 20, 1}},
-  { "wideRoadCameraState", {"wideRoadCameraState", true, 20, 20}},
-  { "drivingModelData", {"drivingModelData", true, 20, 10}},
-  { "modelV2", {"modelV2", true, 20, 40}},
-  { "managerState", {"managerState", true, 2, 1}},
-  { "uploaderState", {"uploaderState", true, 0, 1}},
-  { "navInstruction", {"navInstruction", true, 1, 10}},
-  { "navRoute", {"navRoute", true, 0, -1}},
-  { "navThumbnail", {"navThumbnail", true, 0, -1}},
-  { "navModel", {"navModel", true, 2, 4}},
-  { "mapRenderState", {"mapRenderState", true, 2, 1}},
-  { "uiPlan", {"uiPlan", true, 20, 40}},
-  { "qRoadEncodeIdx", {"qRoadEncodeIdx", false, 20, -1}},
-  { "userFlag", {"userFlag", true, 0, 1}},
-  { "microphone", {"microphone", true, 10, 10}},
-  { "uiDebug", {"uiDebug", true, 0, 1}},
-  { "testJoystick", {"testJoystick", true, 0, -1}},
-  { "roadEncodeData", {"roadEncodeData", false, 20, -1}},
-  { "driverEncodeData", {"driverEncodeData", false, 20, -1}},
-  { "wideRoadEncodeData", {"wideRoadEncodeData", false, 20, -1}},
-  { "qRoadEncodeData", {"qRoadEncodeData", false, 20, -1}},
-  { "livestreamWideRoadEncodeIdx", {"livestreamWideRoadEncodeIdx", false, 20, -1}},
-  { "livestreamRoadEncodeIdx", {"livestreamRoadEncodeIdx", false, 20, -1}},
-  { "livestreamDriverEncodeIdx", {"livestreamDriverEncodeIdx", false, 20, -1}},
-  { "livestreamWideRoadEncodeData", {"livestreamWideRoadEncodeData", false, 20, -1}},
-  { "livestreamRoadEncodeData", {"livestreamRoadEncodeData", false, 20, -1}},
-  { "livestreamDriverEncodeData", {"livestreamDriverEncodeData", false, 20, -1}},
-  { "customReservedRawData0", {"customReservedRawData0", true, 0, -1}},
-  { "customReservedRawData1", {"customReservedRawData1", true, 0, -1}},
-  { "customReservedRawData2", {"customReservedRawData2", true, 0, -1}},
-  { "frogpilotCarControl", {"frogpilotCarControl", true, 100, 10}},
-  { "frogpilotCarParams", {"frogpilotCarParams", true, 0, 1}},
-  { "frogpilotCarState", {"frogpilotCarState", true, 100, 10}},
-  { "frogpilotControlsState", {"frogpilotControlsState", true, 100, 10}},
-  { "frogpilotDeviceState", {"frogpilotDeviceState", true, 2, 1}},
-  { "frogpilotModelV2", {"frogpilotModelV2", true, 20, 40}},
-  { "frogpilotNavigation", {"frogpilotNavigation", true, 1, 10}},
-  { "frogpilotOnroadEvents", {"frogpilotOnroadEvents", true, 1, 1}},
-  { "frogpilotPlan", {"frogpilotPlan", true, 20, 5}},
-  { "frogpilotRadarState", {"frogpilotRadarState", true, 20, 5}},
-};
-#endif
-

cereal/services.py

@@ -1,112 +1,77 @@
 #!/usr/bin/env python3
+import os
 from typing import Optional
 
+TICI = os.path.isfile('/TICI')
+RESERVED_PORT = 8022  # sshd
+STARTING_PORT = 8001
+
+
+def new_port(port: int):
+  port += STARTING_PORT
+  return port + 1 if port >= RESERVED_PORT else port
+
 
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
+  def __init__(self, port: int, should_log: bool, frequency: float, decimation: Optional[int] = None):
+    self.port = port
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
 
+DCAM_FREQ = 10. if not TICI else 20.
 
-_services: dict[str, tuple] = {
+services = {
   # service: (should_log, frequency, qlog decimation (optional))
-  # note: the "EncodeIdx" packets will still be in the log
-  "gyroscope": (True, 104., 104),
-  "gyroscope2": (True, 100., 100),
-  "accelerometer": (True, 104., 104),
-  "accelerometer2": (True, 100., 100),
-  "magnetometer": (True, 25., 25),
-  "lightSensor": (True, 100., 100),
-  "temperatureSensor": (True, 2., 200),
-  "temperatureSensor2": (True, 2., 200),
+  "sensorEvents": (True, 100., 100),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "can": (True, 100., 1223),  # decimation gives ~5 msgs in a full segment
+  "can": (True, 100.),
   "controlsState": (True, 100., 10),
-  "pandaStates": (True, 10., 1),
+  "pandaStates": (True, 2., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
-  "roadEncodeIdx": (False, 20., 1),
+  "roadEncodeIdx": (True, 20., 1),
   "liveTracks": (True, 20.),
   "sendcan": (True, 100., 139),
   "logMessage": (True, 0.),
   "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
-  "liveTorqueParameters": (True, 4., 1),
-  "liveDelay": (True, 4., 1),
   "androidLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
-  "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 5),
-  "procLog": (True, 0.5, 15),
+  "procLog": (True, 0.5),
   "gpsLocationExternal": (True, 10., 10),
-  "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
-  "qcomGnss": (True, 2.),
-  "gnssMeasurements": (True, 10., 10),
-  "clocks": (True, 0.1, 1),
+  "clocks": (True, 1., 1),
   "ubloxRaw": (True, 20.),
-  "livePose": (True, 20., 4),
-  "liveLocationKalman": (True, 20.),
+  "liveLocationKalman": (True, 20., 5),
   "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 5),
+  "lateralPlan": (True, 20., 5),
   "thumbnail": (True, 0.2, 1),
-  "onroadEvents": (True, 1., 1),
+  "carEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
   "roadCameraState": (True, 20., 20),
-  "driverCameraState": (True, 20., 20),
-  "driverEncodeIdx": (False, 20., 1),
-  "driverStateV2": (True, 20., 10),
-  "driverMonitoringState": (True, 20., 10),
-  "wideRoadEncodeIdx": (False, 20., 1),
+  "driverCameraState": (True, DCAM_FREQ, DCAM_FREQ),
+  "driverEncodeIdx": (True, DCAM_FREQ, 1),
+  "driverState": (True, DCAM_FREQ, DCAM_FREQ / 2),
+  "driverMonitoringState": (True, DCAM_FREQ, DCAM_FREQ / 2),
+  "wideRoadEncodeIdx": (True, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
-  "drivingModelData": (True, 20., 10),
   "modelV2": (True, 20., 40),
   "managerState": (True, 2., 1),
   "uploaderState": (True, 0., 1),
-  "navInstruction": (True, 1., 10),
+  "navInstruction": (True, 0.),
   "navRoute": (True, 0.),
   "navThumbnail": (True, 0.),
-  "navModel": (True, 2., 4.),
-  "mapRenderState": (True, 2., 1.),
-  "uiPlan": (True, 20., 40.),
-  "qRoadEncodeIdx": (False, 20.),
-  "userFlag": (True, 0., 1),
-  "microphone": (True, 10., 10),
 
   # debug
-  "uiDebug": (True, 0., 1),
-  "testJoystick": (True, 0.),
-  "roadEncodeData": (False, 20.),
-  "driverEncodeData": (False, 20.),
-  "wideRoadEncodeData": (False, 20.),
-  "qRoadEncodeData": (False, 20.),
-  "livestreamWideRoadEncodeIdx": (False, 20.),
-  "livestreamRoadEncodeIdx": (False, 20.),
-  "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20.),
-  "livestreamRoadEncodeData": (False, 20.),
-  "livestreamDriverEncodeData": (False, 20.),
-  "customReservedRawData0": (True, 0.),
-  "customReservedRawData1": (True, 0.),
-  "customReservedRawData2": (True, 0.),
-
-  # FrogPilot
-  "frogpilotCarControl": (True, 100., 10),
-  "frogpilotCarParams": (True, 0.02, 1),
-  "frogpilotCarState": (True, 100., 10),
-  "frogpilotControlsState": (True, 100., 10),
-  "frogpilotDeviceState": (True, 2., 1),
-  "frogpilotModelV2": (True, 20., 40),
-  "frogpilotNavigation": (True, 1., 10),
-  "frogpilotOnroadEvents": (True, 1., 1),
-  "frogpilotPlan": (True, 20., 5),
-  "frogpilotRadarState": (True, 20., 5),
+  "testJoystick": (False, 0.),
 }
-SERVICE_LIST = {name: Service(*vals) for
-                idx, (name, vals) in enumerate(_services.items())}
+service_list = {name: Service(new_port(idx), *vals) for  # type: ignore
+                idx, (name, vals) in enumerate(services.items())}
 
 
 def build_header():
@@ -114,19 +79,14 @@ def build_header():
   h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
   h += "#ifndef __SERVICES_H\n"
   h += "#define __SERVICES_H\n"
-
-  h += "#include <map>\n"
-  h += "#include <string>\n"
-
-  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
-  h += "static std::map<std::string, service> services = {\n"
-  for k, v in SERVICE_LIST.items():
+  h += "struct service { char name[0x100]; int port; bool should_log; int frequency; int decimation; };\n"
+  h += "static struct service services[] = {\n"
+  for k, v in service_list.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation)
+    h += '  { "%s", %d, %s, %d, %d },\n' % \
+         (k, v.port, should_log, v.frequency, decimation)
   h += "};\n"
-
   h += "#endif\n"
   return h
 

cereal/visionipc/ipc.cc

@@ -0,0 +1,121 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifdef __APPLE__
+#define getsocket() socket(AF_UNIX, SOCK_STREAM, 0)
+#else
+#define getsocket() socket(AF_UNIX, SOCK_SEQPACKET, 0)
+#endif
+
+#include "ipc.h"
+
+int ipc_connect(const char* socket_path) {
+  int err;
+
+  int sock = getsocket();
+
+  if (sock < 0) return -1;
+  struct sockaddr_un addr = {
+    .sun_family = AF_UNIX,
+  };
+  snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", socket_path);
+  err = connect(sock, (struct sockaddr*)&addr, sizeof(addr));
+  if (err != 0) {
+    close(sock);
+    return -1;
+  }
+
+  return sock;
+}
+
+int ipc_bind(const char* socket_path) {
+  int err;
+
+  unlink(socket_path);
+
+  int sock = getsocket();
+
+  struct sockaddr_un addr = {
+    .sun_family = AF_UNIX,
+  };
+  snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", socket_path);
+  err = bind(sock, (struct sockaddr *)&addr, sizeof(addr));
+  assert(err == 0);
+
+  err = listen(sock, 3);
+  assert(err == 0);
+
+  return sock;
+}
+
+
+int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
+                          int *out_num_fds) {
+  char control_buf[CMSG_SPACE(sizeof(int) * num_fds)];
+  memset(control_buf, 0, CMSG_SPACE(sizeof(int) * num_fds));
+
+  struct iovec iov = {
+    .iov_base = buf,
+    .iov_len = buf_size,
+  };
+  struct msghdr msg = {
+    .msg_iov = &iov,
+    .msg_iovlen = 1,
+  };
+
+  if (num_fds > 0) {
+    assert(fds);
+
+    msg.msg_control = control_buf;
+    msg.msg_controllen = CMSG_SPACE(sizeof(int) * num_fds);
+  }
+
+  if (send) {
+    if (num_fds) {
+      struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+      assert(cmsg);
+      cmsg->cmsg_level = SOL_SOCKET;
+      cmsg->cmsg_type = SCM_RIGHTS;
+      cmsg->cmsg_len = CMSG_LEN(sizeof(int) * num_fds);
+      memcpy(CMSG_DATA(cmsg), fds, sizeof(int) * num_fds);
+    }
+    return sendmsg(fd, &msg, 0);
+  } else {
+    int r = recvmsg(fd, &msg, 0);
+    if (r < 0) return r;
+
+    int recv_fds = 0;
+    if (msg.msg_controllen > 0) {
+      struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+      assert(cmsg);
+      assert(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS);
+      recv_fds = (cmsg->cmsg_len - CMSG_LEN(0));
+      assert(recv_fds > 0 && (recv_fds % sizeof(int)) == 0);
+      recv_fds /= sizeof(int);
+
+      assert(fds && recv_fds <= num_fds);
+      memcpy(fds, CMSG_DATA(cmsg), sizeof(int) * recv_fds);
+    }
+
+    if (msg.msg_flags) {
+      for (int i=0; i<recv_fds; i++) {
+        close(fds[i]);
+      }
+      return -1;
+    }
+
+    if (fds) {
+      assert(out_num_fds);
+      *out_num_fds = recv_fds;
+    }
+    return r;
+  }
+}

cereal/visionipc/ipc.h

@@ -0,0 +1,7 @@
+#pragma once
+#include <cstddef>
+
+int ipc_connect(const char* socket_path);
+int ipc_bind(const char* socket_path);
+int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
+                          int *out_num_fds);

cereal/visionipc/visionbuf.cc

@@ -0,0 +1,49 @@
+#include "visionbuf.h"
+
+#define ALIGN(x, align) (((x) + (align)-1) & ~((align)-1))
+
+#ifdef QCOM
+// from libadreno_utils.so
+extern "C" void compute_aligned_width_and_height(int width,
+                                                 int height,
+                                                 int bpp,
+                                                 int tile_mode,
+                                                 int raster_mode,
+                                                 int padding_threshold,
+                                                 int *aligned_w,
+                                                 int *aligned_h);
+#endif
+
+void visionbuf_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h) {
+#ifdef QCOM
+  compute_aligned_width_and_height(ALIGN(width, 32), ALIGN(height, 32), 3, 0, 0, 512, aligned_w, aligned_h);
+#else
+  *aligned_w = width; *aligned_h = height;
+#endif
+}
+
+void VisionBuf::init_rgb(size_t init_width, size_t init_height, size_t init_stride) {
+  this->rgb = true;
+  this->width = init_width;
+  this->height = init_height;
+  this->stride = init_stride;
+}
+
+void VisionBuf::init_yuv(size_t init_width, size_t init_height){
+  this->rgb = false;
+  this->width = init_width;
+  this->height = init_height;
+
+  this->y = (uint8_t *)this->addr;
+  this->u = this->y + (this->width * this->height);
+  this->v = this->u + (this->width / 2 * this->height / 2);
+}
+
+
+uint64_t VisionBuf::get_frame_id() {
+  return *frame_id;
+}
+
+void VisionBuf::set_frame_id(uint64_t id) {
+  *frame_id = id;
+}

cereal/visionipc/visionbuf.h

@@ -1,6 +1,5 @@
 #pragma once
-
-#include "msgq/visionipc/visionipc.h"
+#include "visionipc.h"
 
 #define CL_USE_DEPRECATED_OPENCL_1_2_APIS
 #ifdef __APPLE__
@@ -13,11 +12,13 @@
 #define VISIONBUF_SYNC_TO_DEVICE 1
 
 enum VisionStreamType {
+  VISION_STREAM_RGB_BACK,
+  VISION_STREAM_RGB_FRONT,
+  VISION_STREAM_RGB_WIDE,
   VISION_STREAM_ROAD,
   VISION_STREAM_DRIVER,
   VISION_STREAM_WIDE_ROAD,
-
-  VISION_STREAM_MAP,
+  VISION_STREAM_RGB_MAP,
   VISION_STREAM_MAX,
 };
 
@@ -33,11 +34,11 @@ class VisionBuf {
   size_t width = 0;
   size_t height = 0;
   size_t stride = 0;
-  size_t uv_offset = 0;
 
   // YUV
   uint8_t * y = nullptr;
-  uint8_t * uv = nullptr;
+  uint8_t * u = nullptr;
+  uint8_t * v = nullptr;
 
   // Visionipc
   uint64_t server_id = 0;
@@ -55,7 +56,7 @@ class VisionBuf {
   void import();
   void init_cl(cl_device_id device_id, cl_context ctx);
   void init_rgb(size_t width, size_t height, size_t stride);
-  void init_yuv(size_t width, size_t height, size_t stride, size_t uv_offset);
+  void init_yuv(size_t width, size_t height);
   int sync(int dir);
   int free();
 

cereal/visionipc/visionbuf_cl.cc

@@ -1,4 +1,4 @@
-#include "msgq/visionipc/visionbuf.h"
+#include "visionbuf.h"
 
 #include <atomic>
 #include <stdio.h>
@@ -34,8 +34,8 @@ static void *malloc_with_fd(size_t len, int *fd) {
 
 void VisionBuf::allocate(size_t length) {
   this->len = length;
-  this->mmap_len = this->len + sizeof(uint64_t);
-  this->addr = malloc_with_fd(this->mmap_len, &this->fd);
+  this->mmap_len = this->len;
+  this->addr = malloc_with_fd(this->len, &this->fd);
   this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
 }
 

cereal/visionipc/visionbuf_ion.cc

@@ -14,7 +14,7 @@
 
 #include <msm_ion.h>
 
-#include "msgq/visionipc/visionbuf.h"
+#include "visionbuf.h"
 
 // keep trying if x gets interrupted by a signal
 #define HANDLE_EINTR(x)                                       \
@@ -40,35 +40,30 @@
 #define DEVICE_PAGE_SIZE_CL 4096
 #define PADDING_CL 0
 
-struct IonFileHandle {
-  IonFileHandle() {
-    fd = open("/dev/ion", O_RDWR | O_NONBLOCK);
-    assert(fd >= 0);
+static int ion_fd = -1;
+static void ion_init() {
+  if (ion_fd == -1) {
+    ion_fd = open("/dev/ion", O_RDWR | O_NONBLOCK);
   }
-  ~IonFileHandle() {
-    close(fd);
-  }
-  int fd = -1;
-};
-
-int ion_fd() {
-  static IonFileHandle fh;
-  return fh.fd;
 }
 
 void VisionBuf::allocate(size_t length) {
+  int err;
+
+  ion_init();
+
   struct ion_allocation_data ion_alloc = {0};
   ion_alloc.len = length + PADDING_CL + sizeof(uint64_t);
   ion_alloc.align = 4096;
   ion_alloc.heap_id_mask = 1 << ION_IOMMU_HEAP_ID;
   ion_alloc.flags = ION_FLAG_CACHED;
 
-  int err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_ALLOC, &ion_alloc));
+  err = HANDLE_EINTR(ioctl(ion_fd, ION_IOC_ALLOC, &ion_alloc));
   assert(err == 0);
 
   struct ion_fd_data ion_fd_data = {0};
   ion_fd_data.handle = ion_alloc.handle;
-  err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_SHARE, &ion_fd_data));
+  err = HANDLE_EINTR(ioctl(ion_fd, ION_IOC_SHARE, &ion_fd_data));
   assert(err == 0);
 
   void *mmap_addr = mmap(NULL, ion_alloc.len,
@@ -90,10 +85,12 @@ void VisionBuf::import(){
   int err;
   assert(this->fd >= 0);
 
+  ion_init();
+
   // Get handle
   struct ion_fd_data fd_data = {0};
   fd_data.fd = this->fd;
-  err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_IMPORT, &fd_data));
+  err = HANDLE_EINTR(ioctl(ion_fd, ION_IOC_IMPORT, &fd_data));
   assert(err == 0);
 
   this->handle = fd_data.handle;
@@ -139,7 +136,7 @@ int VisionBuf::sync(int dir) {
      ION_IOC_INV_CACHES : ION_IOC_CLEAN_CACHES;
 
   custom_data.arg = (unsigned long)&flush_data;
-  return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_CUSTOM, &custom_data));
+  return HANDLE_EINTR(ioctl(ion_fd, ION_IOC_CUSTOM, &custom_data));
 }
 
 int VisionBuf::free() {
@@ -157,5 +154,5 @@ int VisionBuf::free() {
   if (err != 0) return err;
 
   struct ion_handle_data handle_data = {.handle = this->handle};
-  return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_FREE, &handle_data));
+  return HANDLE_EINTR(ioctl(ion_fd, ION_IOC_FREE, &handle_data));
 }

cereal/visionipc/visionipc.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+
+constexpr int VISIONIPC_MAX_FDS = 128;
+
+struct VisionIpcBufExtra {
+  uint32_t frame_id;
+  uint64_t timestamp_sof;
+  uint64_t timestamp_eof;
+};
+
+struct VisionIpcPacket {
+  uint64_t server_id;
+  size_t idx;
+  struct VisionIpcBufExtra extra;
+};

cereal/visionipc/visionipc.pxd

@@ -0,0 +1,39 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool
+
+cdef extern from "visionbuf.h":
+  cdef enum VisionStreamType:
+    pass
+
+  cdef cppclass VisionBuf:
+    void * addr
+    size_t len
+    size_t width
+    size_t height
+    size_t stride
+
+cdef extern from "visionipc.h":
+  struct VisionIpcBufExtra:
+    uint32_t frame_id
+    uint64_t timestamp_sof
+    uint64_t timestamp_eof
+
+cdef extern from "visionipc_server.h":
+  cdef cppclass VisionIpcServer:
+    VisionIpcServer(string, void*, void*)
+    void create_buffers(VisionStreamType, size_t, bool, size_t, size_t)
+    VisionBuf * get_buffer(VisionStreamType)
+    void send(VisionBuf *, VisionIpcBufExtra *, bool)
+    void start_listener()
+
+cdef extern from "visionipc_client.h":
+  cdef cppclass VisionIpcClient:
+    VisionIpcClient(string, VisionStreamType, bool, void*, void*)
+    VisionBuf * recv(VisionIpcBufExtra *, int)
+    bool connect(bool)
+    bool is_connected()

cereal/visionipc/visionipc_client.cc

@@ -0,0 +1,129 @@
+#include <chrono>
+#include <cassert>
+#include <iostream>
+#include <thread>
+
+#include "visionipc/ipc.h"
+#include "visionipc/visionipc_client.h"
+#include "visionipc/visionipc_server.h"
+#include "logger/logger.h"
+
+VisionIpcClient::VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id, cl_context ctx) : name(name), type(type), device_id(device_id), ctx(ctx) {
+  msg_ctx = Context::create();
+  sock = SubSocket::create(msg_ctx, get_endpoint_name(name, type), "127.0.0.1", conflate, false);
+
+  poller = Poller::create();
+  poller->registerSocket(sock);
+}
+
+// Connect is not thread safe. Do not use the buffers while calling connect
+bool VisionIpcClient::connect(bool blocking){
+  connected = false;
+
+  // Cleanup old buffers on reconnect
+  for (size_t i = 0; i < num_buffers; i++){
+    if (buffers[i].free() != 0) {
+      LOGE("Failed to free buffer %zu", i);
+    }
+  }
+
+  num_buffers = 0;
+
+  // Connect to server socket and ask for all FDs of type
+  std::string path = "/tmp/visionipc_" + name;
+
+  int socket_fd = -1;
+  while (socket_fd < 0) {
+    socket_fd = ipc_connect(path.c_str());
+
+    if (socket_fd < 0) {
+      if (blocking){
+        std::cout << "VisionIpcClient connecting" << std::endl;
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+      } else {
+        return false;
+      }
+    }
+  }
+
+  // Send stream type to server to request FDs
+  int r = ipc_sendrecv_with_fds(true, socket_fd, &type, sizeof(type), nullptr, 0, nullptr);
+  assert(r == sizeof(type));
+
+  // Get FDs
+  int fds[VISIONIPC_MAX_FDS];
+  VisionBuf bufs[VISIONIPC_MAX_FDS];
+  r = ipc_sendrecv_with_fds(false, socket_fd, &bufs, sizeof(bufs), fds, VISIONIPC_MAX_FDS, &num_buffers);
+
+  assert(num_buffers >= 0);
+  assert(r == sizeof(VisionBuf) * num_buffers);
+
+  // Import buffers
+  for (size_t i = 0; i < num_buffers; i++){
+    buffers[i] = bufs[i];
+    buffers[i].fd = fds[i];
+    buffers[i].import();
+    if (buffers[i].rgb) {
+      buffers[i].init_rgb(buffers[i].width, buffers[i].height, buffers[i].stride);
+    } else {
+      buffers[i].init_yuv(buffers[i].width, buffers[i].height);
+    }
+
+    if (device_id) buffers[i].init_cl(device_id, ctx);
+  }
+
+  close(socket_fd);
+  connected = true;
+  return true;
+}
+
+VisionBuf * VisionIpcClient::recv(VisionIpcBufExtra * extra, const int timeout_ms){
+  auto p = poller->poll(timeout_ms);
+
+  if (!p.size()){
+    return nullptr;
+  }
+
+  Message * r = sock->receive(true);
+  if (r == nullptr){
+    return nullptr;
+  }
+
+  // Get buffer
+  assert(r->getSize() == sizeof(VisionIpcPacket));
+  VisionIpcPacket *packet = (VisionIpcPacket*)r->getData();
+
+  assert(packet->idx < num_buffers);
+  VisionBuf * buf = &buffers[packet->idx];
+
+  if (buf->server_id != packet->server_id){
+    connected = false;
+    delete r;
+    return nullptr;
+  }
+
+  if (extra) {
+    *extra = packet->extra;
+  }
+
+  if (buf->sync(VISIONBUF_SYNC_TO_DEVICE) != 0) {
+    LOGE("Failed to sync buffer");
+  }
+
+  delete r;
+  return buf;
+}
+
+
+
+VisionIpcClient::~VisionIpcClient(){
+  for (size_t i = 0; i < num_buffers; i++){
+    if (buffers[i].free() != 0) {
+      LOGE("Failed to free buffer %zu", i);
+    }
+  }
+
+  delete sock;
+  delete poller;
+  delete msg_ctx;
+}

cereal/visionipc/visionipc_client.h

@@ -1,11 +1,11 @@
 #pragma once
-
-#include <set>
+#include <vector>
 #include <string>
+#include <unistd.h>
 
-#include "msgq/ipc.h"
-#include "msgq/visionipc/visionbuf.h"
-
+#include "messaging/messaging.h"
+#include "visionipc/visionipc.h"
+#include "visionipc/visionbuf.h"
 
 class VisionIpcClient {
 private:
@@ -14,12 +14,15 @@ private:
   SubSocket * sock;
   Poller * poller;
 
+  VisionStreamType type;
+
   cl_device_id device_id = nullptr;
   cl_context ctx = nullptr;
 
+  void init_msgq(bool conflate);
+
 public:
   bool connected = false;
-  VisionStreamType type;
   int num_buffers = 0;
   VisionBuf buffers[VISIONIPC_MAX_FDS];
   VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
@@ -27,5 +30,4 @@ public:
   VisionBuf * recv(VisionIpcBufExtra * extra=nullptr, const int timeout_ms=100);
   bool connect(bool blocking=true);
   bool is_connected() { return connected; }
-  static std::set<VisionStreamType> getAvailableStreams(const std::string &name, bool blocking = true);
 };

cereal/visionipc/visionipc_pyx.pyx

@@ -0,0 +1,93 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+import numpy as np
+cimport numpy as cnp
+from cython.view cimport array
+from libc.string cimport memcpy
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool
+from libcpp.string cimport string
+
+from .visionipc cimport VisionIpcServer as cppVisionIpcServer
+from .visionipc cimport VisionIpcClient as cppVisionIpcClient
+from .visionipc cimport VisionBuf as cppVisionBuf
+from .visionipc cimport VisionIpcBufExtra
+
+cpdef enum VisionStreamType:
+  VISION_STREAM_RGB_BACK
+  VISION_STREAM_RGB_FRONT
+  VISION_STREAM_RGB_WIDE
+  VISION_STREAM_ROAD
+  VISION_STREAM_DRIVER
+  VISION_STREAM_WIDE_ROAD
+
+
+cdef class VisionIpcServer:
+  cdef cppVisionIpcServer * server
+
+  def __init__(self, string name):
+    self.server = new cppVisionIpcServer(name, NULL, NULL)
+
+  def create_buffers(self,  VisionStreamType tp, size_t num_buffers, bool rgb, size_t width, size_t height):
+    self.server.create_buffers(tp, num_buffers, rgb, width, height)
+
+  def send(self, VisionStreamType tp, bytes data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0):
+    cdef cppVisionBuf * buf = self.server.get_buffer(tp)
+
+    # Populate buffer
+    assert buf.len == len(data)
+    memcpy(buf.addr, <char*>data, len(data))
+
+    cdef VisionIpcBufExtra extra
+    extra.frame_id = frame_id
+    extra.timestamp_sof = timestamp_sof
+    extra.timestamp_eof = timestamp_eof
+
+    self.server.send(buf, &extra, False)
+
+  def start_listener(self):
+    self.server.start_listener()
+
+  def __dealloc__(self):
+    del self.server
+
+
+cdef class VisionIpcClient:
+  cdef cppVisionBuf * buf
+  cdef cppVisionIpcClient * client
+
+  def __cinit__(self, string name, VisionStreamType stream, bool conflate):
+    self.client = new cppVisionIpcClient(name, stream, conflate, NULL, NULL)
+    self.buf = NULL
+
+  def __dealloc__(self):
+    del self.client
+
+  @property
+  def width(self):
+    return None if not self.buf else self.buf.width
+
+  @property
+  def height(self):
+    return None if not self.buf else self.buf.height
+
+  @property
+  def stride(self):
+    return None if not self.buf else self.buf.stride
+
+  def recv(self, int timeout_ms=100):
+    self.buf = self.client.recv(NULL, timeout_ms)
+    if not self.buf:
+      return None
+    cdef cnp.ndarray dat = np.empty(self.buf.len, dtype=np.uint8)
+    cdef char[:] dat_view = dat
+    memcpy(&dat_view[0], self.buf.addr, self.buf.len)
+    return dat
+
+  def connect(self, bool blocking):
+    return self.client.connect(blocking)
+
+  def is_connected(self):
+    return self.client.is_connected()

cereal/visionipc/visionipc_server.cc

@@ -0,0 +1,184 @@
+#include <iostream>
+#include <chrono>
+#include <cassert>
+#include <random>
+
+#include <poll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "messaging/messaging.h"
+#include "visionipc/ipc.h"
+#include "visionipc/visionipc_server.h"
+#include "logger/logger.h"
+
+std::string get_endpoint_name(std::string name, VisionStreamType type){
+  if (messaging_use_zmq()){
+    assert(name == "camerad");
+    return std::to_string(9000 + static_cast<int>(type));
+  } else {
+    return "visionipc_" + name + "_" + std::to_string(type);
+  }
+}
+
+VisionIpcServer::VisionIpcServer(std::string name, cl_device_id device_id, cl_context ctx) : name(name), device_id(device_id), ctx(ctx) {
+  msg_ctx = Context::create();
+
+  std::random_device rd("/dev/urandom");
+  std::uniform_int_distribution<uint64_t> distribution(0,std::numeric_limits<uint64_t>::max());
+  server_id = distribution(rd);
+}
+
+void VisionIpcServer::create_buffers(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height){
+  // TODO: assert that this type is not created yet
+  assert(num_buffers < VISIONIPC_MAX_FDS);
+  int aligned_w = 0, aligned_h = 0;
+
+  size_t size = 0;
+  size_t stride = 0; // Only used for RGB
+
+  if (rgb) {
+    visionbuf_compute_aligned_width_and_height(width, height, &aligned_w, &aligned_h);
+    size = (size_t)aligned_w * (size_t)aligned_h * 3;
+    stride = aligned_w * 3;
+  } else {
+    size = width * height * 3 / 2;
+  }
+
+  // Create map + alloc requested buffers
+  for (size_t i = 0; i < num_buffers; i++){
+    VisionBuf* buf = new VisionBuf();
+    buf->allocate(size);
+    buf->idx = i;
+    buf->type = type;
+
+    if (device_id) buf->init_cl(device_id, ctx);
+
+    rgb ? buf->init_rgb(width, height, stride) : buf->init_yuv(width, height);
+
+    buffers[type].push_back(buf);
+  }
+
+  cur_idx[type] = 0;
+
+  // Create msgq publisher for each of the `name` + type combos
+  // TODO: compute port number directly if using zmq
+  sockets[type] = PubSocket::create(msg_ctx, get_endpoint_name(name, type), false);
+}
+
+
+void VisionIpcServer::start_listener(){
+  listener_thread = std::thread(&VisionIpcServer::listener, this);
+}
+
+
+void VisionIpcServer::listener(){
+  std::cout << "Starting listener for: " << name << std::endl;
+
+  std::string path = "/tmp/visionipc_" + name;
+  int sock = ipc_bind(path.c_str());
+  assert(sock >= 0);
+
+  while (!should_exit){
+    // Wait for incoming connection
+    struct pollfd polls[1] = {{0}};
+    polls[0].fd = sock;
+    polls[0].events = POLLIN;
+
+    int ret = poll(polls, 1, 100);
+    if (ret < 0) {
+      if (errno == EINTR || errno == EAGAIN) continue;
+      std::cout << "poll failed, stopping listener" << std::endl;
+      break;
+    }
+
+    if (should_exit) break;
+    if (!polls[0].revents) {
+      continue;
+    }
+
+    // Handle incoming request
+    int fd = accept(sock, NULL, NULL);
+    assert(fd >= 0);
+
+    VisionStreamType type = VisionStreamType::VISION_STREAM_MAX;
+    int r = ipc_sendrecv_with_fds(false, fd, &type, sizeof(type), nullptr, 0, nullptr);
+    assert(r == sizeof(type));
+    if (buffers.count(type) <= 0) {
+      std::cout << "got request for invalid buffer type: " << type << std::endl;
+      close(fd);
+      continue;
+    }
+
+    int fds[VISIONIPC_MAX_FDS];
+    int num_fds = buffers[type].size();
+    VisionBuf bufs[VISIONIPC_MAX_FDS];
+
+    for (int i = 0; i < num_fds; i++){
+      fds[i] = buffers[type][i]->fd;
+      bufs[i] = *buffers[type][i];
+
+      // Remove some private openCL/ion metadata
+      bufs[i].buf_cl = 0;
+      bufs[i].copy_q = 0;
+      bufs[i].handle = 0;
+
+      bufs[i].server_id = server_id;
+    }
+
+    r = ipc_sendrecv_with_fds(true, fd, &bufs, sizeof(VisionBuf) * num_fds, fds, num_fds, nullptr);
+
+    close(fd);
+  }
+
+  std::cout << "Stopping listener for: " << name << std::endl;
+  close(sock);
+}
+
+
+
+VisionBuf * VisionIpcServer::get_buffer(VisionStreamType type){
+  // Do we want to keep track if the buffer has been sent out yet and warn user?
+  assert(buffers.count(type));
+  auto b = buffers[type];
+  return b[cur_idx[type]++ % b.size()];
+}
+
+void VisionIpcServer::send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync){
+  if (sync) {
+    if (buf->sync(VISIONBUF_SYNC_FROM_DEVICE) != 0) {
+      LOGE("Failed to sync buffer");
+    }
+  }
+  assert(buffers.count(buf->type));
+  assert(buf->idx < buffers[buf->type].size());
+
+  // Send over correct msgq socket
+  VisionIpcPacket packet = {0};
+  packet.server_id = server_id;
+  packet.idx = buf->idx;
+  packet.extra = *extra;
+
+  sockets[buf->type]->send((char*)&packet, sizeof(packet));
+}
+
+VisionIpcServer::~VisionIpcServer(){
+  should_exit = true;
+  listener_thread.join();
+
+  // VisionBuf cleanup
+  for( auto const& [type, buf] : buffers ) {
+    for (VisionBuf* b : buf){
+      if (b->free() != 0) {
+        LOGE("Failed to free buffer");
+      }
+      delete b;
+    }
+  }
+
+  // Messaging cleanup
+  for( auto const& [type, sock] : sockets ) {
+    delete sock;
+  }
+  delete msg_ctx;
+}

cereal/visionipc/visionipc_server.h

@@ -5,11 +5,11 @@
 #include <atomic>
 #include <map>
 
-#include "msgq/ipc.h"
-#include "msgq/visionipc/visionbuf.h"
+#include "messaging/messaging.h"
+#include "visionipc/visionipc.h"
+#include "visionipc/visionbuf.h"
 
 std::string get_endpoint_name(std::string name, VisionStreamType type);
-std::string get_ipc_path(const std::string &name);
 
 class VisionIpcServer {
  private:
@@ -23,6 +23,7 @@ class VisionIpcServer {
 
   std::map<VisionStreamType, std::atomic<size_t> > cur_idx;
   std::map<VisionStreamType, std::vector<VisionBuf*> > buffers;
+  std::map<VisionStreamType, std::map<VisionBuf*, size_t> > idxs;
 
   Context * msg_ctx;
   std::map<VisionStreamType, PubSocket*> sockets;
@@ -36,7 +37,6 @@ class VisionIpcServer {
   VisionBuf * get_buffer(VisionStreamType type);
 
   void create_buffers(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height);
-  void create_buffers_with_sizes(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset);
   void send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync=true);
   void start_listener();
 };

cereal/visionipc/visionipc_tests.cc

@@ -2,10 +2,8 @@
 #include <chrono>
 
 #include "catch2/catch.hpp"
-
-#include "msgq/visionipc/visionipc_server.h"
-#include "msgq/visionipc/visionipc_client.h"
-
+#include "visionipc_server.h"
+#include "visionipc_client.h"
 
 static void zmq_sleep(int milliseconds=1000){
   if (messaging_use_zmq()){
@@ -24,17 +22,6 @@ TEST_CASE("Connecting"){
   REQUIRE(client.connected);
 }
 
-TEST_CASE("getAvailableStreams"){
-  VisionIpcServer server("camerad");
-  server.create_buffers(VISION_STREAM_ROAD, 1, false, 100, 100);
-  server.create_buffers(VISION_STREAM_WIDE_ROAD, 1, false, 100, 100);
-  server.start_listener();
-  auto available_streams = VisionIpcClient::getAvailableStreams("camerad");
-  REQUIRE(available_streams.size() == 2);
-  REQUIRE(available_streams.count(VISION_STREAM_ROAD) == 1);
-  REQUIRE(available_streams.count(VISION_STREAM_WIDE_ROAD) == 1);
-}
-
 TEST_CASE("Check buffers"){
   size_t width = 100, height = 200, num_buffers = 5;
   VisionIpcServer server("camerad");
@@ -53,11 +40,11 @@ TEST_CASE("Check buffers"){
 TEST_CASE("Check yuv/rgb"){
   VisionIpcServer server("camerad");
   server.create_buffers(VISION_STREAM_ROAD, 1, false, 100, 100);
-  server.create_buffers(VISION_STREAM_MAP, 1, true, 100, 100);
+  server.create_buffers(VISION_STREAM_RGB_BACK, 1, true, 100, 100);
   server.start_listener();
 
   VisionIpcClient client_yuv = VisionIpcClient("camerad", VISION_STREAM_ROAD, false);
-  VisionIpcClient client_rgb = VisionIpcClient("camerad", VISION_STREAM_MAP, false);
+  VisionIpcClient client_rgb = VisionIpcClient("camerad", VISION_STREAM_RGB_BACK, false);
   client_yuv.connect();
   client_rgb.connect();
 

common/SConscript

@@ -0,0 +1,4 @@
+Import('envCython', 'common')
+
+envCython.Program('clock.so', 'clock.pyx')
+envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [common, 'zmq'])

common/api/__init__.py

@@ -2,18 +2,15 @@ import jwt
 import os
 import requests
 from datetime import datetime, timedelta
-from openpilot.system.hardware.hw import Paths
-from openpilot.system.version import get_version
-
-from openpilot.frogpilot.common.frogpilot_utilities import use_konik_server
+from common.basedir import PERSIST
+from selfdrive.version import get_version
 
 API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
-KONIK_API_HOST = os.getenv('API_HOST', 'https://api.konik.ai')
 
-class Api:
+class Api():
   def __init__(self, dongle_id):
     self.dongle_id = dongle_id
-    with open(Paths.persist_root()+'/comma/id_rsa') as f:
+    with open(PERSIST+'/comma/id_rsa') as f:
       self.private_key = f.read()
 
   def get(self, *args, **kwargs):
@@ -25,13 +22,13 @@ class Api:
   def request(self, method, endpoint, timeout=None, access_token=None, **params):
     return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
 
-  def get_token(self, expiry_hours=1):
+  def get_token(self):
     now = datetime.utcnow()
     payload = {
       'identity': self.dongle_id,
       'nbf': now,
       'iat': now,
-      'exp': now + timedelta(hours=expiry_hours)
+      'exp': now + timedelta(hours=1)
     }
     token = jwt.encode(payload, self.private_key, algorithm='RS256')
     if isinstance(token, bytes):
@@ -46,4 +43,4 @@ def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
 
   headers['User-Agent'] = "openpilot-" + get_version()
 
-  return requests.request(method, (KONIK_API_HOST if use_konik_server() else API_HOST) + "/" + endpoint, timeout=timeout, headers=headers, params=params)
+  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)

common/basedir.py

@@ -1,4 +1,11 @@
 import os
+from pathlib import Path
 
+from selfdrive.hardware import PC
 
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
+
+if PC:
+  PERSIST = os.path.join(str(Path.home()), ".comma", "persist")
+else:
+  PERSIST = "/persist"

common/clock.pyx

@@ -0,0 +1,24 @@
+# distutils: language = c++
+# cython: language_level = 3
+from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
+
+IF UNAME_SYSNAME == "Darwin":
+  # Darwin doesn't have a CLOCK_BOOTTIME
+  CLOCK_BOOTTIME = CLOCK_MONOTONIC_RAW
+ELSE:
+  from posix.time cimport CLOCK_BOOTTIME
+
+cdef double readclock(clockid_t clock_id):
+  cdef timespec ts
+  cdef double current
+
+  clock_gettime(clock_id, &ts)
+  current = ts.tv_sec + (ts.tv_nsec / 1000000000.)
+  return current
+
+def monotonic_time():
+  return readclock(CLOCK_MONOTONIC_RAW)
+
+def sec_since_boot():
+  return readclock(CLOCK_BOOTTIME)
+

common/conversions.py

@@ -1,27 +0,0 @@
-import numpy as np
-
-class Conversions:
-  # Speed
-  MPH_TO_KPH = 1.609344
-  KPH_TO_MPH = 1. / MPH_TO_KPH
-  MS_TO_KPH = 3.6
-  KPH_TO_MS = 1. / MS_TO_KPH
-  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
-  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
-  MS_TO_KNOTS = 1.9438
-  KNOTS_TO_MS = 1. / MS_TO_KNOTS
-
-  # Distance
-  METER_TO_FOOT = 3.28084
-  FOOT_TO_METER = 1. / METER_TO_FOOT
-  METER_TO_MILE = METER_TO_FOOT / 5280
-  MILE_TO_METER = 1. / METER_TO_MILE
-  CM_TO_INCH = 1. / 2.54
-  INCH_TO_CM = 1. / CM_TO_INCH
-
-  # Angle
-  DEG_TO_RAD = np.pi / 180.
-  RAD_TO_DEG = 1. / DEG_TO_RAD
-
-  # Mass
-  LB_TO_KG = 0.453592

common/ffi_wrapper.py

@@ -1,8 +1,55 @@
+import os
+import sys
+import fcntl
+import hashlib
 import platform
-
+from cffi import FFI
 
 def suffix():
   if platform.system() == "Darwin":
     return ".dylib"
   else:
     return ".so"
+
+def ffi_wrap(name, c_code, c_header, tmpdir="/tmp/ccache", cflags="", libraries=None):
+  if libraries is None:
+    libraries = []
+
+  cache = name + "_" + hashlib.sha1(c_code.encode('utf-8')).hexdigest()
+  try:
+    os.mkdir(tmpdir)
+  except OSError:
+    pass
+
+  fd = os.open(tmpdir, 0)
+  fcntl.flock(fd, fcntl.LOCK_EX)
+  try:
+    sys.path.append(tmpdir)
+    try:
+      mod = __import__(cache)
+    except Exception:
+      print(f"cache miss {cache}")
+      compile_code(cache, c_code, c_header, tmpdir, cflags, libraries)
+      mod = __import__(cache)
+  finally:
+    os.close(fd)
+
+  return mod.ffi, mod.lib
+
+
+def compile_code(name, c_code, c_header, directory, cflags="", libraries=None):
+  if libraries is None:
+    libraries = []
+
+  ffibuilder = FFI()
+  ffibuilder.set_source(name, c_code, source_extension='.cpp', libraries=libraries)
+  ffibuilder.cdef(c_header)
+  os.environ['OPT'] = "-fwrapv -O2 -DNDEBUG -std=c++1z"
+  os.environ['CFLAGS'] = cflags
+  ffibuilder.compile(verbose=True, debug=False, tmpdir=directory)
+
+
+def wrap_compiled(name, directory):
+  sys.path.append(directory)
+  mod = __import__(name)
+  return mod.ffi, mod.lib

common/file_helpers.py

@@ -1,6 +1,60 @@
 import os
+import shutil
 import tempfile
-import contextlib
+from atomicwrites import AtomicWriter
+
+
+def mkdirs_exists_ok(path):
+  if path.startswith('http://') or path.startswith('https://'):
+    raise ValueError('URL path')
+  try:
+    os.makedirs(path)
+  except OSError:
+    if not os.path.isdir(path):
+      raise
+
+
+def rm_not_exists_ok(path):
+  try:
+    os.remove(path)
+  except OSError:
+    if os.path.exists(path):
+      raise
+
+
+def rm_tree_or_link(path):
+  if os.path.islink(path):
+    os.unlink(path)
+  elif os.path.isdir(path):
+    shutil.rmtree(path)
+
+
+def get_tmpdir_on_same_filesystem(path):
+  normpath = os.path.normpath(path)
+  parts = normpath.split("/")
+  if len(parts) > 1 and parts[1] == "scratch":
+    return "/scratch/tmp"
+  elif len(parts) > 2 and parts[2] == "runner":
+    return f"/{parts[1]}/runner/tmp"
+  return "/tmp"
+
+
+class NamedTemporaryDir():
+  def __init__(self, temp_dir=None):
+    self._path = tempfile.mkdtemp(dir=temp_dir)
+
+  @property
+  def name(self):
+    return self._path
+
+  def close(self):
+    shutil.rmtree(self._path)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
 
 
 class CallbackReader:
@@ -22,16 +76,38 @@ class CallbackReader:
     return chunk
 
 
-@contextlib.contextmanager
-def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
-                        overwrite: bool = False):
-  """Write to a file atomically using a temporary file in the same directory as the destination file."""
-  dir_name = os.path.dirname(path)
+def _get_fileobject_func(writer, temp_dir):
+  def _get_fileobject():
+    return writer.get_fileobject(dir=temp_dir)
+  return _get_fileobject
 
-  if not overwrite and os.path.exists(path):
-    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+def monkeypatch_os_link():
+  # This is neccesary on EON/C2, where os.link is patched out of python
+  if not hasattr(os, 'link'):
+    from cffi import FFI
+    ffi = FFI()
+    ffi.cdef("int link(const char *oldpath, const char *newpath);")
+    libc = ffi.dlopen(None)
 
-  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
-    yield tmp_file
-    tmp_file_name = tmp_file.name
-  os.replace(tmp_file_name, path)
+    def link(src, dest):
+      return libc.link(src.encode(), dest.encode())
+    os.link = link
+
+def atomic_write_on_fs_tmp(path, **kwargs):
+  """Creates an atomic writer using a temporary file in a temporary directory
+     on the same filesystem as path.
+  """
+  # TODO(mgraczyk): This use of AtomicWriter relies on implementation details to set the temp
+  #                 directory.
+  monkeypatch_os_link()
+  writer = AtomicWriter(path, **kwargs)
+  return writer._open(_get_fileobject_func(writer, get_tmpdir_on_same_filesystem(path)))
+
+
+def atomic_write_in_dir(path, **kwargs):
+  """Creates an atomic writer using a temporary file in the same directory
+     as the destination file.
+  """
+  monkeypatch_os_link()
+  writer = AtomicWriter(path, **kwargs)
+  return writer._open(_get_fileobject_func(writer, os.path.dirname(path)))

common/git.py

@@ -1,42 +0,0 @@
-from functools import cache
-import subprocess
-from openpilot.common.run import run_cmd, run_cmd_default
-
-
-@cache
-def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
-  return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
-
-
-@cache
-def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
-  return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
-
-
-@cache
-def get_short_branch(cwd: str = None) -> str:
-  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
-
-
-@cache
-def get_branch(cwd: str = None) -> str:
-  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
-
-
-@cache
-def get_origin(cwd: str = None) -> str:
-  try:
-    local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
-    tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
-    return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
-  except subprocess.CalledProcessError:  # Not on a branch, fallback
-    return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
-
-
-@cache
-def get_normalized_origin(cwd: str = None) -> str:
-  return get_origin(cwd) \
-    .replace("git@", "", 1) \
-    .replace(".git", "", 1) \
-    .replace("https://", "", 1) \
-    .replace(":", "/", 1)

common/gpio.cc

@@ -1,84 +0,0 @@
-#include "common/gpio.h"
-
-#include <string>
-
-#ifdef __APPLE__
-int gpio_init(int pin_nr, bool output) {
-  return 0;
-}
-
-int gpio_set(int pin_nr, bool high) {
-  return 0;
-}
-
-int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
-  return 0;
-}
-
-#else
-
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <cstring>
-#include <linux/gpio.h>
-#include <sys/ioctl.h>
-
-#include "common/util.h"
-#include "common/swaglog.h"
-
-int gpio_init(int pin_nr, bool output) {
-  char pin_dir_path[50];
-  int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
-                           "/sys/class/gpio/gpio%d/direction", pin_nr);
-  if (pin_dir_path_len <= 0) {
-    return -1;
-  }
-  const char *value = output ? "out" : "in";
-  return util::write_file(pin_dir_path, (void*)value, strlen(value));
-}
-
-int gpio_set(int pin_nr, bool high) {
-  char pin_val_path[50];
-  int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
-                           "/sys/class/gpio/gpio%d/value", pin_nr);
-  if (pin_val_path_len <= 0) {
-    return -1;
-  }
-  return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);
-}
-
-int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
-
-  // Assumed that all interrupt pins are unexported and rights are given to
-  // read from gpiochip0.
-  std::string gpiochip_path = "/dev/gpiochip" + std::to_string(gpiochiop_id);
-  int fd = open(gpiochip_path.c_str(), O_RDONLY);
-  if (fd < 0) {
-    LOGE("Error opening gpiochip0 fd");
-    return -1;
-  }
-
-  // Setup event
-  struct gpioevent_request rq;
-  rq.lineoffset = pin_nr;
-  rq.handleflags = GPIOHANDLE_REQUEST_INPUT;
-
-  /* Requesting both edges as the data ready pulse from the lsm6ds sensor is
-     very short(75us) and is mostly detected as falling edge instead of rising.
-     So if it is detected as rising the following falling edge is skipped. */
-  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
-
-  strncpy(rq.consumer_label, consumer_label, std::size(rq.consumer_label) - 1);
-  int ret = util::safe_ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
-  if (ret == -1) {
-    LOGE("Unable to get line event from ioctl : %s", strerror(errno));
-    close(fd);
-    return -1;
-  }
-
-  close(fd);
-  return rq.fd;
-}
-
-#endif

common/gpio.h

@@ -1,33 +0,0 @@
-#pragma once
-
-// Pin definitions
-#ifdef QCOM2
-  #define GPIO_HUB_RST_N        30
-  #define GPIO_UBLOX_RST_N      32
-  #define GPIO_UBLOX_SAFEBOOT_N 33
-  #define GPIO_GNSS_PWR_EN      34 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
-  #define GPIO_STM_RST_N        124
-  #define GPIO_STM_BOOT0        134
-  #define GPIO_BMX_ACCEL_INT    21
-  #define GPIO_BMX_GYRO_INT     23
-  #define GPIO_BMX_MAGN_INT     87
-  #define GPIO_LSM_INT          84
-  #define GPIOCHIP_INT          0
-#else
-  #define GPIO_HUB_RST_N        0
-  #define GPIO_UBLOX_RST_N      0
-  #define GPIO_UBLOX_SAFEBOOT_N 0
-  #define GPIO_GNSS_PWR_EN      0 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
-  #define GPIO_STM_RST_N        0
-  #define GPIO_STM_BOOT0        0
-  #define GPIO_BMX_ACCEL_INT    0
-  #define GPIO_BMX_GYRO_INT     0
-  #define GPIO_BMX_MAGN_INT     0
-  #define GPIO_LSM_INT          0
-  #define GPIOCHIP_INT          0
-#endif
-
-int gpio_init(int pin_nr, bool output);
-int gpio_set(int pin_nr, bool high);
-
-int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr);

common/gpio.py

@@ -1,54 +1,14 @@
-import os
-from functools import cache
-
-def gpio_init(pin: int, output: bool) -> None:
+def gpio_init(pin, output):
   try:
     with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
       f.write(b"out" if output else b"in")
   except Exception as e:
     print(f"Failed to set gpio {pin} direction: {e}")
 
-def gpio_set(pin: int, high: bool) -> None:
+
+def gpio_set(pin, high):
   try:
     with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
       f.write(b"1" if high else b"0")
   except Exception as e:
     print(f"Failed to set gpio {pin} value: {e}")
-
-def gpio_read(pin: int) -> bool | None:
-  val = None
-  try:
-    with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
-      val = bool(int(f.read().strip()))
-  except Exception as e:
-    print(f"Failed to set gpio {pin} value: {e}")
-
-  return val
-
-def gpio_export(pin: int) -> None:
-  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
-    return
-
-  try:
-    with open("/sys/class/gpio/export", 'w') as f:
-      f.write(str(pin))
-  except Exception:
-    print(f"Failed to export gpio {pin}")
-
-@cache
-def get_irq_action(irq: int) -> list[str]:
-  try:
-    with open(f"/sys/kernel/irq/{irq}/actions") as f:
-      actions = f.read().strip().split(',')
-      return actions
-  except FileNotFoundError:
-    return []
-
-def get_irqs_for_action(action: str) -> list[str]:
-  ret = []
-  with open("/proc/interrupts") as f:
-    for l in f.readlines():
-      irq = l.split(':')[0].strip()
-      if irq.isdigit() and action in get_irq_action(irq):
-        ret.append(irq)
-  return ret

common/kalman/.gitignore

@@ -0,0 +1 @@
+simple_kalman_impl.c

common/kalman/SConscript

@@ -0,0 +1,3 @@
+Import('envCython')
+
+envCython.Program('simple_kalman_impl.so', 'simple_kalman_impl.pyx')

common/kalman/simple_kalman.py

@@ -0,0 +1,3 @@
+# pylint: skip-file
+from common.kalman.simple_kalman_impl import KF1D as KF1D
+assert KF1D

common/kalman/simple_kalman_impl.pxd

@@ -0,0 +1,18 @@
+# cython: language_level = 3
+
+cdef class KF1D:
+  cdef public:
+    double x0_0
+    double x1_0
+    double K0_0
+    double K1_0
+    double A0_0
+    double A0_1
+    double A1_0
+    double A1_1
+    double C0_0
+    double C0_1
+    double A_K_0
+    double A_K_1
+    double A_K_2
+    double A_K_3

common/kalman/simple_kalman_impl.pyx

@@ -0,0 +1,37 @@
+# distutils: language = c++
+# cython: language_level=3
+
+cdef class KF1D:
+  def __init__(self, x0, A, C, K):
+    self.x0_0 = x0[0][0]
+    self.x1_0 = x0[1][0]
+    self.A0_0 = A[0][0]
+    self.A0_1 = A[0][1]
+    self.A1_0 = A[1][0]
+    self.A1_1 = A[1][1]
+    self.C0_0 = C[0]
+    self.C0_1 = C[1]
+    self.K0_0 = K[0][0]
+    self.K1_0 = K[1][0]
+
+    self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
+    self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
+    self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
+    self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
+
+  def update(self, meas):
+    cdef double x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
+    cdef double x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
+    self.x0_0 = x0_0
+    self.x1_0 = x1_0
+
+    return [self.x0_0, self.x1_0]
+
+  @property
+  def x(self):
+    return [[self.x0_0], [self.x1_0]]
+
+  @x.setter
+  def x(self, x):
+    self.x0_0 = x[0][0]
+    self.x1_0 = x[1][0]

common/kalman/simple_kalman_old.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+
+class KF1D:
+  # this EKF assumes constant covariance matrix, so calculations are much simpler
+  # the Kalman gain also needs to be precomputed using the control module
+
+  def __init__(self, x0, A, C, K):
+    self.x = x0
+    self.A = A
+    self.C = np.atleast_2d(C)
+    self.K = K
+
+    self.A_K = self.A - np.dot(self.K, self.C)
+
+    # K matrix needs to  be pre-computed as follow:
+    # import control
+    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
+    # self.K = np.transpose(K)
+
+  def update(self, meas):
+    self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
+    return self.x

common/kalman/tests/test_simple_kalman.py

@@ -0,0 +1,87 @@
+import unittest
+import random
+import timeit
+import numpy as np
+
+from common.kalman.simple_kalman import KF1D
+from common.kalman.simple_kalman_old import KF1D as KF1D_old
+
+
+class TestSimpleKalman(unittest.TestCase):
+  def setUp(self):
+    dt = 0.01
+    x0_0 = 0.0
+    x1_0 = 0.0
+    A0_0 = 1.0
+    A0_1 = dt
+    A1_0 = 0.0
+    A1_1 = 1.0
+    C0_0 = 1.0
+    C0_1 = 0.0
+    K0_0 = 0.12287673
+    K1_0 = 0.29666309
+
+    self.kf_old = KF1D_old(x0=np.array([[x0_0], [x1_0]]),
+                           A=np.array([[A0_0, A0_1], [A1_0, A1_1]]),
+                           C=np.array([C0_0, C0_1]),
+                           K=np.array([[K0_0], [K1_0]]))
+
+    self.kf = KF1D(x0=[[x0_0], [x1_0]],
+                   A=[[A0_0, A0_1], [A1_0, A1_1]],
+                   C=[C0_0, C0_1],
+                   K=[[K0_0], [K1_0]])
+
+  def test_getter_setter(self):
+    self.kf.x = [[1.0], [1.0]]
+    self.assertEqual(self.kf.x, [[1.0], [1.0]])
+
+  def update_returns_state(self):
+      x = self.kf.update(100)
+      self.assertEqual(x, self.kf.x)
+
+  def test_old_equal_new(self):
+    for _ in range(1000):
+      v_wheel = random.uniform(0, 200)
+
+      x_old = self.kf_old.update(v_wheel)
+      x = self.kf.update(v_wheel)
+
+      # Compare the output x, verify that the error is less than 1e-4
+      np.testing.assert_almost_equal(x_old[0], x[0])
+      np.testing.assert_almost_equal(x_old[1], x[1])
+
+  def test_new_is_faster(self):
+    setup = """
+import numpy as np
+
+from common.kalman.simple_kalman import KF1D
+from common.kalman.simple_kalman_old import KF1D as KF1D_old
+
+dt = 0.01
+x0_0 = 0.0
+x1_0 = 0.0
+A0_0 = 1.0
+A0_1 = dt
+A1_0 = 0.0
+A1_1 = 1.0
+C0_0 = 1.0
+C0_1 = 0.0
+K0_0 = 0.12287673
+K1_0 = 0.29666309
+
+kf_old = KF1D_old(x0=np.array([[x0_0], [x1_0]]),
+                  A=np.array([[A0_0, A0_1], [A1_0, A1_1]]),
+                  C=np.array([C0_0, C0_1]),
+                  K=np.array([[K0_0], [K1_0]]))
+
+kf = KF1D(x0=[[x0_0], [x1_0]],
+          A=[[A0_0, A0_1], [A1_0, A1_1]],
+          C=[C0_0, C0_1],
+          K=[[K0_0], [K1_0]])
+    """
+    kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000)
+    kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000)
+    self.assertTrue(kf_speed < kf_old_speed / 4)
+
+if __name__ == "__main__":
+  unittest.main()

common/logging_extra.py

@@ -3,7 +3,6 @@ import os
 import sys
 import copy
 import json
-import time
 import uuid
 import socket
 import logging
@@ -12,8 +11,6 @@ from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
 
-LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
-
 def json_handler(obj):
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
@@ -153,9 +150,9 @@ class SwagLogger(logging.Logger):
   def bind_global(self, **kwargs):
     self.global_ctx.update(kwargs)
 
-  def event(self, event, *args, **kwargs):
+  def event(self, event_name, *args, **kwargs):
     evt = NiceOrderedDict()
-    evt['event'] = event
+    evt['event'] = event_name
     if args:
       evt['args'] = args
     evt.update(kwargs)
@@ -166,15 +163,6 @@ class SwagLogger(logging.Logger):
     else:
       self.info(evt)
 
-  def timestamp(self, event_name):
-    if LOG_TIMESTAMPS:
-      t = time.monotonic()
-      tstp = NiceOrderedDict()
-      tstp['timestamp'] = NiceOrderedDict()
-      tstp['timestamp']["event"] = event_name
-      tstp['timestamp']["time"] = t*1e9
-      self.debug(tstp)
-
   def findCaller(self, stack_info=False, stacklevel=1):
     """
     Find the stack frame of the caller so that we can note the source
@@ -184,33 +172,33 @@ class SwagLogger(logging.Logger):
     #On some versions of IronPython, currentframe() returns None if
     #IronPython isn't run with -X:Frames.
     if f is not None:
-      f = f.f_back
+        f = f.f_back
     orig_f = f
     while f and stacklevel > 1:
-      f = f.f_back
-      stacklevel -= 1
+        f = f.f_back
+        stacklevel -= 1
     if not f:
-      f = orig_f
+        f = orig_f
     rv = "(unknown file)", 0, "(unknown function)", None
     while hasattr(f, "f_code"):
-      co = f.f_code
-      filename = os.path.normcase(co.co_filename)
+        co = f.f_code
+        filename = os.path.normcase(co.co_filename)
 
-      # TODO: is this pylint exception correct?
-      if filename == _srcfile:
-        f = f.f_back
-        continue
-      sinfo = None
-      if stack_info:
-        sio = io.StringIO()
-        sio.write('Stack (most recent call last):\n')
-        traceback.print_stack(f, file=sio)
-        sinfo = sio.getvalue()
-        if sinfo[-1] == '\n':
-          sinfo = sinfo[:-1]
-        sio.close()
-      rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
-      break
+        # TODO: is this pylint exception correct?
+        if filename == _srcfile:  # pylint: disable=comparison-with-callable
+            f = f.f_back
+            continue
+        sinfo = None
+        if stack_info:
+            sio = io.StringIO()
+            sio.write('Stack (most recent call last):\n')
+            traceback.print_stack(f, file=sio)
+            sinfo = sio.getvalue()
+            if sinfo[-1] == '\n':
+                sinfo = sinfo[:-1]
+            sio.close()
+        rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
+        break
     return rv
 
 if __name__ == "__main__":

common/markdown.py

@@ -1,11 +1,14 @@
+from typing import List
+
 HTML_REPLACEMENTS = [
   (r'&', r'&'),
   (r'"', r'"'),
 ]
 
+
 def parse_markdown(text: str, tab_length: int = 2) -> str:
   lines = text.split("\n")
-  output: list[str] = []
+  output: List[str] = []
   list_level = 0
 
   def end_outstanding_lists(level: int, end_level: int) -> int:

common/mock/__init__.py

@@ -1,50 +0,0 @@
-"""
-Utilities for generating mock messages for testing.
-example in common/tests/test_mock.py
-"""
-
-
-import functools
-import threading
-from cereal.messaging import PubMaster
-from cereal.services import SERVICE_LIST
-from openpilot.common.mock.generators import generate_liveLocationKalman
-from openpilot.common.realtime import Ratekeeper
-
-
-MOCK_GENERATOR = {
-  "liveLocationKalman": generate_liveLocationKalman
-}
-
-
-def generate_messages_loop(services: list[str], done: threading.Event):
-  pm = PubMaster(services)
-  rk = Ratekeeper(100)
-  i = 0
-  while not done.is_set():
-    for s in services:
-      should_send = i % (100/SERVICE_LIST[s].frequency) == 0
-      if should_send:
-        message = MOCK_GENERATOR[s]()
-        pm.send(s, message)
-    i += 1
-    rk.keep_time()
-
-
-def mock_messages(services: list[str] | str):
-  if isinstance(services, str):
-    services = [services]
-
-  def decorator(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-      done = threading.Event()
-      t = threading.Thread(target=generate_messages_loop, args=(services, done))
-      t.start()
-      try:
-        return func(*args, **kwargs)
-      finally:
-        done.set()
-        t.join()
-    return wrapper
-  return decorator

common/mock/generators.py

@@ -1,20 +0,0 @@
-from cereal import messaging
-
-
-LOCATION1 = (32.7174, -117.16277)
-LOCATION2 = (32.7558, -117.2037)
-
-LLK_DECIMATION = 10
-RENDER_FRAMES = 15
-DEFAULT_ITERATIONS = RENDER_FRAMES * LLK_DECIMATION
-
-
-def generate_liveLocationKalman(location=LOCATION1):
-  msg = messaging.new_message('liveLocationKalman')
-  msg.liveLocationKalman.positionGeodetic = {'value': [*location, 0], 'std': [0., 0., 0.], 'valid': True}
-  msg.liveLocationKalman.positionECEF = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
-  msg.liveLocationKalman.calibratedOrientationNED = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
-  msg.liveLocationKalman.velocityCalibrated = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
-  msg.liveLocationKalman.status = 'valid'
-  msg.liveLocationKalman.gpsOK = True
-  return msg

common/params.cc

@@ -1,725 +0,0 @@
-#include "common/params.h"
-
-#include <dirent.h>
-#include <sys/file.h>
-
-#include <algorithm>
-#include <cassert>
-#include <csignal>
-#include <unordered_map>
-
-#include "common/queue.h"
-#include "common/swaglog.h"
-#include "common/util.h"
-#include "system/hardware/hw.h"
-
-namespace {
-
-volatile sig_atomic_t params_do_exit = 0;
-void params_sig_handler(int signal) {
-  params_do_exit = 1;
-}
-
-int fsync_dir(const std::string &path) {
-  int result = -1;
-  int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
-  if (fd >= 0) {
-    result = fsync(fd);
-    close(fd);
-  }
-  return result;
-}
-
-bool create_params_path(const std::string &param_path, const std::string &key_path) {
-  // Make sure params path exists
-  if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
-    return false;
-  }
-
-  // See if the symlink exists, otherwise create it
-  if (!util::file_exists(key_path)) {
-    // 1) Create temp folder
-    // 2) Symlink it to temp link
-    // 3) Move symlink to <params>/d
-
-    std::string tmp_path = param_path + "/.tmp_XXXXXX";
-    // this should be OK since mkdtemp just replaces characters in place
-    char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
-    if (tmp_dir == NULL) {
-      return false;
-    }
-
-    std::string link_path = std::string(tmp_dir) + ".link";
-    if (symlink(tmp_dir, link_path.c_str()) != 0) {
-      return false;
-    }
-
-    // don't return false if it has been created by other
-    if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
-  std::string params_path = path.empty() ? Path::params() : path;
-  if (!create_params_path(params_path, params_path + prefix)) {
-    throw std::runtime_error(util::string_format(
-        "Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
-        errno, params_path.c_str(), prefix.c_str()));
-  }
-  return params_path;
-}
-
-class FileLock {
-public:
-  FileLock(const std::string &fn) {
-    if (fn.rfind("/cache", 0) == 0 || fn.rfind("/data/params_backup", 0) == 0) {
-      return;
-    }
-
-    fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
-    if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
-      LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
-    }
-  }
-  ~FileLock() { close(fd_); }
-
-private:
-  int fd_ = -1;
-};
-
-std::unordered_map<std::string, uint32_t> keys = {
-    {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
-    {"AlwaysOnDM", PERSISTENT},
-    {"ApiCache_Device", PERSISTENT},
-    {"ApiCache_NavDestinations", PERSISTENT},
-    {"AssistNowToken", PERSISTENT},
-    {"AthenadPid", PERSISTENT},
-    {"AthenadUploadQueue", PERSISTENT},
-    {"AthenadRecentlyViewedRoutes", PERSISTENT},
-    {"BootCount", PERSISTENT},
-    {"CalibrationParams", PERSISTENT},
-    {"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
-    {"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
-    {"CarBatteryCapacity", PERSISTENT},
-    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"CarParamsCache", CLEAR_ON_MANAGER_START},
-    {"CarParamsPersistent", PERSISTENT},
-    {"CarParamsPrevRoute", PERSISTENT},
-    {"CarVin", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"CompletedTrainingVersion", PERSISTENT},
-    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"CurrentBootlog", PERSISTENT},
-    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"DisablePowerDown", PERSISTENT},
-    {"DisableUpdates", PERSISTENT},
-    {"DisengageOnAccelerator", PERSISTENT},
-    {"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
-    {"DongleId", PERSISTENT},
-    {"DoReboot", CLEAR_ON_MANAGER_START},
-    {"DoShutdown", CLEAR_ON_MANAGER_START},
-    {"DoUninstall", CLEAR_ON_MANAGER_START},
-    {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
-    {"ExperimentalMode", PERSISTENT},
-    {"ExperimentalModeConfirmed", PERSISTENT},
-    {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"ForcePowerDown", PERSISTENT},
-    {"GitBranch", PERSISTENT},
-    {"GitCommit", PERSISTENT},
-    {"GitCommitDate", PERSISTENT},
-    {"GitDiff", PERSISTENT},
-    {"GithubSshKeys", PERSISTENT},
-    {"GithubUsername", PERSISTENT},
-    {"GitRemote", PERSISTENT},
-    {"GsmApn", PERSISTENT},
-    {"GsmMetered", PERSISTENT},
-    {"GsmRoaming", PERSISTENT},
-    {"HardwareSerial", PERSISTENT},
-    {"HasAcceptedTerms", PERSISTENT},
-    {"IMEI", PERSISTENT},
-    {"InstallDate", PERSISTENT},
-    {"IsDriverViewEnabled", CLEAR_ON_ONROAD_TRANSITION},
-    {"IsEngaged", PERSISTENT},
-    {"IsLdwEnabled", PERSISTENT},
-    {"IsMetric", PERSISTENT},
-    {"IsOffroad", CLEAR_ON_MANAGER_START},
-    {"IsOnroad", PERSISTENT},
-    {"IsRhdDetected", PERSISTENT},
-    {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
-    {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
-    {"IsTestedBranch", CLEAR_ON_MANAGER_START},
-    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"LanguageSetting", PERSISTENT},
-    {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
-    {"LastGPSPosition", PERSISTENT},
-    {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
-    {"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
-    {"LastUpdateException", CLEAR_ON_MANAGER_START},
-    {"LastUpdateTime", PERSISTENT},
-    {"LiveDelay", PERSISTENT},
-    {"LiveParameters", PERSISTENT},
-    {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
-    {"LongitudinalPersonality", PERSISTENT},
-    {"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"NavDestinationWaypoints", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"NavPastDestinations", PERSISTENT},
-    {"NavSettingLeftSide", PERSISTENT},
-    {"NavSettingTime24h", PERSISTENT},
-    {"NetworkMetered", PERSISTENT},
-    {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
-    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
-    {"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
-    {"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
-    {"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
-    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
-    {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
-    {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
-    {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
-    {"OpenpilotEnabledToggle", PERSISTENT},
-    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"PandaSignatures", CLEAR_ON_MANAGER_START},
-    {"PrimeType", PERSISTENT},
-    {"RecordFront", PERSISTENT},
-    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
-    {"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"RouteCount", PERSISTENT},
-    {"SecOCKey", PERSISTENT | DONT_LOG},
-    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"SshEnabled", PERSISTENT},
-    {"TermsVersion", PERSISTENT},
-    {"Timezone", PERSISTENT},
-    {"TrainingVersion", PERSISTENT},
-    {"UbloxAvailable", PERSISTENT},
-    {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
-    {"UpdaterAvailableBranches", PERSISTENT},
-    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
-    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
-    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
-    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
-    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
-    {"UpdaterState", CLEAR_ON_MANAGER_START},
-    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
-    {"UpdaterLastFetchTime", PERSISTENT},
-    {"Version", PERSISTENT},
-
-    // FrogPilot parameters
-    {"AccelerationPath", PERSISTENT},
-    {"AccelerationProfile", PERSISTENT},
-    {"AdjacentLeadsUI", PERSISTENT},
-    {"AdjacentPath", PERSISTENT},
-    {"AdjacentPathMetrics", PERSISTENT},
-    {"AdvancedCustomUI", PERSISTENT},
-    {"AdvancedLateralTune", PERSISTENT},
-    {"AdvancedLongitudinalTune", PERSISTENT},
-    {"AggressiveFollow", PERSISTENT},
-    {"AggressiveJerkAcceleration", PERSISTENT},
-    {"AggressiveJerkDanger", PERSISTENT},
-    {"AggressiveJerkDeceleration", PERSISTENT},
-    {"AggressiveJerkSpeed", PERSISTENT},
-    {"AggressiveJerkSpeedDecrease", PERSISTENT},
-    {"AggressivePersonalityProfile", PERSISTENT},
-    {"AlertVolumeControl", PERSISTENT},
-    {"AlwaysOnLateral", PERSISTENT},
-    {"AlwaysOnLateralLKAS", PERSISTENT},
-    {"AlwaysOnLateralMain", PERSISTENT},
-    {"AMapKey1", PERSISTENT | DONT_LOG},
-    {"AMapKey2", PERSISTENT | DONT_LOG},
-    {"ApiCache_DriveStats", PERSISTENT},
-    {"AutomaticallyDownloadModels", PERSISTENT},
-    {"AutomaticUpdates", PERSISTENT},
-    {"AvailableModelNames", PERSISTENT},
-    {"AvailableModels", PERSISTENT},
-    {"BigMap", PERSISTENT},
-    {"BlacklistedModels", PERSISTENT},
-    {"BlindSpotMetrics", PERSISTENT},
-    {"BlindSpotPath", PERSISTENT},
-    {"BorderMetrics", PERSISTENT},
-    {"CalibratedLateralAcceleration", PERSISTENT},
-    {"CalibrationProgress", PERSISTENT},
-    {"CameraView", PERSISTENT},
-    {"CancelModelDownload", CLEAR_ON_MANAGER_START},
-    {"CancelThemeDownload", CLEAR_ON_MANAGER_START},
-    {"CarMake", PERSISTENT},
-    {"CarModel", PERSISTENT},
-    {"CarModelName", PERSISTENT},
-    {"CECurves", PERSISTENT},
-    {"CECurvesLead", PERSISTENT},
-    {"CELead", PERSISTENT},
-    {"CEModelStopTime", PERSISTENT},
-    {"CENavigation", PERSISTENT},
-    {"CENavigationIntersections", PERSISTENT},
-    {"CENavigationLead", PERSISTENT},
-    {"CENavigationTurns", PERSISTENT},
-    {"CESignalSpeed", PERSISTENT},
-    {"CESignalLaneDetection", PERSISTENT},
-    {"CESlowerLead", PERSISTENT},
-    {"CESpeed", PERSISTENT},
-    {"CESpeedLead", PERSISTENT},
-    {"CEStatus", CLEAR_ON_OFFROAD_TRANSITION},
-    {"CEStoppedLead", PERSISTENT},
-    {"ClusterOffset", PERSISTENT},
-    {"ColorToDownload", CLEAR_ON_MANAGER_START},
-    {"Compass", PERSISTENT},
-    {"ConditionalExperimental", PERSISTENT},
-    {"CurvatureData", PERSISTENT | DONT_LOG},
-    {"CurveSpeedController", PERSISTENT},
-    {"CustomAlerts", PERSISTENT},
-    {"CustomColors", PERSISTENT},
-    {"CustomCruise", PERSISTENT},
-    {"CustomCruiseLong", PERSISTENT},
-    {"CustomDistanceIcons", PERSISTENT},
-    {"CustomIcons", PERSISTENT},
-    {"CustomPersonalities", PERSISTENT},
-    {"CustomSignals", PERSISTENT},
-    {"CustomSounds", PERSISTENT},
-    {"CustomUI", PERSISTENT},
-    {"DebugMode", CLEAR_ON_OFFROAD_TRANSITION},
-    {"DecelerationProfile", PERSISTENT},
-    {"DeveloperMetrics", PERSISTENT},
-    {"DeveloperSidebar", PERSISTENT},
-    {"DeveloperSidebarMetric1", PERSISTENT},
-    {"DeveloperSidebarMetric2", PERSISTENT},
-    {"DeveloperSidebarMetric3", PERSISTENT},
-    {"DeveloperSidebarMetric4", PERSISTENT},
-    {"DeveloperSidebarMetric5", PERSISTENT},
-    {"DeveloperSidebarMetric6", PERSISTENT},
-    {"DeveloperSidebarMetric7", PERSISTENT},
-    {"DeveloperWidgets", PERSISTENT},
-    {"DeveloperUI", PERSISTENT},
-    {"DeviceManagement", PERSISTENT},
-    {"DeviceShutdown", PERSISTENT},
-    {"DisableOnroadUploads", PERSISTENT},
-    {"DisableOpenpilotLongitudinal", PERSISTENT},
-    {"DiscordUsername", PERSISTENT},
-    {"DistanceButtonControl", PERSISTENT},
-    {"DistanceIconToDownload", CLEAR_ON_MANAGER_START},
-    {"DisengageVolume", PERSISTENT},
-    {"DoToggleReset", PERSISTENT},
-    {"DoToggleResetStock", PERSISTENT},
-    {"DownloadableColors", PERSISTENT},
-    {"DownloadableDistanceIcons", PERSISTENT},
-    {"DownloadableIcons", PERSISTENT},
-    {"DownloadableSignals", PERSISTENT},
-    {"DownloadableSounds", PERSISTENT},
-    {"DownloadableWheels", PERSISTENT},
-    {"DownloadAllModels", CLEAR_ON_MANAGER_START},
-    {"DriverCamera", PERSISTENT},
-    {"DynamicPathWidth", PERSISTENT},
-    {"DynamicPedalsOnUI", PERSISTENT},
-    {"EngageVolume", PERSISTENT},
-    {"ExperimentalGMTune", PERSISTENT},
-    {"Fahrenheit", PERSISTENT},
-    {"FavoriteDestinations", PERSISTENT | DONT_LOG},
-    {"FlashPanda", CLEAR_ON_MANAGER_START},
-    {"ForceAutoTune", PERSISTENT},
-    {"ForceAutoTuneOff", PERSISTENT},
-    {"ForceFingerprint", PERSISTENT},
-    {"ForceMPHDashboard", PERSISTENT},
-    {"ForceOffroad", CLEAR_ON_MANAGER_START},
-    {"ForceOnroad", CLEAR_ON_MANAGER_START},
-    {"ForceStops", PERSISTENT},
-    {"ForceTorqueController", PERSISTENT},
-    {"FPSCounter", PERSISTENT},
-    {"FrogPilotCarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
-    {"FrogPilotCarParamsPersistent", PERSISTENT},
-    {"FrogPilotDongleId", PERSISTENT | DONT_LOG},
-    {"FrogPilotDrives", PERSISTENT},
-    {"FrogPilotKilometers", PERSISTENT},
-    {"FrogPilotMinutes", PERSISTENT},
-    {"FrogPilotStats", PERSISTENT | DONT_LOG},
-    {"FrogPilotToggles", CLEAR_ON_MANAGER_START},
-    {"FrogPilotTogglesUpdated", CLEAR_ON_MANAGER_START},
-    {"FrogPilotTuningLevels", CLEAR_ON_MANAGER_START},
-    {"FrogsGoMoosTweak", PERSISTENT},
-    {"FullMap", PERSISTENT},
-    {"GasRegenCmd", PERSISTENT},
-    {"GoatScream", PERSISTENT},
-    {"GreenLightAlert", PERSISTENT},
-    {"HideAlerts", PERSISTENT},
-    {"HideLeadMarker", PERSISTENT},
-    {"HideMap", PERSISTENT},
-    {"HideMapIcon", PERSISTENT},
-    {"HideMaxSpeed", PERSISTENT},
-    {"HideSpeed", PERSISTENT},
-    {"HideSpeedLimit", PERSISTENT},
-    {"HigherBitrate", PERSISTENT},
-    {"HolidayThemes", PERSISTENT},
-    {"HumanAcceleration", PERSISTENT},
-    {"HumanFollowing", PERSISTENT},
-    {"IconToDownload", CLEAR_ON_MANAGER_START},
-    {"IncreasedStoppedDistance", PERSISTENT},
-    {"IncreaseThermalLimits", PERSISTENT},
-    {"IssueReported", CLEAR_ON_MANAGER_START},
-    {"KonikDongleId", PERSISTENT},
-    {"KonikMinutes", PERSISTENT},
-    {"LaneChanges", PERSISTENT},
-    {"LaneChangeTime", PERSISTENT},
-    {"LaneDetectionWidth", PERSISTENT},
-    {"LaneLinesWidth", PERSISTENT},
-    {"LastMapsUpdate", PERSISTENT},
-    {"LateralTune", PERSISTENT},
-    {"LeadDepartingAlert", PERSISTENT},
-    {"LeadDetectionThreshold", PERSISTENT},
-    {"LeadInfo", PERSISTENT},
-    {"LKASButtonControl", PERSISTENT},
-    {"LockDoors", PERSISTENT},
-    {"LockDoorsTimer", PERSISTENT},
-    {"LongDistanceButtonControl", PERSISTENT},
-    {"LongitudinalActuatorDelay", PERSISTENT},
-    {"LongitudinalActuatorDelayStock", PERSISTENT},
-    {"LongitudinalTune", PERSISTENT},
-    {"LongPitch", PERSISTENT},
-    {"LoudBlindspotAlert", PERSISTENT},
-    {"LowVoltageShutdown", PERSISTENT},
-    {"ManualUpdateInitiated", CLEAR_ON_MANAGER_START},
-    {"MapAcceleration", PERSISTENT},
-    {"MapboxPublicKey", PERSISTENT | DONT_LOG},
-    {"MapBoxRequests", PERSISTENT},
-    {"MapboxSecretKey", PERSISTENT | DONT_LOG},
-    {"MapDeceleration", PERSISTENT},
-    {"MapdLogLevel", CLEAR_ON_MANAGER_START},
-    {"MapGears", PERSISTENT},
-    {"MapsSelected", PERSISTENT},
-    {"MapSpeedLimit", CLEAR_ON_MANAGER_START},
-    {"MapStyle", PERSISTENT},
-    {"MaxDesiredAcceleration", PERSISTENT},
-    {"MinimumBackupSize", PERSISTENT},
-    {"MinimumLaneChangeSpeed", PERSISTENT},
-    {"Model", PERSISTENT},
-    {"ModelDownloadProgress", CLEAR_ON_MANAGER_START},
-    {"ModelDrivesAndScores", PERSISTENT},
-    {"ModelRandomizer", PERSISTENT},
-    {"ModelToDownload", CLEAR_ON_MANAGER_START},
-    {"ModelUI", PERSISTENT},
-    {"ModelVersions", PERSISTENT},
-    {"NavigationUI", PERSISTENT},
-    {"NextMapSpeedLimit", CLEAR_ON_MANAGER_START},
-    {"NewLongAPI", PERSISTENT},
-    {"NNFF", PERSISTENT},
-    {"NNFFLite", PERSISTENT},
-    {"NNFFModelName", CLEAR_ON_OFFROAD_TRANSITION},
-    {"NoLogging", PERSISTENT},
-    {"NoUploads", PERSISTENT},
-    {"NudgelessLaneChange", PERSISTENT},
-    {"NumericalTemp", PERSISTENT},
-    {"Offset1", PERSISTENT},
-    {"Offset2", PERSISTENT},
-    {"Offset3", PERSISTENT},
-    {"Offset4", PERSISTENT},
-    {"Offset5", PERSISTENT},
-    {"Offset6", PERSISTENT},
-    {"Offset7", PERSISTENT},
-    {"OneLaneChange", PERSISTENT},
-    {"OnroadDistanceButton", PERSISTENT},
-    {"OnroadDistanceButtonPressed", CLEAR_ON_MANAGER_START},
-    {"openpilotMinutes", PERSISTENT},
-    {"OSMDownloadBounds", PERSISTENT},
-    {"OSMDownloadLocations", PERSISTENT},
-    {"OSMDownloadProgress", CLEAR_ON_MANAGER_START},
-    {"OverpassRequests", PERSISTENT},
-    {"PathEdgeWidth", PERSISTENT},
-    {"PathWidth", PERSISTENT},
-    {"PauseAOLOnBrake", PERSISTENT},
-    {"PauseLateralOnSignal", PERSISTENT},
-    {"PauseLateralSpeed", PERSISTENT},
-    {"PedalsOnUI", PERSISTENT},
-    {"PersonalizeOpenpilot", PERSISTENT},
-    {"PreferredSchedule", PERSISTENT},
-    {"PreviousSpeedLimit", PERSISTENT},
-    {"PromptDistractedVolume", PERSISTENT},
-    {"PromptVolume", PERSISTENT},
-    {"QOLLateral", PERSISTENT},
-    {"QOLLongitudinal", PERSISTENT},
-    {"QOLVisuals", PERSISTENT},
-    {"RadarTracksUI", PERSISTENT},
-    {"RainbowPath", PERSISTENT},
-    {"RandomEvents", PERSISTENT},
-    {"RandomThemes", PERSISTENT},
-    {"RefuseVolume", PERSISTENT},
-    {"RelaxedFollow", PERSISTENT},
-    {"RelaxedJerkAcceleration", PERSISTENT},
-    {"RelaxedJerkDanger", PERSISTENT},
-    {"RelaxedJerkDeceleration", PERSISTENT},
-    {"RelaxedJerkSpeed", PERSISTENT},
-    {"RelaxedJerkSpeedDecrease", PERSISTENT},
-    {"RelaxedPersonalityProfile", PERSISTENT},
-    {"ReverseCruise", PERSISTENT},
-    {"RoadEdgesWidth", PERSISTENT},
-    {"RoadName", CLEAR_ON_MANAGER_START},
-    {"RoadNameUI", PERSISTENT},
-    {"RotatingWheel", PERSISTENT},
-    {"ScreenBrightness", PERSISTENT},
-    {"ScreenBrightnessOnroad", PERSISTENT},
-    {"ScreenManagement", PERSISTENT},
-    {"ScreenRecorder", PERSISTENT},
-    {"ScreenTimeout", PERSISTENT},
-    {"ScreenTimeoutOnroad", PERSISTENT},
-    {"SearchInput", PERSISTENT},
-    {"SecOCKeys", PERSISTENT | DONT_LOG},
-    {"SetSpeedLimit", PERSISTENT},
-    {"SetSpeedOffset", PERSISTENT},
-    {"ShowCEMStatus", PERSISTENT},
-    {"ShowCPU", PERSISTENT},
-    {"ShowCSCStatus", PERSISTENT},
-    {"ShowGPU", PERSISTENT},
-    {"ShowIP", PERSISTENT},
-    {"ShowMemoryUsage", PERSISTENT},
-    {"ShownToggleDescriptions", PERSISTENT},
-    {"ShowSLCOffset", PERSISTENT},
-    {"ShowSpeedLimits", PERSISTENT},
-    {"ShowSteering", PERSISTENT},
-    {"ShowStoppingPoint", PERSISTENT},
-    {"ShowStoppingPointMetrics", PERSISTENT},
-    {"ShowStorageLeft", PERSISTENT},
-    {"ShowStorageUsed", PERSISTENT},
-    {"Sidebar", PERSISTENT},
-    {"SignalMetrics", PERSISTENT},
-    {"SignalToDownload", CLEAR_ON_MANAGER_START},
-    {"SLCConfirmation", PERSISTENT},
-    {"SLCConfirmationHigher", PERSISTENT},
-    {"SLCConfirmationLower", PERSISTENT},
-    {"SLCFallback", PERSISTENT},
-    {"SLCLookaheadHigher", PERSISTENT},
-    {"SLCLookaheadLower", PERSISTENT},
-    {"SLCMapboxFiller", PERSISTENT},
-    {"SLCOverride", PERSISTENT},
-    {"SLCPriority1", PERSISTENT},
-    {"SLCPriority2", PERSISTENT},
-    {"SLCPriority3", PERSISTENT},
-    {"SNGHack", PERSISTENT},
-    {"SoundToDownload", CLEAR_ON_MANAGER_START},
-    {"SpeedLimitAccepted", CLEAR_ON_MANAGER_START},
-    {"SpeedLimitChangedAlert", PERSISTENT},
-    {"SpeedLimitController", PERSISTENT},
-    {"SpeedLimitFiller", PERSISTENT},
-    {"SpeedLimits", PERSISTENT | DONT_LOG},
-    {"SpeedLimitsFiltered", PERSISTENT | DONT_LOG},
-    {"SpeedLimitSources", PERSISTENT},
-    {"StandardFollow", PERSISTENT},
-    {"StandardJerkAcceleration", PERSISTENT},
-    {"StandardJerkDanger", PERSISTENT},
-    {"StandardJerkDeceleration", PERSISTENT},
-    {"StandardJerkSpeed", PERSISTENT},
-    {"StandardJerkSpeedDecrease", PERSISTENT},
-    {"StandardPersonalityProfile", PERSISTENT},
-    {"StandbyMode", PERSISTENT},
-    {"StartAccel", PERSISTENT},
-    {"StartAccelStock", PERSISTENT},
-    {"StartupMessageBottom", PERSISTENT},
-    {"StartupMessageTop", PERSISTENT},
-    {"StaticPedalsOnUI", PERSISTENT},
-    {"SteerDelay", PERSISTENT},
-    {"SteerDelayStock", PERSISTENT},
-    {"SteerFriction", PERSISTENT},
-    {"SteerFrictionStock", PERSISTENT},
-    {"SteerLatAccel", PERSISTENT},
-    {"SteerLatAccelStock", PERSISTENT},
-    {"SteerKP", PERSISTENT},
-    {"SteerKPStock", PERSISTENT},
-    {"SteerRatio", PERSISTENT},
-    {"SteerRatioStock", PERSISTENT},
-    {"StockDongleId", PERSISTENT},
-    {"StopAccel", PERSISTENT},
-    {"StopAccelStock", PERSISTENT},
-    {"StoppingDecelRate", PERSISTENT},
-    {"StoppingDecelRateStock", PERSISTENT},
-    {"StoppedTimer", PERSISTENT},
-    {"TacoTune", PERSISTENT},
-    {"TacoTuneHacks", PERSISTENT},
-    {"TestAlert", CLEAR_ON_MANAGER_START},
-    {"TetheringEnabled", PERSISTENT},
-    {"ThemeDownloadProgress", CLEAR_ON_MANAGER_START},
-    {"ThemesDownloaded", PERSISTENT},
-    {"TinygradUpdateAvailable", PERSISTENT},
-    {"TogglesUpdated", PERSISTENT},
-    {"ToyotaDoors", PERSISTENT},
-    {"TrafficFollow", PERSISTENT},
-    {"TrafficJerkAcceleration", PERSISTENT},
-    {"TrafficJerkDanger", PERSISTENT},
-    {"TrafficJerkDeceleration", PERSISTENT},
-    {"TrafficJerkSpeed", PERSISTENT},
-    {"TrafficJerkSpeedDecrease", PERSISTENT},
-    {"TrafficPersonalityProfile", PERSISTENT},
-    {"TuningLevel", PERSISTENT},
-    {"TuningLevelConfirmed", PERSISTENT},
-    {"TurnDesires", PERSISTENT},
-    {"UnlimitedLength", PERSISTENT},
-    {"UnlockDoors", PERSISTENT},
-    {"Updated", PERSISTENT},
-    {"UpdateSpeedLimits", CLEAR_ON_MANAGER_START},
-    {"UpdateSpeedLimitsStatus", CLEAR_ON_MANAGER_START},
-    {"UpdateTinygrad", CLEAR_ON_MANAGER_START},
-    {"UpdateWheelImage", CLEAR_ON_MANAGER_START},
-    {"UseActiveTheme", CLEAR_ON_MANAGER_START},
-    {"UseKonikServer", PERSISTENT},
-    {"UseSI", PERSISTENT},
-    {"UseVienna", PERSISTENT},
-    {"VEgoStarting", PERSISTENT},
-    {"VEgoStartingStock", PERSISTENT},
-    {"VEgoStopping", PERSISTENT},
-    {"VEgoStoppingStock", PERSISTENT},
-    {"VeryLongDistanceButtonControl", PERSISTENT},
-    {"VoltSNG", PERSISTENT},
-    {"WarningImmediateVolume", PERSISTENT},
-    {"WarningSoftVolume", PERSISTENT},
-    {"WheelIcon", PERSISTENT},
-    {"WheelSpeed", PERSISTENT},
-    {"WheelToDownload", CLEAR_ON_MANAGER_START},
-};
-
-} // namespace
-
-
-Params::Params(const std::string &path) {
-  params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
-  params_path = ensure_params_path(params_prefix, path);
-}
-
-Params::~Params() {
-  if (future.valid()) {
-    future.wait();
-  }
-  assert(queue.empty());
-}
-
-std::vector<std::string> Params::allKeys() const {
-  std::vector<std::string> ret;
-  for (auto &p : keys) {
-    ret.push_back(p.first);
-  }
-  return ret;
-}
-
-bool Params::checkKey(const std::string &key) {
-  return keys.find(key) != keys.end();
-}
-
-ParamKeyType Params::getKeyType(const std::string &key) {
-  return static_cast<ParamKeyType>(keys[key]);
-}
-
-int Params::put(const char* key, const char* value, size_t value_size) {
-  // Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
-  // 1) Create temp file
-  // 2) Write data to temp file
-  // 3) fsync() the temp file
-  // 4) rename the temp file to the real name
-  // 5) fsync() the containing directory
-  std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
-  int tmp_fd = mkstemp((char*)tmp_path.c_str());
-  if (tmp_fd < 0) return -1;
-
-  int result = -1;
-  do {
-    // Write value to temp.
-    ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
-    if (bytes_written < 0 || (size_t)bytes_written != value_size) {
-      result = -20;
-      break;
-    }
-
-    // fsync to force persist the changes.
-    if ((result = fsync(tmp_fd)) < 0) break;
-
-    FileLock file_lock(params_path + "/.lock");
-
-    // Move temp into place.
-    if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
-
-    // fsync parent directory
-    result = fsync_dir(getParamPath());
-  } while (false);
-
-  close(tmp_fd);
-  if (result != 0) {
-    ::unlink(tmp_path.c_str());
-  }
-  return result;
-}
-
-int Params::remove(const std::string &key) {
-  FileLock file_lock(params_path + "/.lock");
-  int result = unlink(getParamPath(key).c_str());
-  if (result != 0) {
-    return result;
-  }
-  return fsync_dir(getParamPath());
-}
-
-std::string Params::get(const std::string &key, bool block) {
-  if (!block) {
-    return util::read_file(getParamPath(key));
-  } else {
-    // blocking read until successful
-    params_do_exit = 0;
-    void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
-    void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
-
-    std::string value;
-    while (!params_do_exit) {
-      if (value = util::read_file(getParamPath(key)); !value.empty()) {
-        break;
-      }
-      util::sleep_for(100);  // 0.1 s
-    }
-
-    std::signal(SIGINT, prev_handler_sigint);
-    std::signal(SIGTERM, prev_handler_sigterm);
-    return value;
-  }
-}
-
-std::map<std::string, std::string> Params::readAll() {
-  FileLock file_lock(params_path + "/.lock");
-  return util::read_files_in_dir(getParamPath());
-}
-
-void Params::clearAll(ParamKeyType key_type) {
-  FileLock file_lock(params_path + "/.lock");
-
-  // 1) delete params of key_type
-  // 2) delete files that are not defined in the keys.
-  if (DIR *d = opendir(getParamPath().c_str())) {
-    struct dirent *de = NULL;
-    while ((de = readdir(d))) {
-      if (de->d_type != DT_DIR) {
-        auto it = keys.find(de->d_name);
-        if (it == keys.end() || (it->second & key_type)) {
-          unlink(getParamPath(de->d_name).c_str());
-        }
-      }
-    }
-    closedir(d);
-  }
-
-  fsync_dir(getParamPath());
-}
-
-void Params::putNonBlocking(const std::string &key, const std::string &val) {
-   queue.push(std::make_pair(key, val));
-  // start thread on demand
-  if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
-    future = std::async(std::launch::async, &Params::asyncWriteThread, this);
-  }
-}
-
-void Params::asyncWriteThread() {
-  // TODO: write the latest one if a key has multiple values in the queue.
-  std::pair<std::string, std::string> p;
-  while (queue.try_pop(p, 0)) {
-    // Params::put is Thread-Safe
-    put(p.first, p.second);
-  }
-}

common/params.h

@@ -1,90 +0,0 @@
-#pragma once
-
-#include <future>
-#include <map>
-#include <string>
-#include <tuple>
-#include <utility>
-#include <vector>
-
-#include "common/queue.h"
-
-enum ParamKeyType {
-  PERSISTENT = 0x02,
-  CLEAR_ON_MANAGER_START = 0x04,
-  CLEAR_ON_ONROAD_TRANSITION = 0x08,
-  CLEAR_ON_OFFROAD_TRANSITION = 0x10,
-  DONT_LOG = 0x20,
-  DEVELOPMENT_ONLY = 0x40,
-  ALL = 0xFFFFFFFF
-};
-
-class Params {
-public:
-  explicit Params(const std::string &path = {});
-  ~Params();
-  // Not copyable.
-  Params(const Params&) = delete;
-  Params& operator=(const Params&) = delete;
-
-  std::vector<std::string> allKeys() const;
-  bool checkKey(const std::string &key);
-  ParamKeyType getKeyType(const std::string &key);
-  inline std::string getParamPath(const std::string &key = {}) {
-    return params_path + params_prefix + (key.empty() ? "" : "/" + key);
-  }
-
-  // Delete a value
-  int remove(const std::string &key);
-  void clearAll(ParamKeyType type);
-
-  // helpers for reading values
-  std::string get(const std::string &key, bool block = false);
-  inline bool getBool(const std::string &key, bool block = false) {
-    return get(key, block) == "1";
-  }
-  inline int getInt(const std::string &key, bool block = false) {
-    std::string value = get(key, block);
-    return value.empty() ? 0 : std::stoi(value);
-  }
-  inline float getFloat(const std::string &key, bool block = false) {
-    std::string value = get(key, block);
-    return value.empty() ? 0.0 : std::stof(value);
-  }
-  std::map<std::string, std::string> readAll();
-
-  // helpers for writing values
-  int put(const char *key, const char *val, size_t value_size);
-  inline int put(const std::string &key, const std::string &val) {
-    return put(key.c_str(), val.data(), val.size());
-  }
-  inline int putBool(const std::string &key, bool val) {
-    return put(key.c_str(), val ? "1" : "0", 1);
-  }
-  inline int putInt(const std::string &key, int val) {
-    return put(key.c_str(), std::to_string(val).c_str(), std::to_string(val).size());
-  }
-  inline int putFloat(const std::string &key, float val) {
-    return put(key.c_str(), std::to_string(val).c_str(), std::to_string(val).size());
-  }
-  void putNonBlocking(const std::string &key, const std::string &val);
-  inline void putBoolNonBlocking(const std::string &key, bool val) {
-    putNonBlocking(key, val ? "1" : "0");
-  }
-  inline void putIntNonBlocking(const std::string &key, int val) {
-    putNonBlocking(key, std::to_string(val));
-  }
-  inline void putFloatNonBlocking(const std::string &key, float val) {
-    putNonBlocking(key, std::to_string(val));
-  }
-
-private:
-  void asyncWriteThread();
-
-  std::string params_path;
-  std::string params_prefix;
-
-  // for nonblocking write
-  std::future<void> future;
-  SafeQueue<std::pair<std::string, std::string>> queue;
-};

common/params.py

@@ -1,7 +1,8 @@
-from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
+from common.params_pyx import Params, ParamKeyType, UnknownKeyName, put_nonblocking # pylint: disable=no-name-in-module, import-error
 assert Params
 assert ParamKeyType
 assert UnknownKeyName
+assert put_nonblocking
 
 if __name__ == "__main__":
   import sys

common/params_pyx.pyx

@@ -2,40 +2,29 @@
 # cython: language_level = 3
 from libcpp cimport bool
 from libcpp.string cimport string
-from libcpp.vector cimport vector
+import threading
 
-cdef extern from "common/params.h":
+cdef extern from "selfdrive/common/params.h":
   cpdef enum ParamKeyType:
     PERSISTENT
     CLEAR_ON_MANAGER_START
-    CLEAR_ON_ONROAD_TRANSITION
-    CLEAR_ON_OFFROAD_TRANSITION
-    DEVELOPMENT_ONLY
+    CLEAR_ON_IGNITION_ON
+    CLEAR_ON_IGNITION_OFF
     ALL
 
   cdef cppclass c_Params "Params":
-    c_Params(string) except + nogil
+    c_Params(string) nogil
     string get(string, bool) nogil
-    bool getBool(string, bool) nogil
-    int getInt(string, bool) nogil
-    float getFloat(string, bool) nogil
+    bool getBool(string) nogil
     int remove(string) nogil
     int put(string, string) nogil
-    void putNonBlocking(string, string) nogil
-    void putBoolNonBlocking(string, bool) nogil
-    void putIntNonBlocking(string, int) nogil
-    void putFloatNonBlocking(string, float) nogil
     int putBool(string, bool) nogil
-    int putInt(string, int) nogil
-    int putFloat(string, float) nogil
     bool checkKey(string) nogil
-    string getParamPath(string) nogil
     void clearAll(ParamKeyType)
-    vector[string] allKeys()
 
 
 def ensure_bytes(v):
-  return v.encode() if isinstance(v, str) else v
+  return v.encode() if isinstance(v, str) else v;
 
 class UnknownKeyName(Exception):
   pass
@@ -76,32 +65,18 @@ cdef class Params:
 
     return val if encoding is None else val.decode(encoding)
 
-  def get_bool(self, key, bool block=False):
+  def get_bool(self, key):
     cdef string k = self.check_key(key)
     cdef bool r
     with nogil:
-      r = self.p.getBool(k, block)
-    return r
-
-  def get_int(self, key, bool block=False):
-    cdef string k = self.check_key(key)
-    cdef int r
-    with nogil:
-      r = self.p.getInt(k, block)
-    return r
-
-  def get_float(self, key, bool block=False):
-    cdef string k = self.check_key(key)
-    cdef float r
-    with nogil:
-      r = self.p.getFloat(k, block)
+      r = self.p.getBool(k)
     return r
 
   def put(self, key, dat):
     """
     Warning: This function blocks until the param is written to disk!
     In very rare cases this can take over a second, and your code will hang.
-    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
+    Use the put_nonblocking helper function in time sensitive code, but
     in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
@@ -114,45 +89,17 @@ cdef class Params:
     with nogil:
       self.p.putBool(k, val)
 
-  def put_int(self, key, int val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putInt(k, val)
-
-  def put_float(self, key, float val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putFloat(k, val)
-
-  def put_nonblocking(self, key, dat):
-    cdef string k = self.check_key(key)
-    cdef string dat_bytes = ensure_bytes(dat)
-    with nogil:
-      self.p.putNonBlocking(k, dat_bytes)
-
-  def put_bool_nonblocking(self, key, bool val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putBoolNonBlocking(k, val)
-
-  def put_int_nonblocking(self, key, int val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putIntNonBlocking(k, val)
-
-  def put_float_nonblocking(self, key, float val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putFloatNonBlocking(k, val)
-
-  def remove(self, key):
+  def delete(self, key):
     cdef string k = self.check_key(key)
     with nogil:
       self.p.remove(k)
 
-  def get_param_path(self, key=""):
-    cdef string key_bytes = ensure_bytes(key)
-    return self.p.getParamPath(key_bytes).decode("utf-8")
+def put_nonblocking(key, val, d=""):
+  def f(key, val):
+    params = Params(d)
+    cdef string k = ensure_bytes(key)
+    params.put(k, val)
 
-  def all_keys(self):
-    return self.p.allKeys()
+  t = threading.Thread(target=f, args=(key, val))
+  t.start()
+  return t

common/prefix.h

@@ -1,39 +0,0 @@
-#pragma once
-
-#include <cassert>
-#include <string>
-
-#include "common/params.h"
-#include "common/util.h"
-#include "system/hardware/hw.h"
-
-class OpenpilotPrefix {
-public:
-  OpenpilotPrefix(std::string prefix = {}) {
-    if (prefix.empty()) {
-      prefix = util::random_string(15);
-    }
-    msgq_path = "/dev/shm/" + prefix;
-    bool ret = util::create_directories(msgq_path, 0777);
-    assert(ret);
-    setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
-  }
-
-  ~OpenpilotPrefix() {
-    auto param_path = Params().getParamPath();
-    if (util::file_exists(param_path)) {
-      std::string real_path = util::readlink(param_path);
-      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
-      unlink(param_path.c_str());
-    }
-    if (getenv("COMMA_CACHE") == nullptr) {
-      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
-    }
-    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
-    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
-    unsetenv("OPENPILOT_PREFIX");
-  }
-
-private:
-  std::string msgq_path;
-};

common/prefix.py

@@ -1,53 +0,0 @@
-import os
-import shutil
-import uuid
-
-
-from openpilot.common.params import Params
-from openpilot.system.hardware import PC
-from openpilot.system.hardware.hw import Paths
-from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
-
-class OpenpilotPrefix:
-  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
-    self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
-    self.msgq_path = os.path.join('/dev/shm', self.prefix)
-    self.clean_dirs_on_exit = clean_dirs_on_exit
-    self.shared_download_cache = shared_download_cache
-
-  def __enter__(self):
-    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
-    os.environ['OPENPILOT_PREFIX'] = self.prefix
-    try:
-      os.mkdir(self.msgq_path)
-    except FileExistsError:
-      pass
-    os.makedirs(Paths.log_root(), exist_ok=True)
-
-    if self.shared_download_cache:
-      os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
-
-    return self
-
-  def __exit__(self, exc_type, exc_obj, exc_tb):
-    if self.clean_dirs_on_exit:
-      self.clean_dirs()
-    try:
-      del os.environ['OPENPILOT_PREFIX']
-      if self.original_prefix is not None:
-        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
-    except KeyError:
-      pass
-    return False
-
-  def clean_dirs(self):
-    symlink_path = Params().get_param_path()
-    if os.path.exists(symlink_path):
-      shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
-      os.remove(symlink_path)
-    shutil.rmtree(self.msgq_path, ignore_errors=True)
-    if PC:
-      shutil.rmtree(Paths.log_root(), ignore_errors=True)
-    if not os.environ.get("COMMA_CACHE", False):
-      shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
-    shutil.rmtree(Paths.comma_home(), ignore_errors=True)

common/profiler.py

@@ -0,0 +1,45 @@
+import time
+
+class Profiler():
+  def __init__(self, enabled=False):
+    self.enabled = enabled
+    self.cp = {}
+    self.cp_ignored = []
+    self.iter = 0
+    self.start_time = time.time()
+    self.last_time = self.start_time
+    self.tot = 0.
+
+  def reset(self, enabled=False):
+    self.enabled = enabled
+    self.cp = {}
+    self.cp_ignored = []
+    self.iter = 0
+    self.start_time = time.time()
+    self.last_time = self.start_time
+
+  def checkpoint(self, name, ignore=False):
+    # ignore flag needed when benchmarking threads with ratekeeper
+    if not self.enabled:
+      return
+    tt = time.time()
+    if name not in self.cp:
+      self.cp[name] = 0.
+      if ignore:
+        self.cp_ignored.append(name)
+    self.cp[name] += tt - self.last_time
+    if not ignore:
+      self.tot += tt - self.last_time
+    self.last_time = tt
+
+  def display(self):
+    if not self.enabled:
+      return
+    self.iter += 1
+    print("******* Profiling %d *******" % self.iter)
+    for n, ms in sorted(self.cp.items(), key=lambda x: -x[1]):
+      if n in self.cp_ignored:
+        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f   IGNORED" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
+      else:
+        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
+    print(f"Iter clock: {self.tot / self.iter:2.6f}   TOTAL: {self.tot:2.2f}")

common/ratekeeper.cc

@@ -1,40 +0,0 @@
-#include "common/ratekeeper.h"
-
-#include <algorithm>
-
-#include "common/swaglog.h"
-#include "common/timing.h"
-#include "common/util.h"
-
-RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
-    : name(name),
-      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
-  interval = 1 / rate;
-  last_monitor_time = seconds_since_boot();
-  next_frame_time = last_monitor_time + interval;
-}
-
-bool RateKeeper::keepTime() {
-  bool lagged = monitorTime();
-  if (remaining_ > 0) {
-    util::sleep_for(remaining_ * 1000);
-  }
-  return lagged;
-}
-
-bool RateKeeper::monitorTime() {
-  ++frame_;
-  last_monitor_time = seconds_since_boot();
-  remaining_ = next_frame_time - last_monitor_time;
-
-  bool lagged = remaining_ < 0;
-  if (lagged) {
-    if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
-      LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
-    }
-    next_frame_time = last_monitor_time + interval;
-  } else {
-    next_frame_time += interval;
-  }
-  return lagged;
-}

common/ratekeeper.h

@@ -1,23 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include <string>
-
-class RateKeeper {
-public:
-  RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
-  ~RateKeeper() {}
-  bool keepTime();
-  bool monitorTime();
-  inline uint64_t frame() const { return frame_; }
-  inline double remaining() const { return remaining_; }
-
-private:
-  double interval;
-  double next_frame_time;
-  double last_monitor_time;
-  double remaining_ = 0;
-  float print_delay_threshold = 0;
-  uint64_t frame_ = 0;
-  std::string name;
-};