translation

2025-10-29: move to raylib ui

.dockerignore

@@ -0,0 +1,18 @@
+**/.git
+.DS_Store
+*.dylib
+*.DSYM
+*.d
+*.pyc
+*.pyo
+.*.swp
+.*.swo
+.*.un~
+*.tmp
+*.o
+*.o-*
+*.os
+*.os-*
+
+venv/
+.venv/

.editorconfig

@@ -0,0 +1,11 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{py,pyx,pxd}]
+charset = utf-8
+indent_style = space
+indent_size = 2

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,47 @@
+name: Bug report
+description: For issues with running openpilot on your comma device
+labels: ["bug"]
+body:
+
+  - type: markdown
+    attributes:
+      value: >
+        Before creating a **bug report**, please check the following:
+          * If the issue likely only affects your car model or make, go back and open a **car bug report** instead.
+          * If the issue is related to the driving or driver monitoring models, you should open a [discussion](https://github.com/commaai/openpilot/discussions/categories/model-feedback) instead.
+          * Ensure you're running the latest openpilot release.
+          * Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
+          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
+          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
+
+        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
+
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: Also include a description of how to reproduce the bug
+    validations:
+      required: true
+
+  - type: input
+    id: route
+    attributes:
+      label: Provide a route where the issue occurs
+      description: Ensure the route is fully uploaded at https://useradmin.comma.ai. We cannot look into issues without routes, or at least a Dongle ID.
+      placeholder: 77611a1fac303767|2020-05-11--16-37-07
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: openpilot version
+      description: If you're not on release, provide the commit hash
+      placeholder: 0.8.10
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Additional info
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,14 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Car bug report
+    url: https://github.com/commaai/opendbc/issues/new
+    about: For issues with a particular car make or model
+  - name: Join the Discord
+    url: https://discord.comma.ai
+    about: The community Discord is for both openpilot development and experience discussion
+  - name: Report driving behavior feedback
+    url: https://discord.com/channels/469524606043160576/1254834193066623017
+    about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord
+  - name: Community Wiki
+    url: https://github.com/commaai/openpilot/wiki
+    about: Check out our community wiki

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,8 @@
+---
+name: Enhancement
+about: For openpilot enhancement suggestions
+title: ''
+labels: 'enhancement'
+assignees: ''
+---
+

.github/ISSUE_TEMPLATE/pc_bug_report.yml

@@ -0,0 +1,42 @@
+name: PC bug report
+description: For issues with running openpilot on PC
+labels: ["PC"]
+body:
+
+  - type: markdown
+    attributes:
+      value: >
+        Before creating a **bug report**, please check the following:
+          * Ensure you're running the latest openpilot release.
+          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
+          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
+
+        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
+
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: Also include a description of how to reproduce the bug
+    validations:
+      required: true
+
+  - type: input
+    id: os-version
+    attributes:
+      label: OS Version
+      placeholder: Ubuntu 24.04
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: openpilot version or commit
+      placeholder: bd36f2ec8d3559909678eff2690c10a520938367
+    validations:
+      required: false
+
+  - type: textarea
+    attributes:
+      label: Additional info
+

.github/labeler.yaml

@@ -0,0 +1,27 @@
+CI / testing:
+  - changed-files:
+    - any-glob-to-all-files: "{.github/**,**/test_*,**/test/**,Jenkinsfile}"
+
+car:
+  - changed-files:
+    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
+
+simulation:
+  - changed-files:
+    - any-glob-to-all-files: 'tools/sim/**'
+
+ui:
+  - changed-files:
+    - any-glob-to-all-files: '{selfdrive/ui/**,system/ui/**}'
+
+tools:
+  - changed-files:
+    - any-glob-to-all-files: 'tools/**'
+
+multilanguage:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
+
+autonomy:
+  - changed-files:
+    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"

.github/pull_request_template.md

@@ -0,0 +1,68 @@
+<!-- Please copy and paste the relevant template -->
+
+<!--- ***** Template: Fingerprint *****
+
+**Car**
+Which car (make, model, year) this fingerprint is for
+
+**Route**
+A route with the fingerprint
+
+-->
+
+<!--- ***** Template: Car Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+**Route**
+
+Route: [a route with the bug fix]
+
+
+-->
+
+<!--- ***** Template: Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+
+-->
+
+<!--- ***** Template: Car Port *****
+
+**Checklist**
+
+- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
+- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
+- [ ] route with openpilot:
+- [ ] route with stock system:
+- [ ] car harness used (if comma doesn't sell it, put N/A):
+
+
+-->
+
+<!--- ***** Template: Refactor *****
+
+**Description**
+
+A description of the refactor, including the goals it accomplishes. 
+
+**Verification**
+
+Explain how you tested the refactor for regressions. 
+
+
+-->
+

.gitignore

@@ -1,9 +1,23 @@
+venv/
+.venv/
+.ci_cache
+.env
+.clang-format
 .DS_Store
 .tags
 .ipynb_checkpoints
 .idea
-model2.png
+.overlay_init
+.overlay_consistent
+.sconsign.dblite
+a.out
+.hypothesis
+.cache/
 
+/docs_site/
+
+*.mp4
+*.dylib
 *.DSYM
 *.d
 *.pyc
@@ -11,20 +25,75 @@ model2.png
 .*.swp
 .*.swo
 .*.un~
+*.tmp
 *.o
+*.o-*
+*.os
+*.os-*
 *.so
 *.a
 *.clb
 *.class
 *.pyxbldc
 *.vcd
+*.mo
+*_pyx.cpp
 config.json
 clcache
+compile_commands.json
+compare_runtime*.html
 
-board/obj/
-selfdrive/boardd/boardd
-selfdrive/logcatd/logcatd
-selfdrive/proclogd/proclogd
-selfdrive/ui/ui
-/src/
+selfdrive/pandad/pandad
+cereal/services.h
+cereal/gen
+cereal/messaging/bridge
+selfdrive/ui/translations/tmp
+selfdrive/car/tests/cars_dump
+system/camerad/camerad
+system/camerad/test/ae_gray_test
 
+.coverage*
+coverage.xml
+htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/models/*.pkl
+
+# openpilot log files
+*.bz2
+*.zst
+
+build/
+
+!**/.gitkeep
+
+poetry.toml
+Pipfile
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# rick - keep panda_tici standalone
+panda_tici/

.travis.yml

@@ -1,12 +0,0 @@
-sudo: required
-
-services:
-  - docker
-
-install:
-  - docker build -t tmppilot -f Dockerfile.openpilot .
-
-script:
-  - docker run --rm
-      -v "$(pwd)"/selfdrive/test/tests/plant/out:/tmp/openpilot/selfdrive/test/tests/plant/out
-      tmppilot /bin/sh -c 'cd /tmp/openpilot/selfdrive/test/tests/plant && OPTEST=1 ./test_longitudinal.py'

.vscode/extensions.json

@@ -0,0 +1,11 @@
+{
+  "recommendations": [
+    "ms-python.python",
+    "ms-vscode.cpptools",
+    "elagil.pre-commit-helper",
+    "charliermarsh.ruff",
+    "JamiTech.simply-blame",
+    "k--kato.intellij-idea-keybindings",
+    "trinm1709.dracula-theme-from-intellij"
+  ]
+}

.vscode/launch.json

@@ -0,0 +1,85 @@
+{
+  "version": "0.2.0",
+  "inputs": [
+    {
+      "id": "python_process",
+      "type": "pickString",
+      "description": "Select the process to debug",
+      "options": [
+        "selfdrive/controls/controlsd.py",
+        "system/timed/timed.py",
+        "tools/sim/run_bridge.py"
+      ]
+    },
+    {
+      "id": "cpp_process",
+      "type": "pickString",
+      "description": "Select the process to debug",
+      "options": [
+        "selfdrive/ui/ui"
+      ]
+    },
+    {
+      "id": "args",
+      "description": "Arguments to pass to the process",
+      "type": "promptString"
+    },
+    {
+      "id": "replayArg",
+      "type": "promptString",
+      "description": "Enter route or segment to replay."
+    }
+  ],
+  "configurations": [
+    {
+      "name": "Python: openpilot Process",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${input:python_process}",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "args": "${input:args}"
+    },
+    {
+      "name": "C++: openpilot Process",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "${workspaceFolder}/${input:cpp_process}",
+      "cwd": "${workspaceFolder}"
+    },
+    {
+      "name": "Attach LLDB to Replay drive",
+      "type": "lldb",
+      "request": "attach",
+      "pid": "${command:pickMyProcess}",
+      "initCommands": [
+        "script import time; time.sleep(3)"
+      ]
+    },
+    {
+      "name": "Replay drive",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
+      "args": [
+        "${input:replayArg}"
+      ],
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      },
+      "subProcess": true,
+      "stopOnEntry": false
+    }
+  ],
+  "compounds": [
+    {
+      "name": "Replay drive + Safety LLDB",
+      "configurations": [
+        "Replay drive",
+        "Attach LLDB to Replay drive"
+      ]
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,42 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.renderWhitespace": "trailing",
+  "files.trimTrailingWhitespace": true,
+  "terminal.integrated.defaultProfile.linux": "dragonpilot",
+  "terminal.integrated.profiles.linux": {
+    "dragonpilot": {
+      "path": "bash",
+      "args": ["-c", "distrobox enter dp"]
+    }
+  },
+  "search.exclude": {
+    "**/.git": true,
+    "**/.venv": true,
+    "**/__pycache__": true,
+    "msgq_repo/": true,
+    "opendbc/": true,
+    "rednose/": true,
+    "rednose_repo/": true,
+    "openpilot/": true,
+    "teleoprtc_repo/": true,
+    "tinygrad/": true,
+    "tinygrad_repo/": true
+  },
+  "files.exclude": {
+    "**/.git": true,
+    "**/.venv": true,
+    "**/__pycache__": true
+  },
+  "python.analysis.exclude": [
+    "**/.git",
+    "**/.venv",
+    "**/__pycache__",
+    // exclude directories that should be using the symlinked version
+    "common/**",
+    "selfdrive/**",
+    "system/**",
+    "third_party/**",
+    "tools/**",
+  ]
+}

BRANCHES.md

@@ -0,0 +1,89 @@
+```mermaid
+flowchart TD
+    B000["devel-staging"] ---> CORE["core"]
+    CORE ---> CORE_001["core-feat/params"]
+    CORE_001 ---> CORE_002["core-feat/panel"]
+    CORE_002 ---> MIN["min"]
+    MIN ---> MIN_001["min-feat/dev/c3"]
+    MIN ---> MIN_002["min-feat/dev/o3"]
+    MIN ---> MIN_003["min-feat/lat/alka"]
+    MIN ---> MIN_004["min-feat/ui/display-mode"]
+    MIN ---> MIN_005["min-feat/dev/model-selector"]
+    MIN ---> MIN_006["min-feat/lat/lca"]
+    MIN ---> MIN_007["min-feat/dev/on-off-road"]
+    MIN ---> MIN_008["min-feat/ui/hide-hud"]
+    MIN ---> MIN_009["min-feat/lon/ext-radar"]
+    MIN ---> MIN_010["min-feat/lat/road-edge-detection"]
+    MIN ---> MIN_011["min-feat/ui/rainbow-path"]
+    MIN ---> MIN_012["min-feat/lon/acm"]
+    MIN ---> MIN_013["min-feat/lon/aem"]
+    MIN ---> MIN_014["min-feat/lon/dtsc"]
+    MIN ---> MIN_015["min-feat/dev/alert-mode"]
+    MIN ---> MIN_016["min-feat/dev/auto-shutdown"]
+    MIN ---> MIN_017["min-feat/ui/lead-stats"]
+    MIN ---> MIN_018["min-feat/ui/border-indicator"]
+    MIN ---> MIN_019["min-feat/dev/dashy"]
+    MIN ---> MIN_010["min-feat/dev/delay-loggerd"]
+    MIN ---> MIN_021["min-feat/dev/disable-connect"]
+    MIN ---> MIN_022["min-feat/dev/tether-on-boot"]
+    MIN_001 ---> FULL["full"]
+    MIN_002 ---> FULL
+    MIN_003 ---> FULL
+    MIN_004 ---> FULL
+    MIN_005 ---> FULL
+    MIN_006 ---> FULL
+    MIN_007 ---> FULL
+    MIN_008 ---> FULL
+    MIN_009 ---> FULL
+    MIN_010 ---> FULL
+    MIN_011 ---> FULL
+    MIN_012 ---> FULL
+    MIN_013 ---> FULL
+    MIN_014 ---> FULL
+    MIN_015 ---> FULL
+    MIN_016 ---> FULL
+    MIN_017 ---> FULL
+    MIN_018 ---> FULL
+    MIN_019 ---> FULL
+    MIN_020 ---> FULL
+    MIN_021 ---> FULL
+    MIN_022 ---> FULL
+    FULL ---> TOYOTA_001[brand/toyota/safety-common]
+    FULL ---> TOYOTA_002[brand/toyota/door-auto-lock-unlock]
+    FULL ---> TOYOTA_003[brand/toyota/tss1-sng]
+    FULL ---> TOYOTA_004[brand/toyota/radar-filter]
+    FULL ---> TOYOTA_005[brand/toyota/sdsu]
+    FULL ---> TOYOTA_006[brand/toyota/dsu-bypass]
+    FULL ---> TOYOTA_007[brand/toyota/zss]
+    FULL ---> TOYOTA_008[brand/toyota/stock-lon]
+    FULL ---> VAG_001[brand/vag/a0-sng]
+    FULL ---> VAG_002[brand/vag/pq-steering-patch]
+    FULL ---> VAG_003[brand/vag/pq-no-dashcam]
+    FULL ---> VAG_004[brand/vag/avoid-eps-lockout]
+    FULL ---> HKG_001[brand/hkg/smdps]
+    FULL ---> HONDA_001[brand/honda/eps-mod]
+    FULL ---> SUBARU_001[brand/subaru/torque-3071]
+    TOYOTA_001 ---> TOYOTA[pre-toyota]
+    TOYOTA_002 ---> TOYOTA
+    TOYOTA_003 ---> TOYOTA
+    TOYOTA_004 ---> TOYOTA
+    TOYOTA_005 ---> TOYOTA
+    TOYOTA_006 ---> TOYOTA
+    TOYOTA_007 ---> TOYOTA
+    TOYOTA_008 ---> TOYOTA
+    VAG_001 ---> VAG[pre-vag]
+    VAG_002 ---> VAG
+    VAG_003 ---> VAG
+    VAG_004 ---> VAG
+    HKG_001 ---> HKG[pre-hkg]
+    HONDA_001 ---> HONDA[pre-honda]
+    SUBARU_001 ---> SUBARU[pre-subaru]
+    TOYOTA ---> PRE[pre]
+    VAG ---> PRE
+    HKG ---> PRE
+    HONDA ---> PRE
+    SUBARU ---> PRE
+    PRE ---> PRE_PATCH[pre-patch]
+    PRE_PATCH ---> PREBUILD[pre-build]
+    PREBUILD ---> VERSION[x.x.x]
+```

Dockerfile.openpilot

@@ -1,19 +1,14 @@
-FROM ubuntu:16.04
-ENV PYTHONUNBUFFERED 1
+FROM ghcr.io/commaai/openpilot-base:latest
 
-RUN apt-get update && apt-get install -y build-essential clang vim screen wget bzip2 git libglib2.0-0 python-pip capnproto libcapnp-dev libzmq5-dev libffi-dev libusb-1.0-0
-RUN pip install numpy==1.11.2 scipy==0.18.1 matplotlib
+ENV PYTHONUNBUFFERED=1
 
-COPY requirements_openpilot.txt /tmp/
-RUN pip install -r /tmp/requirements_openpilot.txt
+ENV OPENPILOT_PATH=/home/batman/openpilot
 
-ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH
+RUN mkdir -p ${OPENPILOT_PATH}
+WORKDIR ${OPENPILOT_PATH}
 
-COPY ./common /tmp/openpilot/common
-COPY ./cereal /tmp/openpilot/cereal
-COPY ./opendbc /tmp/openpilot/opendbc
-COPY ./selfdrive /tmp/openpilot/selfdrive
-COPY ./phonelibs /tmp/openpilot/phonelibs
-COPY ./pyextra /tmp/openpilot/pyextra
+COPY . ${OPENPILOT_PATH}/
 
-RUN mkdir -p /tmp/openpilot/selfdrive/test/out
+ENV UV_BIN="/home/batman/.local/bin/"
+ENV PATH="$UV_BIN:$PATH"
+RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc)

Dockerfile.openpilot_base

@@ -0,0 +1,81 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED=1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX=1
+
+RUN dbus-uuidgen > /etc/machine-id
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /home/$USER
+COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /home/$USER && \
+    tools/install_python_dependencies.sh && \
+    rm -rf tools/ pyproject.toml uv.lock .cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -0,0 +1,269 @@
+def retryWithDelay(int maxRetries, int delay, Closure body) {
+  for (int i = 0; i < maxRetries; i++) {
+    try {
+      return body()
+    } catch (Exception e) {
+      sleep(delay)
+    }
+  }
+  throw Exception("Failed after ${maxRetries} retries")
+}
+
+def device(String ip, String step_label, String cmd) {
+  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
+    def ssh_cmd = """
+ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
+
+set -e
+
+export TERM=xterm-256color
+
+shopt -s huponexit # kill all child processes when the shell exits
+
+export CI=1
+export PYTHONWARNINGS=error
+export LOGPRINT=debug
+export TEST_DIR=${env.TEST_DIR}
+export SOURCE_DIR=${env.SOURCE_DIR}
+export GIT_BRANCH=${env.GIT_BRANCH}
+export GIT_COMMIT=${env.GIT_COMMIT}
+export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
+export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
+export AZURE_TOKEN='${env.AZURE_TOKEN}'
+# only use 1 thread for tici tests since most require HIL
+export PYTEST_ADDOPTS="-n0 -s"
+
+
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+source ~/.bash_profile
+if [ -f /TICI ]; then
+  source /etc/profile
+
+  rm -rf /tmp/tmp*
+  rm -rf ~/.commacache
+  rm -rf /dev/shm/*
+  rm -rf /dev/tmp/tmp*
+
+  if ! systemctl is-active --quiet systemd-resolved; then
+    echo "restarting resolved"
+    sudo systemctl start systemd-resolved
+    sleep 3
+  fi
+
+  # restart aux USB
+  if [ -e /sys/bus/usb/drivers/hub/3-0:1.0 ]; then
+    echo "restarting aux usb"
+    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/unbind
+    sleep 0.5
+    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/bind
+  fi
+fi
+if [ -f /data/openpilot/launch_env.sh ]; then
+  source /data/openpilot/launch_env.sh
+fi
+
+ln -snf ${env.TEST_DIR} /data/pythonpath
+
+cd ${env.TEST_DIR} || true
+time ${cmd}
+END"""
+
+    sh script: ssh_cmd, label: step_label
+  }
+}
+
+def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
+  stage(stageName) {
+    if (currentBuild.result != null) {
+        return
+    }
+
+    if (isReplay()) {
+      error("REPLAYING TESTS IS NOT ALLOWED. FIX THEM INSTEAD.")
+    }
+
+    def extra = extra_env.collect { "export ${it}" }.join('\n');
+    def branch = env.BRANCH_NAME ?: 'master';
+    def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes'
+
+    lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
+      docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
+        timeout(time: 35, unit: 'MINUTES') {
+          retry (3) {
+            def date = sh(script: 'date', returnStdout: true).trim();
+            device(device_ip, "set time", "date -s '" + date + "'")
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          }
+          steps.each { item ->
+            def name = item[0]
+            def cmd = item[1]
+
+            def args = item[2]
+            def diffPaths = args.diffPaths ?: []
+            def cmdTimeout = args.timeout ?: 9999
+
+            if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
+              println "Skipping ${name}: no changes in ${diffPaths}."
+              return
+            } else {
+              timeout(time: cmdTimeout, unit: 'SECONDS') {
+                device(device_ip, name, cmd)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+def hasPathChanged(String gitDiff, List<String> paths) {
+  for (path in paths) {
+    if (gitDiff.contains(path)) {
+      return true
+    }
+  }
+  return false
+}
+
+def isReplay() {
+  def replayClass = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause"
+  return currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClass) }
+}
+
+def setupCredentials() {
+  withCredentials([
+    string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
+  ]) {
+    env.AZURE_TOKEN = "${AZURE_TOKEN}"
+  }
+
+  withCredentials([
+    string(credentialsId: 'ci_artifacts_pat', variable: 'CI_ARTIFACTS_TOKEN'),
+  ]) {
+    env.CI_ARTIFACTS_TOKEN = "${CI_ARTIFACTS_TOKEN}"
+  }
+
+  withCredentials([
+    string(credentialsId: 'post_comments_github_pat', variable: 'GITHUB_COMMENTS_TOKEN'),
+  ]) {
+    env.GITHUB_COMMENTS_TOKEN = "${GITHUB_COMMENTS_TOKEN}"
+  }
+}
+
+def step(String name, String cmd, Map args = [:]) {
+  return [name, cmd, args]
+}
+
+node {
+  env.CI = "1"
+  env.PYTHONWARNINGS = "error"
+  env.TEST_DIR = "/data/openpilot"
+  env.SOURCE_DIR = "/data/openpilot_source/"
+  setupCredentials()
+
+  env.GIT_BRANCH = checkout(scm).GIT_BRANCH
+  env.GIT_COMMIT = checkout(scm).GIT_COMMIT
+
+  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
+                         'release-tici', 'release-tizi', 'release-tizi-staging', 'testing-closet*', 'hotfix-*']
+  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
+
+  if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
+    properties([
+        disableConcurrentBuilds(abortPrevious: true)
+    ])
+  }
+
+  try {
+    if (env.BRANCH_NAME == 'devel-staging') {
+      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh"),
+      ])
+    }
+
+    if (env.BRANCH_NAME == '__nightly') {
+      parallel (
+        'nightly': {
+          deviceStage("build nightly", "tizi-needs-can", [], [
+            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
+          ])
+        },
+        'nightly-dev': {
+          deviceStage("build nightly-dev", "tizi-needs-can", [], [
+            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
+          ])
+        },
+      )
+    }
+
+    if (!env.BRANCH_NAME.matches(excludeRegex)) {
+    parallel (
+      'onroad tests': {
+        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("check dirty", "release/check-dirty.sh"),
+          step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
+        ])
+      },
+      'HW + Unit Tests': {
+        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
+          step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
+          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
+          step("test manager", "pytest system/manager/test/test_manager.py"),
+        ])
+      },
+      'loopback': {
+        deviceStage("loopback", "tizi-loopback", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+        ])
+      },
+      'camerad OX03C10': {
+        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+      },
+      'camerad OS04C10': {
+        deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+      },
+      'sensord': {
+        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+        ])
+      },
+      'replay': {
+        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+        ])
+      },
+      'tizi': {
+        deviceStage("tizi", "tizi", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          // TODO: enable once new AGNOS is available
+          // step("test esim", "pytest system/hardware/tici/tests/test_esim.py"),
+          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
+        ])
+      },
+
+    )
+    }
+  } catch (Exception e) {
+    currentBuild.result = 'FAILED'
+    throw e
+  }
+}

LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2018, Comma.ai, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.md

@@ -0,0 +1,30 @@
+Copyright (c) 2019, Rick Lan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, and/or sublicense, 
+for non-commercial purposes only, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in 
+  all copies or substantial portions of the Software.
+- Commercial use (e.g. use in a product, service, or activity intended to 
+  generate revenue) is prohibited without explicit written permission from 
+  the copyright holder.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+---
+Copyright (c) 2018, Comma.ai, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy 
+of this software and associated documentation files (the "Software"), to deal 
+in the Software without restriction, including without limitation the rights 
+to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the 
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in 
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -1,9 +0,0 @@
-
-code_dir := $(shell pwd)
-
-# TODO: Add a global build system
-
-.PHONY: all
-all:
-	cd selfdrive && PYTHONPATH=$(code_dir) PREPAREONLY=1 ./manager.py
-

README.md

@@ -1,141 +1,74 @@
-Welcome to openpilot
-======
+![](dragonpilot/selfdrive/assets/dragonpilot.png)
 
-[openpilot](http://github.com/commaai/openpilot) is an open source driving agent.
+[Read this in English](README_EN.md)
 
-Currently it performs the functions of Adaptive Cruise Control (ACC) and Lane Keeping Assist System (LKAS) for Hondas, Acuras and Toyotas. It's about on par with Tesla Autopilot at launch, and better than [all other manufacturers](http://www.thedrive.com/tech/5707/the-war-for-autonomous-driving-part-iii-us-vs-germany-vs-japan).
+# **🐲 dragonpilot - 賦予您的愛車「龍」之魂**
 
-The openpilot codebase has been written to be concise and enable rapid prototyping. We look forward to your contributions - improving real vehicle automation has never been easier.
+**我們與您一同翱翔於更智慧、更貼心的駕駛旅程。**
 
-Here are [some](https://www.youtube.com/watch?v=9OwTJFuDI7g) [videos](https://www.youtube.com/watch?v=64Wvt5pYQmE) [of](https://www.youtube.com/watch?v=6IW7Nejsr3A) [it](https://www.youtube.com/watch?v=-VN1YcC83nA) [running](https://www.youtube.com/watch?v=EQJZvVeihZk). And a really cool [tutorial](https://www.youtube.com/watch?v=PwOnsT2UW5o).
+## **👋 嘿, 朋友，歡迎您的到來！**
 
-Hardware
-------
+`dragonpilot` 誕生於 2019 年，由三位早期的 openpilot 華人玩家共同創立。初衷很簡單：為廣大的華人用戶、玩家們提供一個友善的交流環境、更簡便的設定協助，並加入更多適合在地使用的貼心功能。
 
-Right now openpilot supports the [NEO research platform](http://github.com/commaai/neo) and the [EON Dashcam DevKit](https://shop.comma.ai/products/eon-dashcam-devkit). We'd like to support other platforms as well.
+我們深知在地化的重要性，特別是語言的親切感。因此，我們率先導入了完整的中文介面，讓 `dragonpilot` 迅速在華語地區累積了口碑，也讓華人的使用者數量在全球名列前茅。這份來自在地的支持，是我們持續前進的最大動力。
 
-Install openpilot on a neo device by entering ``https://openpilot.comma.ai`` during NEOS setup.
+我們以功能強大的 [openpilot](https://github.com/commaai/openpilot) 為基礎——這套據美國消費者報告評測優於市售車方案的開源輔助駕駛系統——融入了更多在地化的巧思與客製化的溫度，希望能打造出最符合您需求的駕駛夥伴。(您也可以參考我們 repo 中保留的 [openpilot 原始說明檔案](README_OPENPILOT.md))
 
-Supported Cars
-------
+取名 `dragonpilot`，是因為我們希望它能像神話中的「龍」一樣，既強大又充滿智慧，為您的行車安全保駕護航。龍，在我們華人文化中，更是吉祥與力量的象徵，也代表著我們的根源與驕傲。
 
-- Acura ILX 2016 with AcuraWatch Plus
-  - Due to use of the cruise control for gas, it can only be enabled above 25 mph
+## **✨ dragonpilot 的里程碑**
 
-- Honda Civic 2016-2017 with Honda Sensing
-  - Due to limitations in steering firmware, steering is disabled below 12 mph
-  - Note that the hatchback model is not supported
+我們不僅保留了 openpilot 的核心優勢，更達成了許多從社群回饋中誕生的里程碑，這些是我們引以為傲的足跡：
 
-- Honda CR-V Touring 2015-2016
-  - Can only be enabled above 25 mph
+* **🚘 全時置中車道維持 (ALKA)**
 
-- Toyota RAV-4 2016+ non-hybrid with TSS-P
-  - By default it uses stock Toyota ACC for longitudinal control
-  - openpilot longitudinal control available after unplugging the [Driving Support ECU](https://community.comma.ai/wiki/index.php/Toyota#Rav4_.28for_openpilot.29) and can be enabled above 20 mph
+  這不只是一個功能，更是 `dragonpilot` 的哲學。我們最早於 [0.6.2 版本](https://github.com/dragonpilot-community/dragonpilot/blob/2861467183d62151024320447ba04d18fc3fe1e6/selfdrive/car/toyota/carstate.py#L199) 時便實現了這個功能，其開發歷程始於 2017 Lexus IS300h，接著擴展至 Toyota 全車系，並逐步延伸到其他支援的品牌。它能溫柔地輔助您，讓車輛始終穩定地保持在車道中央，提供一份額外的安心與從容。
 
-- Toyota Prius 2017 (alpha!)
-  - By default it uses stock Toyota ACC for longitudinal control
-  - openpilot longitudinal control available after unplugging the [Driving Support ECU](https://community.comma.ai/wiki/index.php/Toyota#Prius_.28for_openpilot.29)
-  - Lateral control needs improvements
+* **🌐 率先導入多國語言介面**
 
-- Toyota RAV-4 2017 hybrid (alpha!)
-  - By default it uses stock Toyota ACC for longitudinal control
-  - openpilot longitudinal control available after unplugging the [Driving Support ECU](https://community.comma.ai/wiki/index.php/Toyota#Rav4_.28for_openpilot.29) and can do stop and go
+  在官方 openpilot 還未支援前，我們便已將多國語言介面實現。`dragonpilot` 完整支援繁體中文、簡體中文與英文，讓操作毫無隔閡。
 
-In Progress Cars
-------
-- Probably all TSS-P Toyota with Steering Assist.
-  - 'Full Speed Range Dynamic Radar Cruise Control' is required to enable stop-and-go. Only the Prius, Camry and C-HR have this option.
-  - Even though the Tundra, Sequoia and the Land Cruiser have TSS-P, they don't have Steering Assist and are not supported.
+* **💻 唯一同時支援多硬體平台**
 
-Community WIP Cars
-------
+  我們是唯一曾致力於讓專案同時兼容 EON、comma two、comma 3 與 Jetson 平台的社群分支，這份努力是為了服務最廣大的玩家社群。
+  此外，在 comma.ai 團隊於 0.10.0 版本宣布停止支持 comma 3 後，我們仍是唯一一個完整同時支援 comma 3、comma 3X 以及 O3、O3L、O3XL（O3 系列為副廠硬體）的社群分支。
 
-- [Chevy Volt 2016-2018 Premier with Driver Confidence II](https://github.com/commaai/openpilot/pull/104)
+* **📜 曾榮獲官方認證第一大分支**
 
-- [Classic Tesla Model S (pre-AP)](https://github.com/commaai/openpilot/pull/145)
+  基於活躍的社群與功能創新，`dragonpilot` 曾一度成長為 comma ai 官方認證的第一大 openpilot 分支，這份榮耀屬於每一位參與者。
 
-- [Honda Odyssey 2018 with Honda Sensing](https://github.com/commaai/openpilot/pull/155)
+## **🧑‍💻 設計理念 - 少即是多 (Less is More)**
 
-- [Honda Pilot 2017 with Honda Sensing](https://github.com/commaai/openpilot/pull/161)
+隨著 openpilot 的 AI 模型日益強大，許多過去需要手動微調的功能，現在都已能透過更先進的模型來實現。因此，我們現在的開發重心回歸到 **「最小化修改」(minimal changes)** 的核心原則上。
 
-- [Acura RDX 2018 with AcuraWatch Plus](https://github.com/commaai/openpilot/pull/162)
+我們的目標是為您提供最純粹、最接近官方的 openpilot 駕駛感受，同時保留 `dragonpilot` 那些經過時間考驗、最受社群喜愛的經典功能。我們相信，在強大的 AI 基礎上，簡潔即是力量。
 
-Directory structure
-------
+## **🛠️ 硬件的足跡 - 一路走來的夥伴們**
 
-- cereal        -- The messaging spec used for all logs on the phone
-- common        -- Library like functionality we've developed here
-- opendbc       -- Files showing how to interpret data from cars
-- panda         -- Code used to communicate on CAN and LIN
-- phonelibs     -- Libraries used on the phone
-- selfdrive     -- Code needed to drive the car
-  - assets        -- Fonts for ui
-  - boardd        -- Daemon to talk to the board
-  - car           -- Code that talks to the car and implements CarInterface
-  - common        -- Shared C/C++ code for the daemons
-  - controls      -- Python controls (PID loops etc) for the car
-  - debug         -- Tools to help you debug and do car ports
-  - logcatd       -- Android logcat as a service
-  - loggerd       -- Logger and uploader of car data
-  - proclogd      -- Logs information from proc
-  - radar         -- Code that talks to the radar and implements RadarInterface
-  - sensord       -- IMU / GPS interface code
-  - test/plant    -- Car simulator running code through virtual maneuvers
-  - ui            -- The UI
-  - visiond       -- embedded vision pipeline
+從最早的 **EON**，到官方的 **comma two / three (C2/C3/C3X)**，再到社群中各式各樣充滿智慧的**副廠機 (如 C1.5, O2, O3, O3L, O3XL 等)**，甚至我們也曾探索過在 [**Jetson Xavier NX**](https://github.com/eFiniLan/xnxpilot) 上的可能性。
 
-To understand how the services interact, see `selfdrive/service_list.yaml`
+目前最新版本主要支援： comma3 / 3X 以及 O3 / O3L / O3XL 等社群硬體。
+針對 EON / C1.5 / C2 等舊款硬體，最後支援的版本位於 [d2 分支](https://github.com/dragonpilot-community/dragonpilot/tree/d2)。
+無論您手上是哪一款設備，都代表著您對開源駕駛輔助的一份熱情。
 
-Testing on PC
-------
+## **🫂 加入我們，成為「尋龍者」的一份子**
 
-There is rudimentary infrastructure to run a basic simulation and generate a report of openpilot's behavior in different scenarios.
+`dragonpilot` 的成長，離不開每一位使用者的貢獻與回饋。我們是一個以**公開、透明**為原則的溫暖社群，希望在這裡能與所有對 openpilot / dragonpilot 有興趣的用戶分享、交流開發與使用上的經驗。
 
-```bash
-# Requires working docker
-./run_docker_tests.sh
-```
+[**歡迎加入我們的 Facebook 社團進行交流！**](https://www.facebook.com/groups/930190251238639)
 
-The results are written to `selfdrive/test/plant/out/index.html`
+## **❤️ 特別感謝**
 
-More extensive testing infrastructure and simulation environments are coming soon.
+`dragonpilot` 從創立至今，從未打算透過 Patreon 等平台進行任何形式的募資。我們的初衷是建立一個讓大家能一起學習、一起成長的社群。It's all about fun, not money.
 
-Adding Car Support
-------
+然而，我們仍要對那些自發性支持本專案的朋友們，致上最誠摯的感謝。正是因為有您們的鼓勵，我們才有更大的動力持續前進。
 
-comma.ai offers [bounties](http://comma.ai/bounties.html) for adding additional car support.
+[**我們的贊助者名單**](SPONSORS.md)
 
-CR-V Touring support came in through this program. Chevy Volt is close. Accord is close as well.
+### **安全聲明**
 
-User Data / chffr Account / Crash Reporting
-------
+`dragonpilot` 是一種駕駛**輔助**系統，並非全自動駕駛。它旨在減輕您的駕駛疲勞，提升行車安全，但駕駛人仍需時刻保持專注，並隨時準備接管車輛。請務必遵守您所在地區的交通法規。
 
-By default openpilot creates an account and includes a client for chffr, our dashcam app. We use your data to train better models and improve openpilot for everyone.
+**最後，再次感謝您的到來。**
 
-It's open source software, so you are free to disable it if you wish.
-
-It logs the road facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-It does not log the user facing camera or the microphone.
-
-By using it, you agree to [our privacy policy](https://beta.comma.ai/privacy.html). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma.ai. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma.ai for the use of this data.
-
-Contributing
-------
-
-We welcome both pull requests and issues on
-[github](http://github.com/commaai/openpilot). See the TODO file for a list of
-good places to start.
-
-Want to get paid to work on openpilot? [comma.ai is hiring](http://comma.ai/positions.html)
-
-Licensing
-------
-
-openpilot is released under the MIT license.
-
-Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
-
-**THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
-YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
-NO WARRANTY EXPRESSED OR IMPLIED.**
+**期待與您一同在智慧駕駛的道路上，乘「龍」而行！**

README_EN.md

@@ -0,0 +1,74 @@
+![](dragonpilot/selfdrive/assets/dragonpilot.png)
+
+[Read this in Chinese](README.md)
+
+# **🐲 dragonpilot - Bringing the Spirit of the Dragon to Your Car**
+
+**Join us on a smarter, more thoughtful driving journey.**
+
+## **👋 Welcome, friend!**
+
+`dragonpilot` was launched in 2019 by three early openpilot enthusiasts from the Chinese community. Our mission was simple: create a friendly space for users to share experiences, provide easier setup help, and add features tailored for local needs.
+
+Localization has always been at the heart of what we do—starting with a fully Chinese interface. This made `dragonpilot` quickly popular in Chinese-speaking regions and helped our user base grow into one of the largest worldwide. That community support is what keeps us moving forward.
+
+Built on top of the powerful [openpilot](https://github.com/commaai/openpilot)—an open-source driver assistance system rated by Consumer Reports as outperforming commercial offerings—we add localized refinements and user-focused features to create a driving companion that truly fits your needs. (You can also see the [original openpilot README](README_OPENPILOT.md) preserved in our repo.)
+
+The name `dragonpilot` reflects our vision: like the dragon of mythology, it is strong and wise, guarding your safety on the road. In Chinese culture, the dragon is also a symbol of luck and strength, representing our roots and pride.
+
+## **✨ Milestones**
+
+Beyond carrying forward openpilot's core strengths, we've reached several milestones inspired by community feedback:
+
+* **🚘 Always Lane Keep Assist (ALKA)**
+
+  More than a feature—it's part of the `dragonpilot` philosophy. Introduced as early as [version 0.6.2](https://github.com/dragonpilot-community/dragonpilot/blob/2861467183d62151024320447ba04d18fc3fe1e6/selfdrive/car/toyota/carstate.py#L199), first tested on a 2017 Lexus IS300h, then expanded to Toyota's lineup and beyond. ALKA helps keep your vehicle steadily centered, giving you extra confidence on the road.
+
+* **🌐 First to add multilingual support**
+
+  Before openpilot officially supported it, we had already introduced multiple languages. `dragonpilot` fully supports Traditional Chinese, Simplified Chinese, and English.
+
+* **💻 Only community fork to support multiple hardware platforms at once**
+
+  We uniquely worked to make the project run on EON, comma two, comma 3, and Jetson—serving the widest range of users possible.
+  Additionally, after the comma.ai team deprecated the comma 3 in version 0.10.0, we remain the only community fork to offer full, simultaneous support for the comma 3, comma 3X, and the O3, O3L, and O3XL (the O3 series being third-party hardware).
+
+* **📜 Once recognized as the #1 openpilot fork**
+
+  Thanks to an active community and continuous innovation, `dragonpilot` was once the largest openpilot fork officially recognized by comma ai. This honor belongs to everyone who contributed.
+
+## **🧑‍💻 Design Philosophy - Less is More**
+
+As openpilot's AI grows stronger, many features that once required manual tuning are now handled by advanced models. That's why our focus has returned to **“minimal changes.”**
+
+We aim to give you the purest, most official-like openpilot driving experience—while preserving `dragonpilot`'s classic, community-loved features. With a solid AI foundation, simplicity is strength.
+
+## **🛠️ Hardware Journey**
+
+From the early **EON**, to official devices like **comma two / three (C2/C3/C3X)**, to creative community builds (**C1.5, O2, O3, O3L, O3XL, etc.**), and even experiments with [**Jetson Xavier NX**](https://github.com/eFiniLan/xnxpilot).
+
+Currently, the latest versions support: **comma3 / 3X** and community hardware like **O3 / O3L / O3XL**.
+Older devices such as **EON / C1.5 / C2** are supported in the [d2 branch](https://github.com/dragonpilot-community/dragonpilot/tree/d2).
+Whatever device you're on, it represents your passion for open-source driver assistance.
+
+## **🫂 Join Us – Become a “Dragon Seeker”**
+
+`dragonpilot` thrives thanks to every user's contributions and feedback. We're an open, transparent, and welcoming community where enthusiasts can share experiences with openpilot and `dragonpilot`.
+
+[**Join our Facebook group here!**](https://www.facebook.com/groups/930190251238639)
+
+## **❤️ Special Thanks**
+
+Since day one, `dragonpilot` has never asked for funding through Patreon or similar platforms. Our vision is a community where everyone learns and grows together. It's about fun, not money.
+
+That said, we're deeply grateful to those who voluntarily supported the project. Your encouragement keeps us motivated to keep building.
+
+[**See our sponsors**](SPONSORS.md)
+
+### **Safety Notice**
+
+`dragonpilot` is a driver **assistance** system, not full self-driving. It reduces fatigue and improves safety, but you must remain alert and ready to take control at all times. Always follow your local traffic laws.
+
+**Thanks again for being here.**
+
+**We look forward to riding the “dragon” with you on the road to smarter driving!**

README_OPENPILOT.md

@@ -0,0 +1,107 @@
+<div align="center" style="text-align: center;">
+
+<h1>openpilot</h1>
+
+<p>
+  <b>openpilot is an operating system for robotics.</b>
+  <br>
+  Currently, it upgrades the driver assistance system in 300+ supported cars.
+</p>
+
+<h3>
+  <a href="https://docs.comma.ai">Docs</a>
+  <span> · </span>
+  <a href="https://docs.comma.ai/contributing/roadmap/">Roadmap</a>
+  <span> · </span>
+  <a href="https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md">Contribute</a>
+  <span> · </span>
+  <a href="https://discord.comma.ai">Community</a>
+  <span> · </span>
+  <a href="https://comma.ai/shop">Try it on a comma 3X</a>
+</h3>
+
+Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
+
+[![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg)](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
+[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
+
+</div>
+
+<table>
+  <tr>
+    <td><a href="https://youtu.be/NmBfgOanCyk" title="Video By Greer Viau"><img src="https://github.com/commaai/openpilot/assets/8762862/2f7112ae-f748-4f39-b617-fabd689c3772"></a></td>
+    <td><a href="https://youtu.be/VHKyqZ7t8Gw" title="Video By Logan LeGrand"><img src="https://github.com/commaai/openpilot/assets/8762862/92351544-2833-40d7-9e0b-7ef7ae37ec4c"></a></td>
+    <td><a href="https://youtu.be/SUIZYzxtMQs" title="A drive to Taco Bell"><img src="https://github.com/commaai/openpilot/assets/8762862/05ceefc5-2628-439c-a9b2-89ce77dc6f63"></a></td>
+  </tr>
+</table>
+
+
+Using openpilot in a car
+------
+
+To use openpilot in a car, you need four things:
+1. **Supported Device:** a comma 3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x).
+2. **Software:** The setup procedure for the comma 3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
+3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
+4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3X to your car.
+
+We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
+
+### Branches
+| branch           | URL                                    | description                                                                         |
+|------------------|----------------------------------------|-------------------------------------------------------------------------------------|
+| `release3`         | openpilot.comma.ai                      | This is openpilot's release branch.                                                 |
+| `release3-staging` | openpilot-test.comma.ai                | This is the staging branch for releases. Use it to get new releases slightly early. |
+| `nightly`          | openpilot-nightly.comma.ai             | This is the bleeding edge development branch. Do not expect this to be stable.      |
+| `nightly-dev`      | installer.comma.ai/commaai/nightly-dev | Same as nightly, but includes experimental development features for some cars.      |
+
+To start developing openpilot
+------
+
+openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot).
+
+* Join the [community Discord](https://discord.comma.ai)
+* Check out [the contributing docs](docs/CONTRIBUTING.md)
+* Check out the [openpilot tools](tools/)
+* Code documentation lives at https://docs.comma.ai
+* Information about running openpilot lives on the [community wiki](https://github.com/commaai/openpilot/wiki)
+
+Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs#open-positions) and offers lots of [bounties](https://comma.ai/bounties) for external contributors.
+
+Safety and Testing
+----
+
+* openpilot observes [ISO26262](https://en.wikipedia.org/wiki/ISO_26262) guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
+* openpilot has software-in-the-loop [tests](.github/workflows/selfdrive_tests.yaml) that run on every commit.
+* The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
+* panda has software-in-the-loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
+* Internally, we have a hardware-in-the-loop Jenkins test suite that builds and unit tests the various processes.
+* panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
+* We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
+
+<details>
+<summary>MIT Licensed</summary>
+
+openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
+
+Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
+
+**THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
+YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
+NO WARRANTY EXPRESSED OR IMPLIED.**
+</details>
+
+<details>
+<summary>User Data and comma Account</summary>
+
+By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
+
+openpilot is open source software: the user is free to disable data collection if they wish to do so.
+
+openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
+The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
+
+By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
+</details>

SAFETY.md

@@ -1,73 +0,0 @@
-openpilot Safety
-======
-
-openpilot is an Adaptive Cruise Control (ACC) and Lane Keeping Assist (LKA) system. 
-Like other ACC and LKA systems, openpilot requires the driver to be alert and to 
-pay attention at all times. We repeat, **driver alertness is necessary, but not 
-sufficient, for openpilot to be used safely**.
-
-Even with an attentive driver, we must make further efforts for the system to be
-safe. We have designed openpilot with two other safety considerations.
-
-1. The driver must always be capable to immediately retake manual control of the vehicle, 
-   by stepping on either pedal or by pressing the cancel button.
-2. The vehicle must not alter its trajectory too quickly for the driver to safely
-   react. This means that while the system is engaged, the actuators are constrained
-   to operate within reasonable limits.
-
-Following are details of the car specific safety implementations:
-
-Honda/Acura
-------
-
-  - While the system is engaged, gas, brake and steer limits are subject to the same limits used by
-    the stock system.
-
-  - Without an interceptor, the gas is controlled by the Powertrain Control Module (PCM). 
-    The PCM limits acceleration to what is reasonable for a cruise control system.  With an
-    interceptor, the gas is clipped to 60%.
-
-  - The brake is controlled by the 0x1FA CAN message. This message allows full
-    braking, although the board and the software clip it to 1/4th of the max.
-    This is around .3g of braking.
-
-  - Steering is controlled by the 0xE4 CAN message. The Electronic Power Steering (EPS) 
-    controller in the car limits the torque to a very small amount, so regardless of the 
-    message, the controller cannot jerk the wheel.
-
-  - Brake and gas pedal pressed signals are contained in the 0x17C CAN message. A rising edge of
-    either signal triggers a disengagement, which is enforced by the board and in software. The
-    green led on the board signifies if the board is allowing control messages.
-
-  - Honda CAN uses both a counter and a checksum to ensure integrity and prevent
-    replay of the same message.
-
-Toyota
-------
-
-  - While the system is engaged, gas, brake and steer limits are subject to the same limits used by
-    the stock system.
-
-  - With the stock Driving Support Unit (DSU) enabled, the acceleration is controlled 
-    by the stock system and is subject to the stock adaptive cruise control limits. Without the
-    stock DSU connected, the acceleration command is controlled by the 0x343 CAN message and its
-    value is limited by the board and the software to between .3g of deceleration and .15g of
-    acceleration. The acceleration command is ignored by the Engine Control Module (ECM) while the
-    cruise control system is disengaged.
-
-  - Steering torque is controlled through the 0x2E4 CAN message and it's limited by the board and in
-    software to a value of -1500 and 1500. In addition, the vehicle EPS unit will not respond to
-    commands outside these limits.  A steering torque rate limit is enforced by the board and in
-    software so that the commanded steering torque must rise from 0 to max value no faster than
-    1.5s. Commanded steering torque is limited by the board and in software to be no more than 500
-    units above the actual EPS generated motor torque to ensure limited differences between
-    commanded and actual torques.
-
-  - Brake and gas pedal pressed signals are contained in the 0x224 and 0x1D2 CAN messages,
-    respectively. A rising edge of either signal triggers a disengagement, which is enforced by the
-    board and in software. Additionally, the cruise control system disengages on the rising edge of
-    the brake pedal pressed signal.
-
-  - The cruise control system state is contained in the 0x1D2 message. No control messages are
-    allowed if the cruise control system is not active. This is enforced by the software and the
-    board. The green led on the board signifies if the board is allowing control messages.

SConstruct

@@ -0,0 +1,224 @@
+import os
+import subprocess
+import sys
+import sysconfig
+import platform
+import shlex
+import numpy as np
+
+import SCons.Errors
+
+SCons.Warnings.warningAsException(True)
+
+Decider('MD5-timestamp')
+
+SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+
+AddOption('--kaitai', action='store_true', help='Regenerate kaitai struct parsers')
+AddOption('--asan', action='store_true', help='turn on ASAN')
+AddOption('--ubsan', action='store_true', help='turn on UBSan')
+AddOption('--mutation', action='store_true', help='generate mutation-ready code')
+AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
+AddOption('--minimal',
+          action='store_false',
+          dest='extras',
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
+          help='the minimum build to run openpilot. no tests, tools, etc.')
+
+# Detect platform
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and os.path.isfile('/TICI'):
+  arch = "larch64"
+assert arch in [
+  "larch64",  # linux tici arm64
+  "aarch64",  # linux pc arm64
+  "x86_64",   # linux pc x64
+  "Darwin",   # macOS arm64 (x86 not supported)
+]
+
+env = Environment(
+  ENV={
+    "PATH": os.environ['PATH'],
+    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+  },
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Wunused",
+    "-Werror",
+    "-Wshadow",
+    "-Wno-unknown-warning-option",
+    "-Wno-inconsistent-missing-override",
+    "-Wno-c99-designator",
+    "-Wno-reorder-init-list",
+    "-Wno-vla-cxx-extension",
+  ],
+  CFLAGS=["-std=gnu11"],
+  CXXFLAGS=["-std=c++1z"],
+  CPPPATH=[
+    "#",
+    "#msgq",
+    "#third_party",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/acados/include",
+    "#third_party/acados/include/blasfeo/include",
+    "#third_party/acados/include/hpipm/include",
+    "#third_party/catch2/include",
+    "#third_party/libyuv/include",
+  ],
+  LIBPATH=[
+    "#common",
+    "#msgq_repo",
+    "#third_party",
+    "#selfdrive/pandad",
+    "#rednose/helpers",
+    f"#third_party/libyuv/{arch}/lib",
+    f"#third_party/acados/{arch}/lib",
+  ],
+  RPATH=[],
+  CYTHONCFILESUFFIX=".cpp",
+  COMPILATIONDB_USE_ABSPATH=True,
+  REDNOSE_ROOT="#",
+  tools=["default", "cython", "compilation_db", "rednose_filter"],
+  toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
+)
+
+# Arch-specific flags and paths
+if arch == "larch64":
+  env.Append(CPPPATH=["#third_party/opencl/include"])
+  env.Append(LIBPATH=[
+    "/usr/local/lib",
+    "/system/vendor/lib64",
+    "/usr/lib/aarch64-linux-gnu",
+  ])
+  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57"]
+  env.Append(CCFLAGS=arch_flags)
+  env.Append(CXXFLAGS=arch_flags)
+elif arch == "Darwin":
+  env.Append(LIBPATH=[
+    f"{brew_prefix}/lib",
+    f"{brew_prefix}/opt/openssl@3.0/lib",
+    f"{brew_prefix}/opt/llvm/lib/c++",
+    "/System/Library/Frameworks/OpenGL.framework/Libraries",
+  ])
+  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CPPPATH=[
+    f"{brew_prefix}/include",
+    f"{brew_prefix}/opt/openssl@3.0/include",
+  ])
+else:
+  env.Append(LIBPATH=[
+    "/usr/lib",
+    "/usr/local/lib",
+  ])
+
+# Sanitizers and extra CCFLAGS from CLI
+if GetOption('asan'):
+  env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
+  env.Append(LINKFLAGS=["-fsanitize=address"])
+elif GetOption('ubsan'):
+  env.Append(CCFLAGS=["-fsanitize=undefined"])
+  env.Append(LINKFLAGS=["-fsanitize=undefined"])
+
+_extra_cc = shlex.split(GetOption('ccflags') or '')
+if _extra_cc:
+  env.Append(CCFLAGS=_extra_cc)
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
+
+# progress output
+node_interval = 5
+node_count = 0
+def progress_function(node):
+  global node_count
+  node_count += node_interval
+  sys.stderr.write("progress: %d\n" % node_count)
+if os.environ.get('SCONS_PROGRESS'):
+  Progress(progress_function, interval=node_interval)
+
+# ********** Cython build environment **********
+py_include = sysconfig.get_paths()['include']
+envCython = env.Clone()
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove("-Werror")
+
+envCython["LIBS"] = []
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+np_version = SCons.Script.Value(np.__version__)
+Export('envCython', 'np_version')
+
+Export('env', 'arch')
+
+# Setup cache dir
+cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
+
+# ********** start building stuff **********
+
+# Build common module
+SConscript(['common/SConscript'])
+Import('_common')
+common = [_common, 'json11', 'zmq']
+Export('common')
+
+# Build messaging (cereal + msgq + socketmaster + their dependencies)
+# Enable swaglog include in submodules
+env_swaglog = env.Clone()
+env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
+SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
+
+SConscript(['cereal/SConscript'])
+
+Import('socketmaster', 'msgq')
+messaging = [socketmaster, msgq, 'capnp', 'kj',]
+Export('messaging')
+
+
+# Build other submodules
+SConscript(['panda/SConscript'])
+SConscript(['panda_tici/SConscript'])
+
+# Build rednose library
+SConscript(['rednose/SConscript'])
+
+# Build system services
+SConscript([
+  'system/ubloxd/SConscript',
+  'system/loggerd/SConscript',
+])
+
+if arch == "larch64":
+  SConscript(['system/camerad/SConscript'])
+
+# Build openpilot
+SConscript(['third_party/SConscript'])
+
+SConscript(['selfdrive/SConscript'])
+
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
+  SConscript(['tools/replay/SConscript'])
+  if arch != "larch64":
+    SConscript(['tools/cabana/SConscript'])
+
+
+env.CompilationDatabase('compile_commands.json')

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Suspected vulnerabilities can be reported to both `adeeb@comma.ai` and `security@comma.ai`.

SPONSORS.md

@@ -0,0 +1,26 @@
+# Sponsors 贊助者
+
+我們誠摯感謝以下贊助者提供的硬體資源，讓專案能夠在多種平台上進行測試與驗證。
+
+We sincerely thank the following sponsors for providing hardware resources, which enable the project to be tested and validated across multiple platforms.
+
+---
+
+## 贊助者列表 Sponsors
+
+| 贊助者 Sponsor | 設備 Deices | 備註 Notes |
+| -------------- | ----------- | ---------- |
+| BlueGood | <ul><li>Radar Filter x 2</li><li>sDSU x1</li></ul> | - |
+| Chia Chun Lee | <ul><li>C3 Quick Mount x1</li></ul> | - |
+| CloudJ | <ul><li>C3X x1</li></ul> | - |
+| FareWay | <ul><li>EON Quick Mount x1</li><li>C2/C3 Quick Mount x1</li></ul> | Special thanks for fixing my good old EON |
+| Fred Wang | <ul><li>Oneplus 3t x1</li></ul> | - |
+| Saber Huang | <ul><li>O3L x1</li></ul> | - |
+| [馬威 Mr. One](https://shop61532546.taobao.com/) | <ul><li>O3 x 1</li><li>O3 (Dev) x1</li><li>O3XL x1</li><li>Red Panda x1</li><li>Panda Jungle v1 x1</li></ul> | - |
+| 門文梁 | <ul><li>C1.5 x1</li></ul> | - |
+
+---
+
+🙏 沒有你們的支持，我們無法讓專案在這麼多硬體平台上持續成長與驗證。
+
+Without your support, this project could not continue to grow and be validated across so many hardware platforms.

apk/external/.gitignore

@@ -1,2 +0,0 @@
-src/*
-out/*

apk/external/patcher.py

@@ -1,122 +0,0 @@
-#!/usr/bin/env python2.7
-
-import os
-import sys
-import glob
-import shutil
-import urllib2
-import hashlib
-import subprocess
-
-
-EXTERNAL_PATH = os.path.dirname(os.path.abspath(__file__))
-
-if os.path.exists("/init.qcom.rc"):
-  # android
-  APKPATCH = os.path.join(EXTERNAL_PATH, 'tools/apkpatch_android')
-  SIGNAPK = os.path.join(EXTERNAL_PATH, 'tools/signapk_android')
-else:
-  APKPATCH = os.path.join(EXTERNAL_PATH, 'tools/apkpatch')
-  SIGNAPK = os.path.join(EXTERNAL_PATH, 'tools/signapk')
-
-APKS = {
-  'com.waze': {
-    'src': 'https://apkcache.s3.amazonaws.com/com.waze_1021278.apk',
-    'src_sha256': 'f00957e93e2389f9e30502ac54994b98ac769314b0963c263d4e8baa625ab0c2',
-    'patch': 'com.waze.apkpatch',
-    'out_sha256': '9ec8b0ea3c78c666342865b1bfb66e368a3f5c911df2ad12835206ec8b19f444'
-  },
-  'com.spotify.music': {
-    'src': 'https://apkcache.s3.amazonaws.com/com.spotify.music_24382006.apk',
-    'src_sha256': '0610fea68ee7ba5f8e4e0732ad429d729dd6cbb8bc21222c4c99db6cb09fbff4',
-    'patch': 'com.spotify.music.apkpatch',
-    'out_sha256': '5a3d6f478c7e40403a98ccc8906d7e0ae12b06543b41f5df52149dd09c647c11'
-  },
-}
-
-def sha256_path(path):
-  with open(path, 'rb') as f:
-    return hashlib.sha256(f.read()).hexdigest()
-
-def remove(path):
-  try:
-    os.remove(path)
-  except OSError:
-    pass
-
-def process(download, patch):
-  # clean up any junk apks
-  for out_apk in glob.glob(os.path.join(EXTERNAL_PATH, 'out/*.apk')):
-    app = os.path.basename(out_apk)[:-4]
-    if app not in APKS:
-      print "remove junk", out_apk
-      remove(out_apk)
-
-  complete = True
-  for k,v in APKS.iteritems():
-    apk_path = os.path.join(EXTERNAL_PATH, 'out', k+'.apk')
-    print "checking", apk_path
-    if os.path.exists(apk_path) and sha256_path(apk_path) == v['out_sha256']:
-      # nothing to do
-      continue
-
-    complete = False
-
-    remove(apk_path)
-
-    src_path = os.path.join(EXTERNAL_PATH, 'src', v['src_sha256'])
-    if not os.path.exists(src_path) or sha256_path(src_path) != v['src_sha256']:
-      if not download:
-        continue
-
-      print "downloading", v['src'], "to", src_path
-      # download it
-      resp = urllib2.urlopen(v['src'])
-      data = resp.read()
-      with open(src_path, 'wb') as src_f:
-        src_f.write(data)
-
-      if sha256_path(src_path) != v['src_sha256']:
-        print "download was corrupted..."
-        continue
-
-    if not patch:
-      continue
-
-    # ignoring lots of TOCTTOU here...
-
-    apk_temp = "/tmp/"+k+".patched"
-    remove(apk_temp)
-    apk_temp2 = "/tmp/"+k+".signed"
-    remove(apk_temp2)
-
-    try:
-      print "patching", v['patch']
-      subprocess.check_call([APKPATCH, 'apply', src_path, apk_temp, os.path.join(EXTERNAL_PATH, v['patch'])])
-      print "signing", apk_temp
-      subprocess.check_call([SIGNAPK,
-        os.path.join(EXTERNAL_PATH, 'tools/certificate.pem'), os.path.join(EXTERNAL_PATH, 'tools/key.pk8'),
-        apk_temp, apk_temp2])
-
-      out_sha256 = sha256_path(apk_temp2) if os.path.exists(apk_temp2) else None
-
-      if out_sha256 == v['out_sha256']:
-        print "done", apk_path
-        shutil.move(apk_temp2, apk_path)
-      else:
-        print "patch was corrupted", apk_temp2, out_sha256
-    finally:
-      remove(apk_temp)
-      remove(apk_temp2)
-
-  return complete
-
-if __name__ == "__main__":
-  ret = True
-  if len(sys.argv) == 2 and sys.argv[1] == "download":
-    ret = process(True, False)
-  elif len(sys.argv) == 2 and sys.argv[1] == "patch":
-    ret = process(False, True)
-  else:
-    ret = process(True, True)
-  sys.exit(0 if ret else 1)

apk/external/tools/apkpatch_android

@@ -1,7 +0,0 @@
-#!/system/bin/sh
-
-DIR="$(cd "$(dirname "$0")" && pwd)"
-
-export LD_LIBRARY_PATH=/system/lib64
-export CLASSPATH="$DIR"/ApkPatch.android.jar
-exec app_process "$DIR" ApkPatch "$@"

apk/external/tools/certificate.pem

@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE-----
-MIICtTCCAh4CCQDm79UqF+Dc5zANBgkqhkiG9w0BAQUFADCBnjELMAkGA1UEBhMC
-SUQxEzARBgNVBAgTCkphd2EgQmFyYXQxEDAOBgNVBAcTB0JhbmR1bmcxEjAQBgNV
-BAoTCUxvbmRhdGlnYTETMBEGA1UECxMKQW5kcm9pZERldjEaMBgGA1UEAxMRTG9y
-ZW5zaXVzIFcuIEwuIFQxIzAhBgkqhkiG9w0BCQEWFGxvcmVuekBsb25kYXRpZ2Eu
-bmV0MB4XDTEwMDUwNTA5MjEzOFoXDTEzMDEyODA5MjEzOFowgZ4xCzAJBgNVBAYT
-AklEMRMwEQYDVQQIEwpKYXdhIEJhcmF0MRAwDgYDVQQHEwdCYW5kdW5nMRIwEAYD
-VQQKEwlMb25kYXRpZ2ExEzARBgNVBAsTCkFuZHJvaWREZXYxGjAYBgNVBAMTEUxv
-cmVuc2l1cyBXLiBMLiBUMSMwIQYJKoZIhvcNAQkBFhRsb3JlbnpAbG9uZGF0aWdh
-Lm5ldDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAy2oWtbdVXMHGiS6cA3qi
-3VfZt5Vz9jTlux+TEcGx5h18ZKwclyo+z2B0L/p5bYdnrTdFEiD7IxvX+h3lu0JV
-B9rdXZdyrzXNOw5YFrsn2k7hKvB8KEBaga1gZEwodlc6N14H3FbZdZkIA9V716Pu
-e5CWBZ2VqU03lUJmKnpH8c8CAwEAATANBgkqhkiG9w0BAQUFAAOBgQBpNgXh8dw9
-uMjZxzLUXovV5ptHd61jAcZlQlffqPsz6/2QNfIShVdGH9jkm0IudfKkbvvOKive
-a77t9c4sDh2Sat2L/rx6BfTuS1+y9wFr1Ee8Rrr7wGHhRkx2qqGrXGVWqXn8aE3E
-P6e7BTPF0ibS+tG8cdDPEisqGFxw36nTNQ==
------END CERTIFICATE-----

apk/external/tools/signapk_android

@@ -1,7 +0,0 @@
-#!/system/bin/sh
-
-DIR="$(cd "$(dirname "$0")" && pwd)"
-
-export LD_LIBRARY_PATH=/system/lib64
-export CLASSPATH="$DIR"/signapk.android.jar
-exec app_process "$DIR" com.android.signapk.SignApk "$@"

cereal/.gitignore

@@ -1 +0,0 @@
-gen

cereal/Makefile

@@ -1,52 +0,0 @@
-PWD := $(shell pwd)
-
-SRCS := log.capnp car.capnp
-
-GENS := gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++
-
-UNAME_M ?= $(shell uname -m)
-
-# only generate C++ for docker tests
-ifneq ($(OPTEST),1)
-	GENS += gen/c/car.capnp.c gen/c/log.capnp.c gen/c/c++.capnp.h gen/c/java.capnp.h
-
-# Dont build java on the phone...
-ifeq ($(UNAME_M),x86_64)
-	GENS += gen/java/Car.java gen/java/Log.java
-endif
-
-endif
-
-ifeq ($(UNAME_M),aarch64)
-CAPNPC=PATH=$(PWD)/../phonelibs/capnp-cpp/aarch64/bin/:$$PATH capnpc
-else
-CAPNPC=capnpc
-endif
-
-.PHONY: all
-all: $(GENS)
-
-.PHONY: clean
-clean:
-	rm -rf gen
-
-gen/c/%.capnp.c: %.capnp
-	@echo "[ CAPNPC C ] $@"
-	mkdir -p gen/c/
-	$(CAPNPC) '$<' -o c:gen/c/
-
-gen/cpp/%.capnp.c++: %.capnp
-	@echo "[ CAPNPC C++ ] $@"
-	mkdir -p gen/cpp/
-	$(CAPNPC) '$<' -o c++:gen/cpp/
-
-gen/java/Car.java gen/java/Log.java: $(SRCS)
-	@echo "[ CAPNPC java ] $@"
-	mkdir -p gen/java/
-	$(CAPNPC) $^ -o java:gen/java
-
-# c-capnproto needs some empty headers
-gen/c/c++.capnp.h gen/c/java.capnp.h:
-	mkdir -p gen/c/
-	touch '$@'
-

cereal/README.md

@@ -0,0 +1,95 @@
+# What is cereal?
+
+cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
+
+
+## Messaging Spec
+
+You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
+
+All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
+
+### Best Practices
+
+- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
+- In the context of the message they are in, field names should be completely unambiguous.
+- All values should be easy to plot and be human-readable with minimal parsing.
+
+### Maintaining backwards-compatibility
+
+When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
+be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
+things are not. Read more details [here](https://capnproto.org/language.html).
+
+### Custom forks
+
+Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
+spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
+then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
+[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
+fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
+
+An example of compatible changes:
+```diff
+diff --git a/cereal/custom.capnp b/cereal/custom.capnp
+index 3348e859e..3365c7b98 100644
+--- a/cereal/custom.capnp
++++ b/cereal/custom.capnp
+@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
+ # DO rename the structs
+ # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
+-struct CustomReserved0 @0x81c2f05a394cf4af {
++struct SteeringInfo @0x81c2f05a394cf4af {
++  active @0 :Bool;
++  steeringAngleDeg @1 :Float32;
++  steeringRateDeg @2 :Float32;
++  steeringAccelDeg @3 :Float32;
+ }
+
+ struct CustomReserved1 @0xaedffd8f31e7b55d {
+diff --git a/cereal/log.capnp b/cereal/log.capnp
+index 1209f3fd9..b189f58b6 100644
+--- a/cereal/log.capnp
++++ b/cereal/log.capnp
+@@ -2558,14 +2558,14 @@ struct Event {
+
+     # DO change the name of the field
+     # DON'T change anything after the "@"
+-    customReservedRawData0 @124 :Data;
++    rawCanData @124 :Data;
+     customReservedRawData1 @125 :Data;
+     customReservedRawData2 @126 :Data;
+
+     # DO change the name of the field and struct
+     # DON'T change the ID (e.g. @107)
+     # DON'T change which struct it points to
+-    customReserved0 @107 :Custom.CustomReserved0;
++    steeringInfo @107 :Custom.SteeringInfo;
+     customReserved1 @108 :Custom.CustomReserved1;
+     customReserved2 @109 :Custom.CustomReserved2;
+     customReserved3 @110 :Custom.CustomReserved3;
+```
+
+---
+
+Example
+---
+```python
+import cereal.messaging as messaging
+
+# in subscriber
+sm = messaging.SubMaster(['sensorEvents'])
+while 1:
+  sm.update()
+  print(sm['sensorEvents'])
+
+```
+
+```python
+# in publisher
+pm = messaging.PubMaster(['sensorEvents'])
+dat = messaging.new_message('sensorEvents', size=1)
+dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
+pm.send('sensorEvents', dat)
+```

cereal/SConscript

@@ -0,0 +1,20 @@
+Import('env', 'common', 'msgq')
+
+cereal_dir = Dir('.')
+gen_dir = Dir('gen')
+
+# Build cereal
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
+env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
+            schema_files,
+            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
+
+cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
+
+# Build messaging
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+
+socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
+
+Export('cereal', 'socketmaster')

cereal/__init__.py

@@ -1,20 +1,11 @@
 import os
 import capnp
+from importlib.resources import as_file, files
 
-CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
 capnp.remove_import_hook()
 
-if os.getenv("NEWCAPNP"):
-  import tempfile
-  import pyximport
-
-  importers = pyximport.install(build_dir=os.path.join(tempfile.gettempdir(), ".pyxbld"))
-  try:
-    import cereal.gen.cython.log_capnp_cython as log
-    import cereal.gen.cython.car_capnp_cython as car
-  finally:
-    pyximport.uninstall(*importers)
-    del importers
-else:
+with as_file(files("cereal")) as fspath:
+  CEREAL_PATH = fspath.as_posix()
   log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
   car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -1,315 +0,0 @@
-using Cxx = import "c++.capnp";
-$Cxx.namespace("cereal");
-
-using Java = import "java.capnp";
-$Java.package("ai.comma.openpilot.cereal");
-$Java.outerClassname("Car");
-
-@0x8e2af1e708af8b8d;
-
-# ******* events causing controls state machine transition *******
-
-struct CarEvent @0x9b1657f34caf3ad3 {
-  name @0 :EventName;
-  enable @1 :Bool;
-  noEntry @2 :Bool;
-  warning @3 :Bool;
-  userDisable @4 :Bool;
-  softDisable @5 :Bool;
-  immediateDisable @6 :Bool;
-  preEnable @7 :Bool;
-  permanent @8 :Bool;
-
-  enum EventName @0xbaa8c5d505f727de {
-    # TODO: copy from error list
-    commIssue @0;
-    steerUnavailable @1;
-    brakeUnavailable @2;
-    gasUnavailable @3;
-    wrongGear @4;
-    doorOpen @5;
-    seatbeltNotLatched @6;
-    espDisabled @7;
-    wrongCarMode @8;
-    steerTempUnavailable @9;
-    reverseGear @10;
-    buttonCancel @11;
-    buttonEnable @12;
-    pedalPressed @13;
-    cruiseDisabled @14;
-    radarCommIssue @15;
-    dataNeeded @16;
-    speedTooLow @17;
-    outOfSpace @18;
-    overheat @19;
-    calibrationInProgress @20;
-    calibrationInvalid @21;
-    controlsMismatch @22;
-    pcmEnable @23;
-    pcmDisable @24;
-    noTarget @25;
-    radarFault @26;
-    modelCommIssue @27;
-    brakeHold @28;
-    parkBrake @29;
-    manualRestart @30;
-    lowSpeedLockout @31;
-  }
-}
-
-# ******* main car state @ 100hz *******
-# all speeds in m/s
-
-struct CarState {
-  errorsDEPRECATED @0 :List(CarEvent.EventName);
-  events @13 :List(CarEvent);
-
-  # car speed
-  vEgo @1 :Float32;         # best estimate of speed
-  aEgo @16 :Float32;        # best estimate of acceleration
-  vEgoRaw @17 :Float32;     # unfiltered speed from CAN sensors
-  yawRate @22 :Float32;     # best estimate of yaw rate
-  standstill @18 :Bool;
-  wheelSpeeds @2 :WheelSpeeds;
-
-  # gas pedal, 0.0-1.0
-  gas @3 :Float32;        # this is user + computer
-  gasPressed @4 :Bool;    # this is user pedal only
-
-  # brake pedal, 0.0-1.0
-  brake @5 :Float32;      # this is user pedal only
-  brakePressed @6 :Bool;  # this is user pedal only
-  brakeLights @19 :Bool;
-
-  # steering wheel
-  steeringAngle @7 :Float32;   # deg
-  steeringRate @15 :Float32;   # deg/s
-  steeringTorque @8 :Float32;  # TODO: standardize units
-  steeringPressed @9 :Bool;    # if the user is using the steering wheel
-
-  # cruise state
-  cruiseState @10 :CruiseState;
-
-  # gear
-  gearShifter @14 :GearShifter;
-
-  # button presses
-  buttonEvents @11 :List(ButtonEvent);
-  leftBlinker @20 :Bool;
-  rightBlinker @21 :Bool;
-
-  # which packets this state came from
-  canMonoTimes @12: List(UInt64);
-
-  struct WheelSpeeds {
-    # optional wheel speeds
-    fl @0 :Float32;
-    fr @1 :Float32;
-    rl @2 :Float32;
-    rr @3 :Float32;
-  }
-
-  struct CruiseState {
-    enabled @0 :Bool;
-    speed @1 :Float32;
-    available @2 :Bool;
-    speedOffset @3 :Float32;
-    standstill @4 :Bool;
-  }
-
-  enum GearShifter {
-    unknown @0;
-    park @1;
-    drive @2;
-    neutral @3;
-    reverse @4;
-    sport @5;
-    low @6;
-    brake @7;
-  }
-
-
-  # send on change
-  struct ButtonEvent {
-    pressed @0 :Bool;
-    type @1 :Type;
-
-    enum Type {
-      unknown @0;
-      leftBlinker @1;
-      rightBlinker @2;
-      accelCruise @3;
-      decelCruise @4;
-      cancel @5;
-      altButton1 @6;
-      altButton2 @7;
-      altButton3 @8;
-    }
-  }
-}
-
-# ******* radar state @ 20hz *******
-
-struct RadarState {
-  errors @0 :List(Error);
-  points @1 :List(RadarPoint);
-
-  # which packets this state came from
-  canMonoTimes @2 :List(UInt64);
-
-  enum Error {
-    commIssue @0;
-    fault @1;
-  }
-
-  # similar to LiveTracks
-  # is one timestamp valid for all? I think so
-  struct RadarPoint {
-    trackId @0 :UInt64;  # no trackId reuse
-
-    # these 3 are the minimum required
-    dRel @1 :Float32; # m from the front bumper of the car
-    yRel @2 :Float32; # m
-    vRel @3 :Float32; # m/s
-
-    # these are optional and valid if they are not NaN
-    aRel @4 :Float32; # m/s^2
-    yvRel @5 :Float32; # m/s
-
-    # some radars flag measurements VS estimates
-    measured @6 :Bool;
-  }
-}
-
-# ******* car controls @ 100hz *******
-
-struct CarControl {
-  # must be true for any actuator commands to work
-  enabled @0 :Bool;
-
-  gasDEPRECATED @1 :Float32;
-  brakeDEPRECATED @2 :Float32;
-  steeringTorqueDEPRECATED @3 :Float32;
-
-  actuators @6 :Actuators;
-
-  cruiseControl @4 :CruiseControl;
-  hudControl @5 :HUDControl;
-
-  struct Actuators {
-    # range from 0.0 - 1.0
-    gas @0: Float32;
-    brake @1: Float32;
-    # range from -1.0 - 1.0
-    steer @2: Float32;
-  }
-
-  struct CruiseControl {
-    cancel @0: Bool;
-    override @1: Bool;
-    speedOverride @2: Float32;
-    accelOverride @3: Float32;
-  }
-
-  struct HUDControl {
-    speedVisible @0: Bool;
-    setSpeed @1: Float32;
-    lanesVisible @2: Bool;
-    leadVisible @3: Bool;
-    visualAlert @4: VisualAlert;
-    audibleAlert @5: AudibleAlert;
-
-    enum VisualAlert {
-      # these are the choices from the Honda
-      # map as good as you can for your car
-      none @0;
-      fcw @1;
-      steerRequired @2;
-      brakePressed @3;
-      wrongGear @4;
-      seatbeltUnbuckled @5;
-      speedTooHigh @6;
-    }
-
-    enum AudibleAlert {
-      # these are the choices from the Honda
-      # map as good as you can for your car
-      none @0;
-      beepSingle @1;
-      beepTriple @2;
-      beepRepeated @3;
-      chimeSingle @4;
-      chimeDouble @5;
-      chimeRepeated @6;
-      chimeContinuous @7;
-    }
-  }
-}
-
-# ****** car param ******
-
-struct CarParams {
-  carName @0 :Text;
-  radarName @1 :Text;
-  carFingerprint @2 :Text;
-
-  enableSteer @3 :Bool;
-  enableGas @4 :Bool;
-  enableBrake @5 :Bool;
-  enableCruise @6 :Bool;
-  enableCamera @26 :Bool;
-  enableDsu @27 :Bool; # driving support unit
-  enableApgs @28 :Bool; # advanced parking guidance system
-
-  minEnableSpeed @17 :Float32;
-  safetyModel @18 :Int16;
-
-  steerMaxBP @19 :List(Float32);
-  steerMaxV @20 :List(Float32);
-  gasMaxBP @21 :List(Float32);
-  gasMaxV @22 :List(Float32);
-  brakeMaxBP @23 :List(Float32);
-  brakeMaxV @24 :List(Float32);
-
-  longPidDeadzoneBP @32 :List(Float32);
-  longPidDeadzoneV @33 :List(Float32);
-
-  enum SafetyModels {
-    # does NOT match board setting
-    noOutput @0;
-    honda @1;
-    toyota @2;
-    elm327 @3;
-  }
-
-  # things about the car in the manual
-  mass @7 :Float32;             # [kg] running weight
-  wheelbase @8 :Float32;        # [m] distance from rear to front axle
-  centerToFront @9 :Float32;   # [m] GC distance to front axle
-  steerRatio @10 :Float32;       # [] ratio between front wheels and steering wheel angles
-  steerRatioRear @11 :Float32;  # [] rear steering ratio wrt front steering (usually 0)
-
-  # things we can derive
-  rotationalInertia @12 :Float32;    # [kg*m2] body rotational inertia
-  tireStiffnessFront @13 :Float32;   # [N/rad] front tire coeff of stiff
-  tireStiffnessRear @14 :Float32;    # [N/rad] rear tire coeff of stiff
-
-  # Kp and Ki for the lateral control
-  steerKp @15 :Float32;
-  steerKi @16 :Float32;
-  steerKf @25 :Float32;
-
-  # Kp and Ki for the longitudinal control
-  longitudinalKpBP @36 :List(Float32);
-  longitudinalKpV @37 :List(Float32);
-  longitudinalKiBP @38 :List(Float32);
-  longitudinalKiV @39 :List(Float32);
-
-  steerLimitAlert @29 :Bool;
-
-  vEgoStopping @30 :Float32; # Speed at which the car goes into stopping state
-  directAccelControl @31 :Bool; # Does the car have direct accel control or just gas/brake
-  stoppingControl @34 :Bool; # Does the car allows full control even at lows speeds when stopping
-  startAccel @35 :Float32; # Required acceleraton to overcome creep braking
-  steerRateCost @40 :Float32; # Lateral MPC cost on steering rate
-}

cereal/car.capnp

@@ -0,0 +1 @@
+../opendbc_repo/opendbc/car/car.capnp

cereal/custom.capnp

@@ -0,0 +1,74 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0xb526ba661d550a59;
+
+# custom.capnp: a home for empty structs reserved for custom forks
+# These structs are guaranteed to remain reserved and empty in mainline
+# cereal, so use these if you want custom events in your fork.
+
+# DO rename the structs
+# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
+struct DpControlsState @0x81c2f05a394cf4af {
+  alkaActive @0 :Bool;
+}
+
+struct ModelExt @0xaedffd8f31e7b55d {
+  leftEdgeDetected @0 :Bool;
+  rightEdgeDetected @1 :Bool;
+}
+
+struct CustomReserved2 @0xf35cc4560bbf6ec2 {
+}
+
+struct CustomReserved3 @0xda96579883444c35 {
+}
+
+struct CustomReserved4 @0x80ae746ee2596b11 {
+}
+
+struct CustomReserved5 @0xa5cd762cd951a455 {
+}
+
+struct CustomReserved6 @0xf98d843bfd7004a3 {
+}
+
+struct CustomReserved7 @0xb86e6369214c01c8 {
+}
+
+struct CustomReserved8 @0xf416ec09499d9d19 {
+}
+
+struct CustomReserved9 @0xa1680744031fdb2d {
+}
+
+struct CustomReserved10 @0xcb9fd56c7057593a {
+}
+
+struct CustomReserved11 @0xc2243c65e0340384 {
+}
+
+struct CustomReserved12 @0x9ccdc8676701b412 {
+}
+
+struct CustomReserved13 @0xcd96dafb67a082d0 {
+}
+
+struct CustomReserved14 @0xb057204d7deadf3f {
+}
+
+struct CustomReserved15 @0xbd443b539493bc68 {
+}
+
+struct CustomReserved16 @0xfc6241ed8877b611 {
+}
+
+struct CustomReserved17 @0xa30662f84033036c {
+}
+
+struct CustomReserved18 @0xc86a3d38d13eb3ef {
+}
+
+struct CustomReserved19 @0xa4f1eb3323f5f582 {
+}

cereal/java.capnp

@@ -1,28 +0,0 @@
-# Copyright (c) 2013-2015 Sandstorm Development Group, Inc. and contributors
-# Licensed under the MIT License:
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-@0xc5f1af96651f70ea;
-
-annotation package @0x9ee4c8f803b3b596 (file) : Text;
-# Name of the package, such as "org.example.foo", in which the generated code will reside.
-
-annotation outerClassname @0x9b066bb4881f7cd3 (file) : Text;
-# Name of the outer class that will wrap the generated code.

cereal/legacy.capnp

@@ -0,0 +1,574 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0x80ef1ec4889c2a63;
+
+# legacy.capnp: a home for deprecated structs
+
+struct LogRotate @0x9811e1f38f62f2d1 {
+  segmentNum @0 :Int32;
+  path @1 :Text;
+}
+
+struct LiveUI @0xc08240f996aefced {
+  rearViewCam @0 :Bool;
+  alertText1 @1 :Text;
+  alertText2 @2 :Text;
+  awarenessStatus @3 :Float32;
+}
+
+struct UiLayoutState @0x88dcce08ad29dda0 {
+  activeApp @0 :App;
+  sidebarCollapsed @1 :Bool;
+  mapEnabled @2 :Bool;
+  mockEngaged @3 :Bool;
+
+  enum App @0x9917470acf94d285 {
+    home @0;
+    music @1;
+    nav @2;
+    settings @3;
+    none @4;
+  }
+}
+
+struct OrbslamCorrection @0x8afd33dc9b35e1aa {
+  correctionMonoTime @0 :UInt64;
+  prePositionECEF @1 :List(Float64);
+  postPositionECEF @2 :List(Float64);
+  prePoseQuatECEF @3 :List(Float32);
+  postPoseQuatECEF @4 :List(Float32);
+  numInliers @5 :UInt32;
+}
+
+struct EthernetPacket @0xa99a9d5b33cf5859 {
+  pkt @0 :Data;
+  ts @1 :Float32;
+}
+
+struct CellInfo @0xcff7566681c277ce {
+  timestamp @0 :UInt64;
+  repr @1 :Text; # android toString() for now
+}
+
+struct WifiScan @0xd4df5a192382ba0b {
+  bssid @0 :Text;
+  ssid @1 :Text;
+  capabilities @2 :Text;
+  frequency @3 :Int32;
+  level @4 :Int32;
+  timestamp @5 :Int64;
+
+  centerFreq0 @6 :Int32;
+  centerFreq1 @7 :Int32;
+  channelWidth @8 :ChannelWidth;
+  operatorFriendlyName @9 :Text;
+  venueName @10 :Text;
+  is80211mcResponder @11 :Bool;
+  passpoint @12 :Bool;
+
+  distanceCm @13 :Int32;
+  distanceSdCm @14 :Int32;
+
+  enum ChannelWidth @0xcb6a279f015f6b51 {
+    w20Mhz @0;
+    w40Mhz @1;
+    w80Mhz @2;
+    w160Mhz @3;
+    w80Plus80Mhz @4;
+  }
+}
+
+struct LiveEventData @0x94b7baa90c5c321e {
+  name @0 :Text;
+  value @1 :Int32;
+}
+
+struct ModelData @0xb8aad62cffef28a9 {
+  frameId @0 :UInt32;
+  frameAge @12 :UInt32;
+  frameDropPerc @13 :Float32;
+  timestampEof @9 :UInt64;
+  modelExecutionTime @14 :Float32;
+  gpuExecutionTime @16 :Float32;
+  rawPred @15 :Data;
+
+  path @1 :PathData;
+  leftLane @2 :PathData;
+  rightLane @3 :PathData;
+  lead @4 :LeadData;
+  freePath @6 :List(Float32);
+
+  settings @5 :ModelSettings;
+  leadFuture @7 :LeadData;
+  speed @8 :List(Float32);
+  meta @10 :MetaData;
+  longitudinal @11 :LongitudinalData;
+
+  struct PathData @0x8817eeea389e9f08 {
+    points @0 :List(Float32);
+    prob @1 :Float32;
+    std @2 :Float32;
+    stds @3 :List(Float32);
+    poly @4 :List(Float32);
+    validLen @5 :Float32;
+  }
+
+  struct LeadData @0xd1c9bef96d26fa91 {
+    dist @0 :Float32;
+    prob @1 :Float32;
+    std @2 :Float32;
+    relVel @3 :Float32;
+    relVelStd @4 :Float32;
+    relY @5 :Float32;
+    relYStd @6 :Float32;
+    relA @7 :Float32;
+    relAStd @8 :Float32;
+  }
+
+  struct ModelSettings @0xa26e3710efd3e914 {
+    bigBoxX @0 :UInt16;
+    bigBoxY @1 :UInt16;
+    bigBoxWidth @2 :UInt16;
+    bigBoxHeight @3 :UInt16;
+    boxProjection @4 :List(Float32);
+    yuvCorrection @5 :List(Float32);
+    inputTransform @6 :List(Float32);
+  }
+
+  struct MetaData @0x9744f25fb60f2bf8 {
+    engagedProb @0 :Float32;
+    desirePrediction @1 :List(Float32);
+    brakeDisengageProb @2 :Float32;
+    gasDisengageProb @3 :Float32;
+    steerOverrideProb @4 :Float32;
+    desireState @5 :List(Float32);
+  }
+
+  struct LongitudinalData @0xf98f999c6a071122 {
+    distances @2 :List(Float32);
+    speeds @0 :List(Float32);
+    accelerations @1 :List(Float32);
+  }
+}
+
+struct ECEFPoint @0xc25bbbd524983447 {
+  x @0 :Float64;
+  y @1 :Float64;
+  z @2 :Float64;
+}
+
+struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 {
+  x @0 :Float32;
+  y @1 :Float32;
+  z @2 :Float32;
+}
+
+struct GPSPlannerPoints @0xab54c59699f8f9f3 {
+  curPosDEPRECATED @0 :ECEFPointDEPRECATED;
+  pointsDEPRECATED @1 :List(ECEFPointDEPRECATED);
+  curPos @6 :ECEFPoint;
+  points @7 :List(ECEFPoint);
+  valid @2 :Bool;
+  trackName @3 :Text;
+  speedLimit @4 :Float32;
+  accelTarget @5 :Float32;
+}
+
+struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b {
+  valid @0 :Bool;
+  poly @1 :List(Float32);
+  trackName @2 :Text;
+  speed @3 :Float32;
+  acceleration @4 :Float32;
+  pointsDEPRECATED @5 :List(ECEFPointDEPRECATED);
+  points @6 :List(ECEFPoint);
+  xLookahead @7 :Float32;
+}
+
+struct UiNavigationEvent @0x90c8426c3eaddd3b {
+  type @0: Type;
+  status @1: Status;
+  distanceTo @2: Float32;
+  endRoadPointDEPRECATED @3: ECEFPointDEPRECATED;
+  endRoadPoint @4: ECEFPoint;
+
+  enum Type @0xe8db07dcf8fcea05 {
+    none @0;
+    laneChangeLeft @1;
+    laneChangeRight @2;
+    mergeLeft @3;
+    mergeRight @4;
+    turnLeft @5;
+    turnRight @6;
+  }
+
+  enum Status @0xb9aa88c75ef99a1f {
+    none @0;
+    passive @1;
+    approaching @2;
+    active @3;
+  }
+}
+
+struct LiveLocationData @0xb99b2bc7a57e8128 {
+  status @0 :UInt8;
+
+  # 3D fix
+  lat @1 :Float64;
+  lon @2 :Float64;
+  alt @3 :Float32;     # m
+
+  # speed
+  speed @4 :Float32;   # m/s
+
+  # NED velocity components
+  vNED @5 :List(Float32);
+
+  # roll, pitch, heading (x,y,z)
+  roll @6 :Float32;     # WRT to center of earth?
+  pitch @7 :Float32;    # WRT to center of earth?
+  heading @8 :Float32;  # WRT to north?
+
+  # what are these?
+  wanderAngle @9 :Float32;
+  trackAngle @10 :Float32;
+
+  # car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png
+
+  # gyro, in car frame, deg/s
+  gyro @11 :List(Float32);
+
+  # accel, in car frame, m/s^2
+  accel @12 :List(Float32);
+
+  accuracy @13 :Accuracy;
+
+  source @14 :SensorSource;
+  # if we are fixing a location in the past
+  fixMonoTime @15 :UInt64;
+
+  gpsWeek @16 :Int32;
+  timeOfWeek @17 :Float64;
+
+  positionECEF @18 :List(Float64);
+  poseQuatECEF @19 :List(Float32);
+  pitchCalibration @20 :Float32;
+  yawCalibration @21 :Float32;
+  imuFrame @22 :List(Float32);
+
+  struct Accuracy @0x943dc4625473b03f {
+    pNEDError @0 :List(Float32);
+    vNEDError @1 :List(Float32);
+    rollError @2 :Float32;
+    pitchError @3 :Float32;
+    headingError @4 :Float32;
+    ellipsoidSemiMajorError @5 :Float32;
+    ellipsoidSemiMinorError @6 :Float32;
+    ellipsoidOrientationError @7 :Float32;
+  }
+
+  enum SensorSource @0xc871d3cc252af657 {
+    applanix @0;
+    kalman @1;
+    orbslam @2;
+    timing @3;
+    dummy @4;
+  }
+}
+
+struct OrbOdometry @0xd7700859ed1f5b76 {
+  # timing first
+  startMonoTime @0 :UInt64;
+  endMonoTime @1 :UInt64;
+
+  # fundamental matrix and error
+  f @2: List(Float64);
+  err @3: Float64;
+
+  # number of inlier points
+  inliers @4: Int32;
+
+  # for debug only
+  # indexed by endMonoTime features
+  # value is startMonoTime feature match
+  # -1 if no match
+  matches @5: List(Int16);
+}
+
+struct OrbFeatures @0xcd60164a8a0159ef {
+  timestampEof @0 :UInt64;
+  # transposed arrays of normalized image coordinates
+  # len(xs) == len(ys) == len(descriptors) * 32
+  xs @1 :List(Float32);
+  ys @2 :List(Float32);
+  descriptors @3 :Data;
+  octaves @4 :List(Int8);
+
+  # match index to last OrbFeatures
+  # -1 if no match
+  timestampLastEof @5 :UInt64;
+  matches @6: List(Int16);
+}
+
+struct OrbFeaturesSummary @0xd500d30c5803fa4f {
+  timestampEof @0 :UInt64;
+  timestampLastEof @1 :UInt64;
+
+  featureCount @2 :UInt16;
+  matchCount @3 :UInt16;
+  computeNs @4 :UInt64;
+}
+
+struct OrbKeyFrame @0xc8233c0345e27e24 {
+  # this is a globally unique id for the KeyFrame
+  id @0: UInt64;
+
+  # this is the location of the KeyFrame
+  pos @1: ECEFPoint;
+
+  # these are the features in the world
+  # len(dpos) == len(descriptors) * 32
+  dpos @2 :List(ECEFPoint);
+  descriptors @3 :Data;
+}
+
+struct KalmanOdometry @0x92e21bb7ea38793a {
+  trans @0 :List(Float32); # m/s in device frame
+  rot @1 :List(Float32); # rad/s in device frame
+  transStd @2 :List(Float32); # std m/s in device frame
+  rotStd @3 :List(Float32); # std rad/s in device frame
+}
+
+struct OrbObservation @0x9b326d4e436afec7 {
+  observationMonoTime @0 :UInt64;
+  normalizedCoordinates @1 :List(Float32);
+  locationECEF @2 :List(Float64);
+  matchDistance @3: UInt32;
+}
+
+struct CalibrationFeatures @0x8fdfadb254ea867a {
+  frameId @0 :UInt32;
+
+  p0 @1 :List(Float32);
+  p1 @2 :List(Float32);
+  status @3 :List(Int8);
+}
+
+struct NavStatus @0xbd8822120928120c {
+  isNavigating @0 :Bool;
+  currentAddress @1 :Address;
+
+  struct Address @0xce7cd672cacc7814 {
+    title @0 :Text;
+    lat @1 :Float64;
+    lng @2 :Float64;
+    house @3 :Text;
+    address @4 :Text;
+    street @5 :Text;
+    city @6 :Text;
+    state @7 :Text;
+    country @8 :Text;
+  }
+}
+
+struct NavUpdate @0xdb98be6565516acb {
+  isNavigating @0 :Bool;
+  curSegment @1 :Int32;
+  segments @2 :List(Segment);
+
+  struct LatLng @0x9eaef9187cadbb9b {
+    lat @0 :Float64;
+    lng @1 :Float64;
+  }
+
+  struct Segment @0xa5b39b4fc4d7da3f {
+    from @0 :LatLng;
+    to @1 :LatLng;
+    updateTime @2 :Int32;
+    distance @3 :Int32;
+    crossTime @4 :Int32;
+    exitNo @5 :Int32;
+    instruction @6 :Instruction;
+
+    parts @7 :List(LatLng);
+
+    enum Instruction @0xc5417a637451246f {
+      turnLeft @0;
+      turnRight @1;
+      keepLeft @2;
+      keepRight @3;
+      straight @4;
+      roundaboutExitNumber @5;
+      roundaboutExit @6;
+      roundaboutTurnLeft @7;
+      unkn8 @8;
+      roundaboutStraight @9;
+      unkn10 @10;
+      roundaboutTurnRight @11;
+      unkn12 @12;
+      roundaboutUturn @13;
+      unkn14 @14;
+      arrive @15;
+      exitLeft @16;
+      exitRight @17;
+      unkn18 @18;
+      uturn @19;
+      # ...
+    }
+  }
+}
+
+struct TrafficEvent @0xacfa74a094e62626 {
+  type @0 :Type;
+  distance @1 :Float32;
+  action @2 :Action;
+  resuming @3 :Bool;
+
+  enum Type @0xd85d75253435bf4b {
+    stopSign @0;
+    lightRed @1;
+    lightYellow @2;
+    lightGreen @3;
+    stopLight @4;
+  }
+
+  enum Action @0xa6f6ce72165ccb49 {
+    none @0;
+    yield @1;
+    stop @2;
+    resumeReady @3;
+  }
+
+}
+
+
+struct AndroidGnss @0xdfdf30d03fc485bd {
+  union {
+    measurements @0 :Measurements;
+    navigationMessage @1 :NavigationMessage;
+  }
+
+  struct Measurements @0xa20710d4f428d6cd {
+    clock @0 :Clock;
+    measurements @1 :List(Measurement);
+
+    struct Clock @0xa0e27b453a38f450 {
+      timeNanos @0 :Int64;
+      hardwareClockDiscontinuityCount @1 :Int32;
+
+      hasTimeUncertaintyNanos @2 :Bool;
+      timeUncertaintyNanos @3 :Float64;
+
+      hasLeapSecond @4 :Bool;
+      leapSecond @5 :Int32;
+
+      hasFullBiasNanos @6 :Bool;
+      fullBiasNanos @7 :Int64;
+
+      hasBiasNanos @8 :Bool;
+      biasNanos @9 :Float64;
+
+      hasBiasUncertaintyNanos @10 :Bool;
+      biasUncertaintyNanos @11 :Float64;
+
+      hasDriftNanosPerSecond @12 :Bool;
+      driftNanosPerSecond @13 :Float64;
+
+      hasDriftUncertaintyNanosPerSecond @14 :Bool;
+      driftUncertaintyNanosPerSecond @15 :Float64;
+    }
+
+    struct Measurement @0xd949bf717d77614d {
+      svId @0 :Int32;
+      constellation @1 :Constellation;
+
+      timeOffsetNanos @2 :Float64;
+      state @3 :Int32;
+      receivedSvTimeNanos @4 :Int64;
+      receivedSvTimeUncertaintyNanos @5 :Int64;
+      cn0DbHz @6 :Float64;
+      pseudorangeRateMetersPerSecond @7 :Float64;
+      pseudorangeRateUncertaintyMetersPerSecond @8 :Float64;
+      accumulatedDeltaRangeState @9 :Int32;
+      accumulatedDeltaRangeMeters @10 :Float64;
+      accumulatedDeltaRangeUncertaintyMeters @11 :Float64;
+
+      hasCarrierFrequencyHz @12 :Bool;
+      carrierFrequencyHz @13 :Float32;
+      hasCarrierCycles @14 :Bool;
+      carrierCycles @15 :Int64;
+      hasCarrierPhase @16 :Bool;
+      carrierPhase @17 :Float64;
+      hasCarrierPhaseUncertainty @18 :Bool;
+      carrierPhaseUncertainty @19 :Float64;
+      hasSnrInDb @20 :Bool;
+      snrInDb @21 :Float64;
+
+      multipathIndicator @22 :MultipathIndicator;
+
+      enum Constellation @0x9ef1f3ff0deb5ffb {
+        unknown @0;
+        gps @1;
+        sbas @2;
+        glonass @3;
+        qzss @4;
+        beidou @5;
+        galileo @6;
+      }
+
+      enum State @0xcbb9490adce12d72 {
+        unknown @0;
+        codeLock @1;
+        bitSync @2;
+        subframeSync @3;
+        towDecoded @4;
+        msecAmbiguous @5;
+        symbolSync @6;
+        gloStringSync @7;
+        gloTodDecoded @8;
+        bdsD2BitSync @9;
+        bdsD2SubframeSync @10;
+        galE1bcCodeLock @11;
+        galE1c2ndCodeLock @12;
+        galE1bPageSync @13;
+        sbasSync @14;
+      }
+
+      enum MultipathIndicator @0xc04e7b6231d4caa8 {
+        unknown @0;
+        detected @1;
+        notDetected @2;
+      }
+    }
+  }
+
+  struct NavigationMessage @0xe2517b083095fd4e {
+    type @0 :Int32;
+    svId @1 :Int32;
+    messageId @2 :Int32;
+    submessageId @3 :Int32;
+    data @4 :Data;
+    status @5 :Status;
+
+    enum Status @0xec1ff7996b35366f {
+      unknown @0;
+      parityPassed @1;
+      parityRebuilt @2;
+    }
+  }
+}
+
+struct LidarPts @0xe3d6685d4e9d8f7a {
+  r @0 :List(UInt16);        # uint16   m*500.0
+  theta @1 :List(UInt16);    # uint16 deg*100.0
+  reflect @2 :List(UInt8);   # uint8      0-255
+
+  # For storing out of file.
+  idx @3 :UInt64;
+
+  # For storing in file
+  pkt @4 :Data;
+}
+
+

cereal/messaging/__init__.py

@@ -0,0 +1,257 @@
+# must be built with scons
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
+import msgq
+
+import os
+import capnp
+import time
+
+from typing import Optional, List, Union, Dict
+
+from cereal import log
+from cereal.services import SERVICE_LIST
+from openpilot.common.util import MovingAverage
+
+NO_TRAVERSAL_LIMIT = 2**64-1
+
+
+def reset_context():
+  msgq.context = Context()
+
+
+def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
+  with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
+    return msg
+
+
+def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
+  args = {
+    'valid': False,
+    'logMonoTime': int(time.monotonic() * 1e9),
+    **kwargs
+  }
+  dat = log.Event.new_message(**args)
+  if service is not None:
+    if size is None:
+      dat.init(service)
+    else:
+      dat.init(service, size)
+  return dat
+
+
+def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
+  """Receive all message currently available on the queue"""
+  msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
+  return [log_from_bytes(m) for m in msgs]
+
+
+# TODO: print when we drop packets?
+def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+  """Same as drain sock, but only returns latest message. Consider using conflate instead."""
+  dat = None
+
+  while 1:
+    if wait and dat is None:
+      recv = sock.receive()
+    else:
+      recv = sock.receive(non_blocking=True)
+
+    if recv is None:  # Timeout hit
+      break
+
+    dat = recv
+
+  if dat is not None:
+    dat = log_from_bytes(dat)
+
+  return dat
+
+
+def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+  dat = sock.receive()
+  if dat is not None:
+    dat = log_from_bytes(dat)
+  return dat
+
+
+def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+  dat = sock.receive(non_blocking=True)
+  if dat is not None:
+    dat = log_from_bytes(dat)
+  return dat
+
+
+def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
+  """Keep receiving until we get a message"""
+  while True:
+    dat = sock.receive()
+    if dat is not None:
+      return log_from_bytes(dat)
+
+
+class FrequencyTracker:
+  def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
+    freq = max(min(service_freq, update_freq), 1.)
+    if is_poll:
+      min_freq = max_freq = freq
+    else:
+      max_freq = min(freq, update_freq)
+      if service_freq >= 2 * update_freq:
+        min_freq = update_freq
+      elif update_freq >= 2* service_freq:
+        min_freq = freq
+      else:
+        min_freq = min(freq, freq / 2.)
+
+    self.min_freq = min_freq * 0.8
+    self.max_freq = max_freq * 1.2
+    self.avg_dt = MovingAverage(int(10 * freq))
+    self.recent_avg_dt = MovingAverage(int(freq))
+    self.prev_time = 0.0
+
+  def record_recv_time(self, cur_time: float) -> None:
+    # TODO: Handle case where cur_time is less than prev_time
+    if self.prev_time > 1e-5:
+      dt = cur_time - self.prev_time
+
+      self.avg_dt.add_value(dt)
+      self.recent_avg_dt.add_value(dt)
+
+    self.prev_time = cur_time
+
+  @property
+  def valid(self) -> bool:
+    if self.avg_dt.count == 0:
+      return False
+
+    avg_freq = 1.0 / self.avg_dt.get_average()
+    if self.min_freq <= avg_freq <= self.max_freq:
+      return True
+
+    avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
+    return self.min_freq <= avg_freq_recent <= self.max_freq
+
+
+class SubMaster:
+  def __init__(self, services: List[str], poll: Optional[str] = None,
+               ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
+               ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
+    self.frame = -1
+    self.services = services
+    self.seen = {s: False for s in services}
+    self.updated = {s: False for s in services}
+    self.recv_time = {s: 0. for s in services}
+    self.recv_frame = {s: 0 for s in services}
+    self.sock = {}
+    self.data = {}
+    self.logMonoTime = {s: 0 for s in services}
+
+    # zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
+    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
+    self.static_freq_services = set(s for s in services if not on_demand[s])
+    self.alive = {s: on_demand[s] for s in services}
+    self.freq_ok = {s: on_demand[s] for s in services}
+    self.valid = {s: on_demand[s] for s in services}
+
+    self.freq_tracker: Dict[str, FrequencyTracker] = {}
+    self.poller = Poller()
+    polled_services = set([poll, ] if poll is not None else services)
+    self.non_polled_services = set(services) - polled_services
+
+    self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
+    self.ignore_alive = [] if ignore_alive is None else ignore_alive
+    self.ignore_valid = [] if ignore_valid is None else ignore_valid
+
+    self.simulation = bool(int(os.getenv("SIMULATION", "0")))
+
+    # if freq and poll aren't specified, assume the max to be conservative
+    assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used."
+    self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services])
+
+    for s in services:
+      p = self.poller if s not in self.non_polled_services else None
+      self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
+
+      try:
+        data = new_message(s)
+      except capnp.lib.capnp.KjException:
+        data = new_message(s, 0) # lists
+
+      self.data[s] = getattr(data.as_reader(), s)
+      self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
+
+  def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
+    return self.data[s]
+
+  def _check_avg_freq(self, s: str) -> bool:
+    return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive)
+
+  def update(self, timeout: int = 100) -> None:
+    msgs = []
+    for sock in self.poller.poll(timeout):
+      msgs.append(recv_one_or_none(sock))
+
+    # non-blocking receive for non-polled sockets
+    for s in self.non_polled_services:
+      msgs.append(recv_one_or_none(self.sock[s]))
+    self.update_msgs(time.monotonic(), msgs)
+
+  def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
+    self.frame += 1
+    self.updated = dict.fromkeys(self.services, False)
+    for msg in msgs:
+      if msg is None:
+        continue
+
+      s = msg.which()
+      self.seen[s] = True
+      self.updated[s] = True
+
+      self.freq_tracker[s].record_recv_time(cur_time)
+      self.recv_time[s] = cur_time
+      self.recv_frame[s] = self.frame
+      self.data[s] = getattr(msg, s)
+      self.logMonoTime[s] = msg.logMonoTime
+      self.valid[s] = msg.valid
+
+    for s in self.static_freq_services:
+      # alive if delay is within 10x the expected frequency; checks relaxed in simulator
+      self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
+      self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
+
+  def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
+    return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
+
+  def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
+    return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
+
+  def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
+    return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
+
+  def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
+    return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
+
+
+class PubMaster:
+  def __init__(self, services: List[str]):
+    self.sock = {}
+    for s in services:
+      self.sock[s] = pub_sock(s)
+
+  def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None:
+    if not isinstance(dat, bytes):
+      dat = dat.to_bytes()
+    self.sock[s].send(dat)
+
+  def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
+    for _ in range(int(timeout*(1./dt))):
+      if self.sock[s].all_readers_updated():
+        return True
+      time.sleep(dt)
+    return False
+
+  def all_readers_updated(self, s: str) -> bool:
+    return self.sock[s].all_readers_updated()  # type: ignore

cereal/messaging/bridge.cc

@@ -0,0 +1,71 @@
+#include <cassert>
+
+#include "cereal/messaging/msgq_to_zmq.h"
+#include "cereal/services.h"
+#include "common/util.h"
+
+ExitHandler do_exit;
+
+static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
+  std::vector<std::string> service_list;
+  for (const auto& it : services) {
+    std::string name = it.second.name;
+    bool in_whitelist = whitelist_str.find(name) != std::string::npos;
+    if (zmq_to_msgq && !in_whitelist) {
+      continue;
+    }
+    service_list.push_back(name);
+  }
+  return service_list;
+}
+
+void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &ip) {
+  MsgqToZmq bridge;
+  bridge.run(endpoints, ip);
+}
+
+void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
+  auto poller = std::make_unique<ZMQPoller>();
+  auto pub_context = std::make_unique<MSGQContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
+  std::map<SubSocket *, PubSocket *> sub2pub;
+
+  for (auto endpoint : endpoints) {
+    auto pub_sock = new MSGQPubSocket();
+    auto sub_sock = new ZMQSubSocket();
+    pub_sock->connect(pub_context.get(), endpoint);
+    sub_sock->connect(sub_context.get(), endpoint, ip, false);
+
+    poller->registerSocket(sub_sock);
+    sub2pub[sub_sock] = pub_sock;
+  }
+
+  while (!do_exit) {
+    for (auto sub_sock : poller->poll(100)) {
+      std::unique_ptr<Message> msg(sub_sock->receive(true));
+      if (msg) {
+        sub2pub[sub_sock]->sendMessage(msg.get());
+      }
+    }
+  }
+
+  // Clean up allocated sockets
+  for (auto &[sub_sock, pub_sock] : sub2pub) {
+    delete sub_sock;
+    delete pub_sock;
+  }
+}
+
+int main(int argc, char **argv) {
+  bool is_zmq_to_msgq = argc > 2;
+  std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1";
+  std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : "";
+  std::vector<std::string> endpoints = get_services(whitelist_str, is_zmq_to_msgq);
+
+  if (is_zmq_to_msgq) {
+    zmq_to_msgq(endpoints, ip);
+  } else {
+    msgq_to_zmq(endpoints, ip);
+  }
+  return 0;
+}

cereal/messaging/messaging.h

@@ -0,0 +1,102 @@
+#pragma once
+
+#include <cstddef>
+#include <map>
+#include <string>
+#include <vector>
+#include <utility>
+
+#include <capnp/serialize.h>
+
+#include "cereal/gen/cpp/log.capnp.h"
+#include "common/timing.h"
+#include "msgq/ipc.h"
+
+class SubMaster {
+public:
+  SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll = {},
+            const char *address = nullptr, const std::vector<const char *> &ignore_alive = {});
+  void update(int timeout = 1000);
+  void update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages);
+  inline bool allAlive(const std::vector<const char *> &service_list = {}) { return all_(service_list, false, true); }
+  inline bool allValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, false); }
+  inline bool allAliveAndValid(const std::vector<const char *> &service_list = {}) { return all_(service_list, true, true); }
+  void drain();
+  ~SubMaster();
+
+  uint64_t frame = 0;
+  bool updated(const char *name) const;
+  bool alive(const char *name) const;
+  bool valid(const char *name) const;
+  uint64_t rcv_frame(const char *name) const;
+  uint64_t rcv_time(const char *name) const;
+  cereal::Event::Reader &operator[](const char *name) const;
+
+private:
+  bool all_(const std::vector<const char *> &service_list, bool valid, bool alive);
+  Poller *poller_ = nullptr;
+  struct SubMessage;
+  std::map<SubSocket *, SubMessage *> messages_;
+  std::map<std::string, SubMessage *> services_;
+};
+
+class MessageBuilder : public capnp::MallocMessageBuilder {
+public:
+  MessageBuilder() = default;
+
+  cereal::Event::Builder initEvent(bool valid = true) {
+    cereal::Event::Builder event = initRoot<cereal::Event>();
+    event.setLogMonoTime(nanos_since_boot());
+    event.setValid(valid);
+    return event;
+  }
+
+  kj::ArrayPtr<capnp::byte> toBytes() {
+    heapArray_ = capnp::messageToFlatArray(*this);
+    return heapArray_.asBytes();
+  }
+
+  size_t getSerializedSize() {
+    return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word);
+  }
+
+  int serializeToBuffer(unsigned char *buffer, size_t buffer_size) {
+    size_t serialized_size = getSerializedSize();
+    if (serialized_size > buffer_size) { return -1; }
+    kj::ArrayOutputStream out(kj::ArrayPtr<capnp::byte>(buffer, buffer_size));
+    capnp::writeMessage(out, *this);
+    return serialized_size;
+  }
+
+private:
+  kj::Array<capnp::word> heapArray_;
+};
+
+class PubMaster {
+public:
+  PubMaster(const std::vector<const char *> &service_list);
+  inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
+  int send(const char *name, MessageBuilder &msg);
+  ~PubMaster();
+
+private:
+  std::map<std::string, PubSocket *> sockets_;
+};
+
+class AlignedBuffer {
+public:
+  kj::ArrayPtr<const capnp::word> align(const char *data, const size_t size) {
+    words_size = size / sizeof(capnp::word) + 1;
+    if (aligned_buf.size() < words_size) {
+      aligned_buf = kj::heapArray<capnp::word>(words_size < 512 ? 512 : words_size);
+    }
+    memcpy(aligned_buf.begin(), data, size);
+    return aligned_buf.slice(0, words_size);
+  }
+  inline kj::ArrayPtr<const capnp::word> align(Message *m) {
+    return align(m->getData(), m->getSize());
+  }
+private:
+  kj::Array<capnp::word> aligned_buf;
+  size_t words_size;
+};

cereal/messaging/msgq_to_zmq.cc

@@ -0,0 +1,144 @@
+#include "cereal/messaging/msgq_to_zmq.h"
+
+#include <cassert>
+
+#include "common/util.h"
+
+extern ExitHandler do_exit;
+
+// Max messages to process per socket per poll
+constexpr int MAX_MESSAGES_PER_SOCKET = 50;
+
+static std::string recv_zmq_msg(void *sock) {
+  zmq_msg_t msg;
+  zmq_msg_init(&msg);
+  std::string ret;
+  if (zmq_msg_recv(&msg, sock, 0) > 0) {
+    ret.assign((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
+  }
+  zmq_msg_close(&msg);
+  return ret;
+}
+
+void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
+  zmq_context = std::make_unique<ZMQContext>();
+  msgq_context = std::make_unique<MSGQContext>();
+
+  // Create ZMQPubSockets for each endpoint
+  for (const auto &endpoint : endpoints) {
+    auto &socket_pair = socket_pairs.emplace_back();
+    socket_pair.endpoint = endpoint;
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
+    int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
+    if (ret != 0) {
+      printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
+      return;
+    }
+  }
+
+  // Start ZMQ monitoring thread to monitor socket events
+  std::thread thread(&MsgqToZmq::zmqMonitorThread, this);
+
+  // Main loop for processing messages
+  while (!do_exit) {
+    {
+      std::unique_lock lk(mutex);
+      cv.wait(lk, [this]() { return do_exit || !sub2pub.empty(); });
+      if (do_exit) break;
+
+      for (auto sub_sock : msgq_poller->poll(100)) {
+        // Process messages for each socket
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
+        for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
+          auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
+          if (!msg) break;
+
+          while (pub_sock->sendMessage(msg.get()) == -1) {
+            if (errno != EINTR) break;
+          }
+        }
+      }
+    }
+    util::sleep_for(1);  // Give zmqMonitorThread a chance to acquire the mutex
+  }
+
+  thread.join();
+}
+
+void MsgqToZmq::zmqMonitorThread() {
+  std::vector<zmq_pollitem_t> pollitems;
+
+  // Set up ZMQ monitor for each pub socket
+  for (int i = 0; i < socket_pairs.size(); ++i) {
+    std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+
+    void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
+    zmq_connect(monitor_socket, addr.c_str());
+    pollitems.emplace_back(zmq_pollitem_t{.socket = monitor_socket, .events = ZMQ_POLLIN});
+  }
+
+  while (!do_exit) {
+    int ret = zmq_poll(pollitems.data(), pollitems.size(), 1000);
+    if (ret < 0) {
+      if (errno == EINTR) {
+        // Due to frequent EINTR signals from msgq, introduce a brief delay (200 ms)
+        // to reduce CPU usage during retry attempts.
+        util::sleep_for(200);
+      }
+      continue;
+    }
+
+    for (int i = 0; i < pollitems.size(); ++i) {
+      if (pollitems[i].revents & ZMQ_POLLIN) {
+        // First frame in message contains event number and value
+        std::string frame = recv_zmq_msg(pollitems[i].socket);
+        if (frame.empty()) continue;
+
+        uint16_t event_type = *(uint16_t *)(frame.data());
+
+        // Second frame in message contains event address
+        frame = recv_zmq_msg(pollitems[i].socket);
+        if (frame.empty()) continue;
+
+        std::unique_lock lk(mutex);
+        auto &pair = socket_pairs[i];
+        if (event_type & ZMQ_EVENT_ACCEPTED) {
+          printf("socket [%s] connected\n", pair.endpoint.c_str());
+          if (++pair.connected_clients == 1) {
+            // Create new MSGQ subscriber socket and map to ZMQ publisher
+            pair.sub_sock = std::make_unique<MSGQSubSocket>();
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
+            sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
+            registerSockets();
+          }
+        } else if (event_type & ZMQ_EVENT_DISCONNECTED) {
+          printf("socket [%s] disconnected\n", pair.endpoint.c_str());
+          if (pair.connected_clients == 0 || --pair.connected_clients == 0) {
+            // Remove MSGQ subscriber socket from mapping and reset it
+            sub2pub.erase(pair.sub_sock.get());
+            pair.sub_sock.reset(nullptr);
+            registerSockets();
+          }
+        }
+        cv.notify_one();
+      }
+    }
+  }
+
+  // Clean up monitor sockets
+  for (int i = 0; i < pollitems.size(); ++i) {
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
+    zmq_close(pollitems[i].socket);
+  }
+  cv.notify_one();
+}
+
+void MsgqToZmq::registerSockets() {
+  msgq_poller = std::make_unique<MSGQPoller>();
+  for (const auto &socket_pair : socket_pairs) {
+    if (socket_pair.sub_sock) {
+      msgq_poller->registerSocket(socket_pair.sub_sock.get());
+    }
+  }
+}

cereal/messaging/msgq_to_zmq.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <condition_variable>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#define private public
+#include "msgq/impl_msgq.h"
+#include "msgq/impl_zmq.h"
+
+class MsgqToZmq {
+public:
+  MsgqToZmq() {}
+  void run(const std::vector<std::string> &endpoints, const std::string &ip);
+
+protected:
+  void registerSockets();
+  void zmqMonitorThread();
+
+  struct SocketPair {
+    std::string endpoint;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
+    std::unique_ptr<MSGQSubSocket> sub_sock;
+    int connected_clients = 0;
+  };
+
+  std::unique_ptr<MSGQContext> msgq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
+  std::mutex mutex;
+  std::condition_variable cv;
+  std::unique_ptr<MSGQPoller> msgq_poller;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
+  std::vector<SocketPair> socket_pairs;
+};

cereal/messaging/socketmaster.cc

@@ -0,0 +1,203 @@
+#include <assert.h>
+#include <stdlib.h>
+#include <string>
+#include <mutex>
+
+#include "cereal/services.h"
+#include "cereal/messaging/messaging.h"
+
+const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
+
+static inline bool inList(const std::vector<const char *> &list, const char *value) {
+  for (auto &v : list) {
+    if (strcmp(value, v) == 0) return true;
+  }
+  return false;
+}
+
+class MessageContext {
+public:
+  MessageContext() : ctx_(nullptr) {}
+  ~MessageContext() { delete ctx_; }
+  inline Context *context() {
+    std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
+    return ctx_;
+  }
+private:
+  Context *ctx_;
+  std::once_flag init_flag;
+};
+
+MessageContext message_context;
+
+struct SubMaster::SubMessage {
+  std::string name;
+  SubSocket *socket = nullptr;
+  float freq = 0.0f;
+  bool updated = false, alive = false, valid = false, ignore_alive;
+  uint64_t rcv_time = 0, rcv_frame = 0;
+  void *allocated_msg_reader = nullptr;
+  bool is_polled = false;
+  capnp::FlatArrayMessageReader *msg_reader = nullptr;
+  AlignedBuffer aligned_buf;
+  cereal::Event::Reader event;
+};
+
+SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
+                     const char *address, const std::vector<const char *> &ignore_alive) {
+  poller_ = Poller::create();
+  for (auto name : service_list) {
+    assert(services.count(std::string(name)) > 0);
+
+    service serv = services.at(std::string(name));
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
+    assert(socket != 0);
+    bool is_polled = inList(poll, name) || poll.empty();
+    if (is_polled) poller_->registerSocket(socket);
+    SubMessage *m = new SubMessage{
+      .name = name,
+      .socket = socket,
+      .freq = serv.frequency,
+      .ignore_alive = inList(ignore_alive, name),
+      .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
+      .is_polled = is_polled};
+    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
+    messages_[socket] = m;
+    services_[name] = m;
+  }
+}
+
+void SubMaster::update(int timeout) {
+  for (auto &kv : messages_) kv.second->updated = false;
+
+  auto sockets = poller_->poll(timeout);
+
+  // add non-polled sockets for non-blocking receive
+  for (auto &kv : messages_) {
+    SubMessage *m = kv.second;
+    SubSocket *s = kv.first;
+    if (!m->is_polled) sockets.push_back(s);
+  }
+
+  uint64_t current_time = nanos_since_boot();
+
+  std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
+
+  for (auto s : sockets) {
+    Message *msg = s->receive(true);
+    if (msg == nullptr) continue;
+
+    SubMessage *m = messages_.at(s);
+
+    m->msg_reader->~FlatArrayMessageReader();
+    capnp::ReaderOptions options;
+    options.traversalLimitInWords = kj::maxValue; // Don't limit
+    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options);
+    delete msg;
+    messages.push_back({m->name, m->msg_reader->getRoot<cereal::Event>()});
+  }
+
+  update_msgs(current_time, messages);
+}
+
+void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
+  if (++frame == UINT64_MAX) frame = 1;
+
+  for (auto &kv : messages) {
+    auto m_find = services_.find(kv.first);
+    if (m_find == services_.end()){
+      continue;
+    }
+    SubMessage *m = m_find->second;
+    m->event = kv.second;
+    m->updated = true;
+    m->rcv_time = current_time;
+    m->rcv_frame = frame;
+    m->valid = m->event.getValid();
+    if (SIMULATION) m->alive = true;
+  }
+
+  if (!SIMULATION) {
+    for (auto &kv : messages_) {
+      SubMessage *m = kv.second;
+      m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
+    }
+  }
+}
+
+bool SubMaster::all_(const std::vector<const char *> &service_list, bool valid, bool alive) {
+  int found = 0;
+  for (auto &kv : messages_) {
+    SubMessage *m = kv.second;
+    if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
+      found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive));
+    }
+  }
+  return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
+}
+
+void SubMaster::drain() {
+  while (true) {
+    auto polls = poller_->poll(0);
+    if (polls.size() == 0)
+      break;
+
+    for (auto sock : polls) {
+      Message *msg = sock->receive(true);
+      delete msg;
+    }
+  }
+}
+
+bool SubMaster::updated(const char *name) const {
+  return services_.at(name)->updated;
+}
+
+bool SubMaster::alive(const char *name) const {
+  return services_.at(name)->alive;
+}
+
+bool SubMaster::valid(const char *name) const {
+  return services_.at(name)->valid;
+}
+
+uint64_t SubMaster::rcv_frame(const char *name) const {
+  return services_.at(name)->rcv_frame;
+}
+
+uint64_t SubMaster::rcv_time(const char *name) const {
+  return services_.at(name)->rcv_time;
+}
+
+cereal::Event::Reader &SubMaster::operator[](const char *name) const {
+  return services_.at(name)->event;
+}
+
+SubMaster::~SubMaster() {
+  delete poller_;
+  for (auto &kv : messages_) {
+    SubMessage *m = kv.second;
+    m->msg_reader->~FlatArrayMessageReader();
+    free(m->allocated_msg_reader);
+    delete m->socket;
+    delete m;
+  }
+}
+
+PubMaster::PubMaster(const std::vector<const char *> &service_list) {
+  for (auto name : service_list) {
+    assert(services.count(name) > 0);
+    PubSocket *socket = PubSocket::create(message_context.context(), name);
+    assert(socket);
+    sockets_[name] = socket;
+  }
+}
+
+int PubMaster::send(const char *name, MessageBuilder &msg) {
+  auto bytes = msg.toBytes();
+  return send(name, bytes.begin(), bytes.size());
+}
+
+PubMaster::~PubMaster() {
+  for (auto s : sockets_) delete s.second;
+}

cereal/messaging/tests/test_messaging.py

@@ -0,0 +1,185 @@
+import os
+import capnp
+import multiprocessing
+import numbers
+import random
+import threading
+import time
+from parameterized import parameterized
+import pytest
+
+from cereal import log, car
+import cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
+
+events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
+
+def random_sock():
+  return random.choice(events)
+
+def random_socks(num_socks=10):
+  return list({random_sock() for _ in range(num_socks)})
+
+def random_bytes(length=1000):
+  return bytes([random.randrange(0xFF) for _ in range(length)])
+
+def zmq_sleep(t=1):
+  if "ZMQ" in os.environ:
+    time.sleep(t)
+
+
+# TODO: this should take any capnp struct and returrn a msg with random populated data
+def random_carstate():
+  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
+  msg = messaging.new_message("carState")
+  cs = msg.carState
+  for f in fields:
+    setattr(cs, f, random.random() * 10)
+  return msg
+
+# TODO: this should compare any capnp structs
+def assert_carstate(cs1, cs2):
+  for f in car.CarState.schema.non_union_fields:
+    # TODO: check all types
+    val1, val2 = getattr(cs1, f), getattr(cs2, f)
+    if isinstance(val1, numbers.Number):
+      assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
+
+def delayed_send(delay, sock, dat):
+  def send_func():
+    sock.send(dat)
+  threading.Timer(delay, send_func).start()
+
+
+class TestMessaging:
+  def setUp(self):
+    # TODO: ZMQ tests are too slow; all sleeps will need to be
+    # replaced with logic to block on the necessary condition
+    if "ZMQ" in os.environ:
+      pytest.skip()
+
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep()
+
+  @parameterized.expand(events)
+  def test_new_message(self, evt):
+    try:
+      msg = messaging.new_message(evt)
+    except capnp.lib.capnp.KjException:
+      msg = messaging.new_message(evt, random.randrange(200))
+    assert (time.monotonic() - msg.logMonoTime) < 0.1
+    assert not msg.valid
+    assert evt == msg.which()
+
+  @parameterized.expand(events)
+  def test_pub_sock(self, evt):
+    messaging.pub_sock(evt)
+
+  @parameterized.expand(events)
+  def test_sub_sock(self, evt):
+    messaging.sub_sock(evt)
+
+  @parameterized.expand([
+    (messaging.drain_sock, capnp._DynamicStructReader),
+    (messaging.drain_sock_raw, bytes),
+  ])
+  def test_drain_sock(self, func, expected_type):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
+
+    # no wait and no msgs in queue
+    msgs = func(sub_sock)
+    assert isinstance(msgs, list)
+    assert len(msgs) == 0
+
+    # no wait but msgs are queued up
+    num_msgs = random.randrange(3, 10)
+    for _ in range(num_msgs):
+      pub_sock.send(messaging.new_message(sock).to_bytes())
+    time.sleep(0.1)
+    msgs = func(sub_sock)
+    assert isinstance(msgs, list)
+    assert all(isinstance(msg, expected_type) for msg in msgs)
+    assert len(msgs) == num_msgs
+
+  def test_recv_sock(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=100)
+    zmq_sleep()
+
+    # no wait and no msg in queue, socket should timeout
+    recvd = messaging.recv_sock(sub_sock)
+    assert recvd is None
+
+    # no wait and one msg in queue
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    time.sleep(0.01)
+    recvd = messaging.recv_sock(sub_sock)
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    # https://github.com/python/mypy/issues/13038
+    assert_carstate(msg.carState, recvd.carState)
+
+  def test_recv_one(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=1000)
+    zmq_sleep()
+
+    # no msg in queue, socket should timeout
+    recvd = messaging.recv_one(sub_sock)
+    assert recvd is None
+
+    # one msg in queue
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    recvd = messaging.recv_one(sub_sock)
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    assert_carstate(msg.carState, recvd.carState)
+
+  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
+  def test_recv_one_or_none(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock)
+    zmq_sleep()
+
+    # no msg in queue, socket shouldn't block
+    recvd = messaging.recv_one_or_none(sub_sock)
+    assert recvd is None
+
+    # one msg in queue
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    recvd = messaging.recv_one_or_none(sub_sock)
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    assert_carstate(msg.carState, recvd.carState)
+
+  def test_recv_one_retry(self):
+    sock = "carState"
+    sock_timeout = 0.1
+    pub_sock = messaging.pub_sock(sock)
+    sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
+    zmq_sleep()
+
+    # this test doesn't work with ZMQ since multiprocessing interrupts it
+    if "ZMQ" not in os.environ:
+      # wait 5 socket timeouts and make sure it's still retrying
+      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
+      p.start()
+      time.sleep(sock_timeout*5)
+      assert p.is_alive()
+      p.terminate()
+
+    # wait 5 socket timeouts before sending
+    msg = random_carstate()
+    start_time = time.monotonic()
+    delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
+    recvd = messaging.recv_one_retry(sub_sock)
+    assert (time.monotonic() - start_time) >= sock_timeout*5
+    assert isinstance(recvd, capnp._DynamicStructReader)
+    assert_carstate(msg.carState, recvd.carState)

cereal/messaging/tests/test_pub_sub_master.py

@@ -0,0 +1,160 @@
+import random
+import time
+from typing import Sized, cast
+
+import cereal.messaging as messaging
+from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
+                                                  random_bytes, random_carstate, assert_carstate, \
+                                                  zmq_sleep
+from cereal.services import SERVICE_LIST
+
+
+class TestSubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
+
+  def test_init(self):
+    sm = messaging.SubMaster(events)
+    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
+              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
+      assert len(cast(Sized, p)) == len(events)
+
+  def test_init_state(self):
+    socks = random_socks()
+    sm = messaging.SubMaster(socks)
+    assert sm.frame == -1
+    assert not any(sm.updated.values())
+    assert not any(sm.seen.values())
+    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
+    assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
+    assert all(t == 0. for t in sm.recv_time.values())
+    assert all(f == 0 for f in sm.recv_frame.values())
+    assert all(t == 0 for t in sm.logMonoTime.values())
+
+    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
+              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
+      assert len(cast(Sized, p)) == len(socks)
+
+  def test_getitem(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sm = messaging.SubMaster([sock,])
+    zmq_sleep()
+
+    msg = random_carstate()
+    pub_sock.send(msg.to_bytes())
+    sm.update(1000)
+    assert_carstate(msg.carState, sm[sock])
+
+  # TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
+  def test_update(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sm = messaging.SubMaster([sock,])
+    zmq_sleep()
+
+    for i in range(10):
+      msg = messaging.new_message(sock)
+      pub_sock.send(msg.to_bytes())
+      sm.update(1000)
+      assert sm.frame == i
+      assert all(sm.updated.values())
+
+  def test_update_timeout(self):
+    sock = random_sock()
+    sm = messaging.SubMaster([sock,])
+    timeout = random.randrange(1000, 3000)
+    start_time = time.monotonic()
+    sm.update(timeout)
+    t = time.monotonic() - start_time
+    assert t >= timeout/1000.
+    assert t < 3
+    assert not any(sm.updated.values())
+
+  def test_avg_frequency_checks(self):
+    for poll in (True, False):
+      sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
+                               poll=("modelV2" if poll else None),
+                               frequency=(20. if not poll else None))
+
+      checks = {
+        "carState": (20, 20),
+        "modelV2": (20, 20 if poll else 10),
+        "cameraOdometry": (20, 10),
+        "liveCalibration": (4, 4),
+        "carParams": (None, None),
+        "userBookmark": (None, None),
+      }
+
+      for service, (max_freq, min_freq) in checks.items():
+        if max_freq is not None:
+          assert sm._check_avg_freq(service)
+          assert sm.freq_tracker[service].max_freq == max_freq*1.2
+          assert sm.freq_tracker[service].min_freq == min_freq*0.8
+        else:
+          assert not sm._check_avg_freq(service)
+
+  def test_alive(self):
+    pass
+
+  def test_ignore_alive(self):
+    pass
+
+  def test_valid(self):
+    pass
+
+  # SubMaster should always conflate
+  def test_conflate(self):
+    sock = "carState"
+    pub_sock = messaging.pub_sock(sock)
+    sm = messaging.SubMaster([sock,])
+
+    n = 10
+    for i in range(n+1):
+      msg = messaging.new_message(sock)
+      msg.carState.vEgo = i
+      pub_sock.send(msg.to_bytes())
+      time.sleep(0.01)
+    sm.update(1000)
+    assert sm[sock].vEgo == n
+
+
+class TestPubMaster:
+
+  def setup_method(self):
+    # ZMQ pub socket takes too long to die
+    # sleep to prevent multiple publishers error between tests
+    zmq_sleep(3)
+
+  def test_init(self):
+    messaging.PubMaster(events)
+
+  def test_send(self):
+    socks = random_socks()
+    pm = messaging.PubMaster(socks)
+    sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
+    zmq_sleep()
+
+    # PubMaster accepts either a capnp msg builder or bytes
+    for capnp in [True, False]:
+      for i in range(100):
+        sock = socks[i % len(socks)]
+
+        if capnp:
+          try:
+            msg = messaging.new_message(sock)
+          except Exception:
+            msg = messaging.new_message(sock, random.randrange(50))
+        else:
+          msg = random_bytes()
+
+        pm.send(sock, msg)
+        recvd = sub_socks[sock].receive()
+
+        if capnp:
+          msg.clear_write_flag()
+          msg = msg.to_bytes()
+        assert msg == recvd, i

cereal/messaging/tests/test_services.py

@@ -0,0 +1,21 @@
+import os
+import tempfile
+from typing import Dict
+from parameterized import parameterized
+
+import cereal.services as services
+from cereal.services import SERVICE_LIST
+
+
+class TestServices:
+
+  @parameterized.expand(SERVICE_LIST.keys())
+  def test_services(self, s):
+    service = SERVICE_LIST[s]
+    assert service.frequency <= 104
+    assert service.decimation != 0
+
+  def test_generated_header(self):
+    with tempfile.NamedTemporaryFile(suffix=".h") as f:
+      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
+      assert ret == 0, "generated services header is not valid C"

cereal/services.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+from typing import Optional
+
+
+class Service:
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
+    self.should_log = should_log
+    self.frequency = frequency
+    self.decimation = decimation
+
+
+_services: dict[str, tuple] = {
+  # service: (should_log, frequency, qlog decimation (optional))
+  # note: the "EncodeIdx" packets will still be in the log
+  "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
+  "accelerometer": (True, 104., 104),
+  "accelerometer2": (True, 100., 100),
+  "magnetometer": (True, 25.),
+  "lightSensor": (True, 100., 100),
+  "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
+  "gpsNMEA": (True, 9.),
+  "deviceState": (True, 2., 1),
+  "touch": (True, 20., 1),
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10),
+  "selfdriveState": (True, 100., 10),
+  "pandaStates": (True, 10., 1),
+  "peripheralState": (True, 2., 1),
+  "radarState": (True, 20., 5),
+  "roadEncodeIdx": (False, 20., 1),
+  "liveTracks": (True, 20.),
+  "sendcan": (True, 100., 139),
+  "logMessage": (True, 0.),
+  "errorLogMessage": (True, 0., 1),
+  "liveCalibration": (True, 4., 4),
+  "liveTorqueParameters": (True, 4., 1),
+  "liveDelay": (True, 4., 1),
+  "androidLog": (True, 0.),
+  "carState": (True, 100., 10),
+  "carControl": (True, 100., 10),
+  "carOutput": (True, 100., 10),
+  "longitudinalPlan": (True, 20., 10),
+  "driverAssistance": (True, 20., 20),
+  "procLog": (True, 0.5, 15),
+  "gpsLocationExternal": (True, 10., 10),
+  "gpsLocation": (True, 1., 1),
+  "ubloxGnss": (True, 10.),
+  "qcomGnss": (True, 2.),
+  "gnssMeasurements": (True, 10., 10),
+  "clocks": (True, 0.1, 1),
+  "ubloxRaw": (True, 20.),
+  "livePose": (True, 20., 4),
+  "liveParameters": (True, 20., 5),
+  "cameraOdometry": (True, 20., 10),
+  "thumbnail": (True, 1 / 60., 1),
+  "onroadEvents": (True, 1., 1),
+  "carParams": (True, 0.02, 1),
+  "roadCameraState": (True, 20., 20),
+  "driverCameraState": (True, 20., 20),
+  "driverEncodeIdx": (False, 20., 1),
+  "driverStateV2": (True, 20., 10),
+  "driverMonitoringState": (True, 20., 10),
+  "wideRoadEncodeIdx": (False, 20., 1),
+  "wideRoadCameraState": (True, 20., 20),
+  "drivingModelData": (True, 20., 10),
+  "modelV2": (True, 20.),
+  "managerState": (True, 2., 1),
+  "uploaderState": (True, 0., 1),
+  "navInstruction": (True, 1., 10),
+  "navRoute": (True, 0.),
+  "navThumbnail": (True, 0.),
+  "qRoadEncodeIdx": (False, 20.),
+  "userBookmark": (True, 0., 1),
+  "soundPressure": (True, 10., 10),
+  "rawAudioData": (False, 20.),
+  "bookmarkButton": (True, 0., 1),
+  "audioFeedback": (True, 0., 1),
+
+  # debug
+  "uiDebug": (True, 0., 1),
+  "testJoystick": (True, 0.),
+  "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
+  "livestreamWideRoadEncodeIdx": (False, 20.),
+  "livestreamRoadEncodeIdx": (False, 20.),
+  "livestreamDriverEncodeIdx": (False, 20.),
+  "livestreamWideRoadEncodeData": (False, 20.),
+  "livestreamRoadEncodeData": (False, 20.),
+  "livestreamDriverEncodeData": (False, 20.),
+  "customReservedRawData0": (True, 0.),
+  "customReservedRawData1": (True, 0.),
+  "customReservedRawData2": (True, 0.),
+  "dpControlsState": (False, 100., 10),
+  "modelExt": (True, 20.),
+}
+SERVICE_LIST = {name: Service(*vals) for
+                idx, (name, vals) in enumerate(_services.items())}
+
+
+def build_header():
+  h = ""
+  h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
+  h += "#ifndef __SERVICES_H\n"
+  h += "#define __SERVICES_H\n"
+
+  h += "#include <map>\n"
+  h += "#include <string>\n"
+
+  h += "struct service { std::string name; bool should_log; float frequency; int decimation; };\n"
+  h += "static std::map<std::string, service> services = {\n"
+  for k, v in SERVICE_LIST.items():
+    should_log = "true" if v.should_log else "false"
+    decimation = -1 if v.decimation is None else v.decimation
+    h += '  { "%s", {"%s", %s, %f, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation)
+  h += "};\n"
+
+  h += "#endif\n"
+  return h
+
+
+if __name__ == "__main__":
+  print(build_header())

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -0,0 +1,29 @@
+Import('env', 'envCython', 'arch')
+
+common_libs = [
+  'params.cc',
+  'swaglog.cc',
+  'util.cc',
+  'ratekeeper.cc',
+  'clutil.cc',
+]
+
+_common = env.Library('common', common_libs, LIBS="json11")
+Export('_common')
+
+if GetOption('extras'):
+  env.Program('tests/test_common',
+              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+
+# Cython bindings
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
+
+Export('common_python')

common/api.py

@@ -0,0 +1,48 @@
+import jwt
+import os
+import requests
+from datetime import datetime, timedelta, UTC
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.version import get_version
+
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
+
+class Api:
+  def __init__(self, dongle_id):
+    self.dongle_id = dongle_id
+    with open(Paths.persist_root()+'/comma/id_rsa') as f:
+      self.private_key = f.read()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    now = datetime.now(UTC).replace(tzinfo=None)
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours)
+    }
+    if payload_extra is not None:
+      payload.update(payload_extra)
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+
+
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+  headers = {}
+  if access_token is not None:
+    headers['Authorization'] = "JWT " + access_token
+
+  headers['User-Agent'] = "openpilot-" + get_version()
+
+  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)

common/api/__init__.py

@@ -1,15 +0,0 @@
-import requests
-
-from selfdrive.version import version
-
-def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
-  backend = "https://api.commadotai.com/"
-
-  headers = {}
-  if access_token is not None:
-    headers['Authorization'] = "JWT "+access_token
-
-  headers['User-Agent'] = "openpilot-" + version
-
-  return requests.request(method, backend+endpoint, timeout=timeout, headers = headers, params=params)
-

common/basedir.py

@@ -1,4 +1,4 @@
 import os
+
+
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
-
-

common/clutil.cc

@@ -0,0 +1,98 @@
+#include "common/clutil.h"
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+
+#include "common/util.h"
+#include "common/swaglog.h"
+
+namespace {  // helper functions
+
+template <typename Func, typename Id, typename Name>
+std::string get_info(Func get_info_func, Id id, Name param_name) {
+  size_t size = 0;
+  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
+  std::string info(size, '\0');
+  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
+  return info;
+}
+inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
+inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
+
+void cl_print_info(cl_platform_id platform, cl_device_id device) {
+  size_t work_group_size = 0;
+  cl_device_type device_type = 0;
+  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
+  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
+  const char *type_str = "Other...";
+  switch (device_type) {
+    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
+    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
+    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
+  }
+
+  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
+  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
+  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
+  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
+  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
+  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
+  LOGD("max work group size: %zu", work_group_size);
+  LOGD("type = %d, %s", (int)device_type, type_str);
+}
+
+void cl_print_build_errors(cl_program program, cl_device_id device) {
+  cl_build_status status;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
+  size_t log_size;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
+  std::string log(log_size, '\0');
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
+
+  LOGE("build failed; status=%d, log: %s", status, log.c_str());
+}
+
+}  // namespace
+
+cl_device_id cl_get_device_id(cl_device_type device_type) {
+  cl_uint num_platforms = 0;
+  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
+  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
+  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
+
+  for (size_t i = 0; i < num_platforms; ++i) {
+    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
+
+    // Get first device
+    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
+      cl_print_info(platform_ids[i], device_id);
+      return device_id;
+    }
+  }
+  LOGE("No valid openCL platform found");
+  assert(0);
+  return nullptr;
+}
+
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
+void cl_release_context(cl_context context) {
+  clReleaseContext(context);
+}
+
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
+  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
+}
+
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
+  const char *csrc = src.c_str();
+  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
+  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
+    cl_print_build_errors(prg, device_id);
+    assert(0);
+  }
+  return prg;
+}

common/clutil.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#include <string>
+
+#define CL_CHECK(_expr)          \
+  do {                           \
+    assert(CL_SUCCESS == (_expr)); \
+  } while (0)
+
+#define CL_CHECK_ERR(_expr)           \
+  ({                                  \
+    cl_int err = CL_INVALID_VALUE;    \
+    __typeof__(_expr) _ret = _expr;   \
+    assert(_ret&& err == CL_SUCCESS); \
+    _ret;                             \
+  })
+
+cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
+void cl_release_context(cl_context context);
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/constants.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+# conversions
+class CV:
+  # Speed
+  MPH_TO_KPH = 1.609344
+  KPH_TO_MPH = 1. / MPH_TO_KPH
+  MS_TO_KPH = 3.6
+  KPH_TO_MS = 1. / MS_TO_KPH
+  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
+  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
+  MS_TO_KNOTS = 1.9438
+  KNOTS_TO_MS = 1. / MS_TO_KNOTS
+
+  # Angle
+  DEG_TO_RAD = np.pi / 180.
+  RAD_TO_DEG = 1. / DEG_TO_RAD
+
+  # Mass
+  LB_TO_KG = 0.453592
+
+
+ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

common/dbc.py

@@ -1,206 +0,0 @@
-import re
-import os
-import struct
-import bitstring
-from collections import namedtuple
-
-def int_or_float(s):
-  # return number, trying to maintain int format
-  try:
-    return int(s)
-  except ValueError:
-    return float(s)
-
-DBCSignal = namedtuple(
-  "DBCSignal", ["name", "start_bit", "size", "is_little_endian", "is_signed",
-                "factor", "offset", "tmin", "tmax", "units"])
-
-class dbc(object):
-  def __init__(self, fn):
-    self.name, _ = os.path.splitext(os.path.basename(fn))
-    with open(fn) as f:
-      self.txt = f.read().split("\n")
-    self._warned_addresses = set()
-
-    # regexps from https://github.com/ebroecker/canmatrix/blob/master/canmatrix/importdbc.py
-    bo_regexp = re.compile("^BO\_ (\w+) (\w+) *: (\w+) (\w+)")
-    sg_regexp = re.compile("^SG\_ (\w+) : (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] \"(.*)\" (.*)")
-    sgm_regexp = re.compile("^SG\_ (\w+) (\w+) *: (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] \"(.*)\" (.*)")
-
-    # A dictionary which maps message ids to tuples ((name, size), signals).
-    #   name is the ASCII name of the message.
-    #   size is the size of the message in bytes.
-    #   signals is a list signals contained in the message.
-    # signals is a list of DBCSignal in order of increasing start_bit.
-    self.msgs = {}
-
-    # lookup to bit reverse each byte
-    self.bits_index = [(i & ~0b111) + ((-i-1) & 0b111) for i in xrange(64)]
-
-    for l in self.txt:
-      l = l.strip()
-
-      if l.startswith("BO_ "):
-        # new group
-        dat = bo_regexp.match(l)
-
-        if dat is None:
-          print "bad BO", l
-        name = dat.group(2)
-        size = int(dat.group(3))
-        ids = int(dat.group(1), 0) # could be hex
-
-        self.msgs[ids] = ((name, size), [])
-
-      if l.startswith("SG_ "):
-        # new signal
-        dat = sg_regexp.match(l)
-        go = 0
-        if dat is None:
-          dat = sgm_regexp.match(l)
-          go = 1
-        if dat is None:
-          print "bad SG", l
-
-        sgname = dat.group(1)
-        start_bit = int(dat.group(go+2))
-        signal_size = int(dat.group(go+3))
-        is_little_endian = int(dat.group(go+4))==1
-        is_signed = dat.group(go+5)=='-'
-        factor = int_or_float(dat.group(go+6))
-        offset = int_or_float(dat.group(go+7))
-        tmin = int_or_float(dat.group(go+8))
-        tmax = int_or_float(dat.group(go+9))
-        units = dat.group(go+10)
-
-        self.msgs[ids][1].append(
-          DBCSignal(sgname, start_bit, signal_size, is_little_endian,
-                    is_signed, factor, offset, tmin, tmax, units))
-
-    for msg in self.msgs.viewvalues():
-      msg[1].sort(key=lambda x: x.start_bit)
-
-  def encode(self, msg_id, dd):
-    """Encode a CAN message using the dbc.
-
-       Inputs:
-        msg_id: The message ID.
-        dd: A dictionary mapping signal name to signal data.
-    """
-    # TODO: Stop using bitstring, which is super slow.
-    msg_def = self.msgs[msg_id]
-    size = msg_def[0][1]
-
-    bsf = bitstring.Bits(hex="00"*size)
-    for s in msg_def[1]:
-      ival = dd.get(s.name)
-      if ival is not None:
-        ival = (ival / s.factor) - s.offset
-        ival = int(round(ival))
-
-        # should pack this
-        if s.is_little_endian:
-          ss = s.start_bit
-        else:
-          ss = self.bits_index[s.start_bit]
-
-
-        if s.is_signed:
-          tbs = bitstring.Bits(int=ival, length=s.size)
-        else:
-          tbs = bitstring.Bits(uint=ival, length=s.size)
-
-        lpad = bitstring.Bits(bin="0b"+"0"*ss)
-        rpad = bitstring.Bits(bin="0b"+"0"*(8*size-(ss+s.size)))
-        tbs = lpad+tbs+rpad
-
-        bsf |= tbs
-    return bsf.tobytes()
-
-  def decode(self, x, arr=None, debug=False):
-    """Decode a CAN message using the dbc.
-
-       Inputs:
-        x: A collection with elements (address, time, data), where address is
-           the CAN address, time is the bus time, and data is the CAN data as a
-           hex string.
-        arr: Optional list of signals which should be decoded and returned.
-        debug: True to print debugging statements.
-
-       Returns:
-        A tuple (name, data), where name is the name of the CAN message and data
-        is the decoded result. If arr is None, data is a dict of properties.
-        Otherwise data is a list of the same length as arr.
-
-        Returns (None, None) if the message could not be decoded.
-    """
-    
-    if arr is None:
-      out = {}
-    else:
-      out = [None]*len(arr)
-
-    msg = self.msgs.get(x[0])
-    if msg is None:
-      if x[0] not in self._warned_addresses:
-        #print("WARNING: Unknown message address {}".format(x[0]))
-        self._warned_addresses.add(x[0])
-      return None, None
-
-    name = msg[0][0]
-    if debug:
-      print name
-
-    blen = 8*len(x[2])
-
-    st = x[2].rjust(8, '\x00')
-    le, be = None, None
-
-    for s in msg[1]:
-      if arr is not None and s[0] not in arr:
-        continue
-
-      # big or little endian?
-      #   see http://vi-firmware.openxcplatform.com/en/master/config/bit-numbering.html
-      if s[3] is False:
-        ss = self.bits_index[s[1]]
-        if be is None:
-          be = struct.unpack(">Q", st)[0]
-        x2_int = be
-        data_bit_pos = (blen - (ss + s[2]))
-      else:
-        if le is None:
-          le = struct.unpack("<Q", st)[0]
-        x2_int = le
-        ss = s[1]
-        data_bit_pos = ss
-
-      if data_bit_pos < 0:
-        continue
-      ival = (x2_int >> data_bit_pos) & ((1 << (s[2])) - 1)
-
-      if s[4] and (ival & (1<<(s[2]-1))): # signed
-        ival -= (1<<s[2])
-
-      # control the offset
-      ival = (ival * s[5]) + s[6]
-      #if debug:
-      #  print "%40s  %2d %2d  %7.2f %s" % (s[0], s[1], s[2], ival, s[-1])
-
-      if arr is None:
-        out[s[0]] = ival
-      else:
-        out[arr.index(s[0])] = ival
-    return name, out
-
-
-  def get_signals(self, msg):
-    return [sgs.name for sgs in self.msgs[msg][1]]
-    
-if __name__ == "__main__":
-   import sys
-   import os
-   from opendbc import DBC_PATH
-
-   dbc_test = dbc(os.path.join(DBC_PATH, sys.argv[1]))
-   print dbc_test.get_signals(0xe4)

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/file_helpers.py

@@ -0,0 +1,58 @@
+import io
+import os
+import tempfile
+import contextlib
+import zstandard as zstd
+
+LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+                        overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)
+
+
+def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
+  if not should_compress:
+    file_size = os.path.getsize(filepath)
+    file_stream = open(filepath, "rb")
+    return file_stream, file_size
+
+  # Compress the file on the fly
+  compressed_stream = io.BytesIO()
+  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
+
+  with open(filepath, "rb") as f:
+    compressor.copy_stream(f, compressed_stream)
+    compressed_size = compressed_stream.tell()
+    compressed_stream.seek(0)
+    return compressed_stream, compressed_size

common/filter_simple.py

@@ -0,0 +1,17 @@
+class FirstOrderFilter:
+  def __init__(self, x0, rc, dt, initialized=True):
+    self.x = x0
+    self.dt = dt
+    self.update_alpha(rc)
+    self.initialized = initialized
+
+  def update_alpha(self, rc):
+    self.alpha = self.dt / (rc + self.dt)
+
+  def update(self, x):
+    if self.initialized:
+      self.x = (1. - self.alpha) * self.x + self.alpha * x
+    else:
+      self.initialized = True
+      self.x = x
+    return self.x

common/fingerprints.py

@@ -1,62 +0,0 @@
-import os
-
-_FINGERPRINTS = {
-  "ACURA ILX 2016 ACURAWATCH PLUS": {
-    1024L: 5, 513L: 5, 1027L: 5, 1029L: 8, 929L: 4, 1057L: 5, 777L: 8, 1034L: 5, 1036L: 8, 398L: 3, 399L: 7, 145L: 8, 660L: 8, 985L: 3, 923L: 2, 542L: 7, 773L: 7, 800L: 8, 432L: 7, 419L: 8, 420L: 8, 1030L: 5, 422L: 8, 808L: 8, 428L: 8, 304L: 8, 819L: 7, 821L: 5, 57L: 3, 316L: 8, 545L: 4, 464L: 8, 1108L: 8, 597L: 8, 342L: 6, 983L: 8, 344L: 8, 804L: 8, 1039L: 8, 476L: 4, 892L: 8, 490L: 8, 1064L: 7, 882L: 2, 884L: 7, 887L: 8, 888L: 8, 380L: 8, 1365L: 5,
-    # sent messages
-    0xe4: 5, 0x1fa: 8, 0x200: 3, 0x30c: 8, 0x33d: 5,
-  },
-  "HONDA CIVIC 2016 TOURING": {
-    1024L: 5, 513L: 5, 1027L: 5, 1029L: 8, 777L: 8, 1036L: 8, 1039L: 8, 1424L: 5, 401L: 8, 148L: 8, 662L: 4, 985L: 3, 795L: 8, 773L: 7, 800L: 8, 545L: 6, 420L: 8, 806L: 8, 808L: 8, 1322L: 5, 427L: 3, 428L: 8, 304L: 8, 432L: 7, 57L: 3, 450L: 8, 929L: 8, 330L: 8, 1302L: 8, 464L: 8, 1361L: 5, 1108L: 8, 597L: 8, 470L: 2, 344L: 8, 804L: 8, 399L: 7, 476L: 7, 1633L: 8, 487L: 4, 892L: 8, 490L: 8, 493L: 5, 884L: 8, 891L: 8, 380L: 8, 1365L: 5,
-    # sent messages
-    0xe4: 5, 0x1fa: 8, 0x200: 3, 0x30c: 8, 0x33d: 5, 0x35e: 8, 0x39f: 8,
-  },
-  "HONDA CR-V 2016 TOURING": {
-    57L: 3, 145L: 8, 316L: 8, 340L: 8, 342L: 6, 344L: 8, 380L: 8, 398L: 3, 399L: 6, 401L: 8, 420L: 8, 422L: 8, 426L: 8, 432L: 7, 464L: 8, 474L: 5, 476L: 4, 487L: 4, 490L: 8, 493L: 3, 507L: 1, 542L: 7, 545L: 4, 597L: 8, 660L: 8, 661L: 4, 773L: 7, 777L: 8, 800L: 8, 804L: 8, 808L: 8, 882L: 2, 884L: 7, 888L: 8, 891L: 8, 892L: 8, 923L: 2, 929L: 8, 983L: 8, 985L: 3, 1024L: 5, 1027L: 5, 1029L: 8, 1033L: 5, 1036L: 8, 1039L: 8, 1057L: 5, 1064L: 7, 1108L: 8, 1125L: 8, 1296L: 8, 1365L: 5, 1424L: 5, 1600L: 5, 1601L: 8,
-    # sent messages
-    0x194: 4, 0x1fa: 8, 0x30c: 8, 0x33d: 5,
-  },
-  "TOYOTA RAV4 2017": {
-    36L: 8, 37L: 8, 170L: 8, 180L: 8, 186L: 4, 426L: 6, 452L: 8, 464L: 8, 466L: 8, 467L: 8, 547L: 8, 548L: 8, 552L: 4, 562L: 4, 608L: 8, 610L: 5, 643L: 7, 705L: 8, 725L: 2, 740L: 5, 800L: 8, 835L: 8, 836L: 8, 849L: 4, 869L: 7, 870L: 7, 871L: 2, 896L: 8, 897L: 8, 900L: 6, 902L: 6, 905L: 8, 911L: 8, 916L: 3, 918L: 7, 921L: 8, 933L: 8, 944L: 8, 945L: 8, 951L: 8, 955L: 4, 956L: 8, 979L: 2, 998L: 5, 999L: 7, 1000L: 8, 1001L: 8, 1008L: 2, 1014L: 8, 1017L: 8, 1041L: 8, 1042L: 8, 1043L: 8, 1044L: 8, 1056L: 8, 1059L: 1, 1114L: 8, 1161L: 8, 1162L: 8, 1163L: 8, 1176L: 8, 1177L: 8, 1178L: 8, 1179L: 8, 1180L: 8, 1181L: 8, 1190L: 8, 1191L: 8, 1192L: 8, 1196L: 8, 1227L: 8, 1228L: 8, 1235L: 8, 1237L: 8, 1263L: 8, 1279L: 8, 1408L: 8, 1409L: 8, 1410L: 8, 1552L: 8, 1553L: 8, 1554L: 8, 1555L: 8, 1556L: 8, 1557L: 8, 1561L: 8, 1562L: 8, 1568L: 8, 1569L: 8, 1570L: 8, 1571L: 8, 1572L: 8, 1584L: 8, 1589L: 8, 1592L: 8, 1593L: 8, 1595L: 8, 1596L: 8, 1597L: 8, 1600L: 8, 1656L: 8, 1664L: 8, 1728L: 8, 1745L: 8, 1779L: 8, 1904L: 8, 1912L: 8, 1990L: 8, 1998L: 8
-  },
-  "TOYOTA RAV4 2017 HYBRID": {
-   36L: 8, 37L: 8, 170L: 8, 180L: 8, 186L: 4, 426L: 6, 452L: 8, 464L: 8, 466L: 8, 467L: 8, 547L: 8, 548L: 8, 552L: 4, 562L: 4, 608L: 8, 610L: 5, 643L: 7, 705L: 8, 725L: 2, 740L: 5, 800L: 8, 835L: 8, 836L: 8, 849L: 4, 869L: 7, 870L: 7, 871L: 2, 896L: 8, 897L: 8, 900L: 6, 902L: 6, 905L: 8, 911L: 8, 916L: 3, 918L: 7, 921L: 8, 933L: 8, 944L: 8, 945L: 8, 951L: 8, 955L: 8, 956L: 8, 979L: 2, 998L: 5, 999L: 7, 1000L: 8, 1001L: 8, 1008L: 2, 1014L: 8, 1017L: 8, 1041L: 8, 1042L: 8, 1043L: 8, 1044L: 8, 1056L: 8, 1059L: 1, 1114L: 8, 1161L: 8, 1162L: 8, 1163L: 8, 1176L: 8, 1177L: 8, 1178L: 8, 1179L: 8, 1180L: 8, 1181L: 8, 1190L: 8, 1191L: 8, 1192L: 8, 1196L: 8, 1227L: 8, 1228L: 8, 1235L: 8, 1237L: 8, 1263L: 8, 1279L: 8, 1408L: 8, 1409L: 8, 1410L: 8, 1552L: 8, 1553L: 8, 1554L: 8, 1555L: 8, 1556L: 8, 1557L: 8, 1561L: 8, 1562L: 8, 1568L: 8, 1569L: 8, 1570L: 8, 1571L: 8, 1572L: 8, 1584L: 8, 1589L: 8, 1592L: 8, 1593L: 8, 1595L: 8, 1596L: 8, 1597L: 8, 1600L: 8, 1656L: 8, 1664L: 8, 1728L: 8, 1745L: 8, 1779L: 8, 1904L: 8, 1912L: 8, 1990L: 8, 1998L: 8, 581L: 5, 296: 8, 552L: 8, 560L: 7, 552L: 4, 713L: 8, 550L: 8, 608L: 8, 37L: 8, 36L: 8, 950L: 8, 1198L: 8, 1197L: 8, 1199L: 8, 1212L: 8, 953L: 3, 1264L: 8, 1184L: 8, 1005L: 2, 1185L: 8, 1232L: 8, 1186L: 8
- },
-  "TOYOTA PRIUS 2017": {
-    36L: 8, 37L: 8, 166L: 8, 170L: 8, 180L: 8, 295L: 8, 296L: 8, 426L: 6, 452L: 8, 466L: 8, 467L: 8, 550L: 8, 552L: 4, 560L: 7, 562L: 6, 581L: 5, 608L: 8, 610L: 8, 614L: 8, 643L: 7, 658L: 8, 713L: 8, 740L: 5, 742L: 8, 743L: 8, 800L: 8, 810L: 2, 814L: 8, 829L: 2, 830L: 7, 835L: 8, 836L: 8, 863L: 8, 869L: 7, 870L: 7, 871L: 2, 898L: 8, 900L: 6, 902L: 6, 905L: 8, 918L: 8, 921L: 8, 933L: 8, 944L: 8, 945L: 8, 950L: 8, 951L: 8, 953L: 8, 955L: 8, 956L: 8, 971L: 7, 975L: 5, 993L: 8, 998L: 5, 999L: 7, 1000L: 8, 1001L: 8, 1014L: 8, 1017L: 8, 1020L: 8, 1041L: 8, 1042L: 8, 1044L: 8, 1056L: 8, 1057L: 8, 1059L: 1, 1071L: 8, 1077L: 8, 1082L: 8, 1083L: 8, 1084L: 8, 1085L: 8, 1086L: 8, 1114L: 8, 1132L: 8, 1161L: 8, 1162L: 8, 1163L: 8, 1175L: 8, 1227L: 8, 1228L: 8, 1235L: 8, 1237L: 8, 1279L: 8, 1552L: 8, 1553L: 8, 1556L: 8, 1557L: 8, 1568L: 8, 1570L: 8, 1571L: 8, 1572L: 8, 1595L: 8, 1777L: 8, 1779L: 8, 1904L: 8, 1912L: 8, 1990L: 8, 1998L: 8
-  },
-}
-
-# support additional internal only fingerprints
-try:
-  from common.fingerprints_internal import add_additional_fingerprints
-  add_additional_fingerprints(_FINGERPRINTS)
-except ImportError:
-  pass
-
-def eliminate_incompatible_cars(msg, candidate_cars):
-  """Removes cars that could not have sent msg.
-
-     Inputs:
-      msg: A cereal/log CanData message from the car.
-      candidate_cars: A list of cars to consider.
-
-     Returns:
-      A list containing the subset of candidate_cars that could have sent msg.
-  """
-  compatible_cars = []
-  for car_name in candidate_cars:
-    adr = msg.address
-    if msg.src != 0 or (adr in _FINGERPRINTS[car_name] and
-                        _FINGERPRINTS[car_name][adr] == len(msg.dat)):
-      compatible_cars.append(car_name)
-    else:
-      pass
-      #isin = adr in _FINGERPRINTS[car_name]
-      #print "eliminate", car_name, hex(adr), isin, len(msg.dat), msg.dat.encode("hex")
-  return compatible_cars
-
-def all_known_cars():
-  """Returns a list of all known car strings."""
-  return _FINGERPRINTS.keys()
-

common/git.py

@@ -0,0 +1,42 @@
+from functools import cache
+import subprocess
+from openpilot.common.run import run_cmd, run_cmd_default
+
+
+@cache
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
+  return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
+
+
+@cache
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
+  return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
+
+
+@cache
+def get_short_branch(cwd: str = None) -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
+
+
+@cache
+def get_branch(cwd: str = None) -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
+
+
+@cache
+def get_origin(cwd: str = None) -> str:
+  try:
+    local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
+    tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
+    return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
+  except subprocess.CalledProcessError:  # Not on a branch, fallback
+    return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
+
+
+@cache
+def get_normalized_origin(cwd: str = None) -> str:
+  return get_origin(cwd) \
+    .replace("git@", "", 1) \
+    .replace(".git", "", 1) \
+    .replace("https://", "", 1) \
+    .replace(":", "/", 1)

common/gpio.py

@@ -0,0 +1,89 @@
+import os
+import fcntl
+import ctypes
+from functools import cache
+
+def gpio_init(pin: int, output: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
+      f.write(b"out" if output else b"in")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} direction: {e}")
+
+def gpio_set(pin: int, high: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
+      f.write(b"1" if high else b"0")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+def gpio_read(pin: int) -> bool | None:
+  val = None
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
+      val = bool(int(f.read().strip()))
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+  return val
+
+def gpio_export(pin: int) -> None:
+  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
+    return
+
+  try:
+    with open("/sys/class/gpio/export", 'w') as f:
+      f.write(str(pin))
+  except Exception:
+    print(f"Failed to export gpio {pin}")
+
+@cache
+def get_irq_action(irq: int) -> list[str]:
+  try:
+    with open(f"/sys/kernel/irq/{irq}/actions") as f:
+      actions = f.read().strip().split(',')
+      return actions
+  except FileNotFoundError:
+    return []
+
+def get_irqs_for_action(action: str) -> list[str]:
+  ret = []
+  with open("/proc/interrupts") as f:
+    for l in f.readlines():
+      irq = l.split(':')[0].strip()
+      if irq.isdigit() and action in get_irq_action(irq):
+        ret.append(irq)
+  return ret
+
+# *** gpiochip ***
+
+class gpioevent_data(ctypes.Structure):
+  _fields_ = [
+    ("timestamp", ctypes.c_uint64),
+    ("id", ctypes.c_uint32),
+  ]
+
+class gpioevent_request(ctypes.Structure):
+  _fields_ = [
+    ("lineoffset", ctypes.c_uint32),
+    ("handleflags", ctypes.c_uint32),
+    ("eventflags", ctypes.c_uint32),
+    ("label", ctypes.c_char * 32),
+    ("fd", ctypes.c_int)
+  ]
+
+def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
+  GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
+  GPIOHANDLE_REQUEST_INPUT = 0x1
+  GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
+
+  rq = gpioevent_request()
+  rq.lineoffset = pin
+  rq.handleflags = GPIOHANDLE_REQUEST_INPUT
+  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
+  rq.label = label.encode('utf-8')[:31] + b'\0'
+
+  fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
+  fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
+  os.close(fd)
+  return int(rq.fd)

common/gps.py

@@ -0,0 +1,8 @@
+from openpilot.common.params import Params
+
+
+def get_gps_location_service(params: Params) -> str:
+  if params.get_bool("UbloxAvailable"):
+    return "gpsLocationExternal"
+  else:
+    return "gpsLocation"

common/kalman/ekf.py

@@ -1,252 +0,0 @@
-import abc
-import numpy as np
-import numpy.matlib
-# The EKF class contains the framework for an Extended Kalman Filter, but must be subclassed to use.
-# A subclass must implement:
-#   1) calc_transfer_fun(); see bottom of file for more info.
-#   2) __init__() to initialize self.state, self.covar, and self.process_noise appropriately
-
-# Alternatively, the existing implementations of EKF can be used (e.g. EKF2D)
-
-# Sensor classes are optionally used to pass measurement information into the EKF, to keep
-#   sensor parameters and processing methods for a each sensor together.
-# Sensor classes have a read() method which takes raw sensor data and returns
-#   a SensorReading object, which can be passed to the EKF update() method.
-
-# For usage, see run_ekf1d.py in selfdrive/new for a simple example.
-# ekf.predict(dt) should be called between update cycles with the time since it was last called.
-# Ideally, predict(dt) should be called at a relatively constant rate.
-# update() should be called once per sensor, and can be called multiple times between predict steps.
-# Access and set the state of the filter directly with ekf.state and ekf.covar.
-
-
-class SensorReading:
-  # Given a perfect model and no noise, data = obs_model * state
-  def __init__(self, data, covar, obs_model):
-    self.data = data
-    self.obs_model = obs_model
-    self.covar = covar
-
-  def __repr__(self):
-    return "SensorReading(data={}, covar={}, obs_model={})".format(
-      repr(self.data), repr(self.covar), repr(self.obs_model))
-
-
-# A generic sensor class that does no pre-processing of data
-class SimpleSensor:
-  # obs_model can be
-  #   a full observation model matrix, or
-  #   an integer or tuple of indices into ekf.state, indicating which variables are being directly observed
-  # covar can be
-  #   a full covariance matrix
-  #   a float or tuple of individual covars for each component of the sensor reading
-  # dims is the number of states in the EKF
-  def __init__(self, obs_model, covar, dims):
-    # Allow for integer covar/obs_model
-    if not hasattr(obs_model, "__len__"):
-      obs_model = (obs_model, )
-    if not hasattr(covar, "__len__"):
-      covar = (covar, )
-
-    # Full observation model passed
-    if dims in np.array(obs_model).shape:
-      self.obs_model = np.asmatrix(obs_model)
-      self.covar = np.asmatrix(covar)
-    # Indices of unit observations passed
-    else:
-      self.obs_model = np.matlib.zeros((len(obs_model), dims))
-      self.obs_model[:, list(obs_model)] = np.identity(len(obs_model))
-      if np.asarray(covar).ndim == 2:
-        self.covar = np.asmatrix(covar)
-      elif len(covar) == len(obs_model):
-        self.covar = np.matlib.diag(covar)
-      else:
-        self.covar = np.matlib.identity(len(obs_model)) * covar
-
-  def read(self, data, covar=None):
-    if covar:
-      self.covar = covar
-    return SensorReading(data, self.covar, self.obs_model)
-
-
-class EKF:
-  __metaclass__ = abc.ABCMeta
-
-  def __init__(self, debug=False):
-    self.DEBUG = debug
-
-
-  def __str__(self):
-    return "EKF(state={}, covar={})".format(self.state, self.covar)
-
-  # Measurement update
-  # Reading should be a SensorReading object with data, covar, and obs_model attributes
-  def update(self, reading):
-    # Potential improvements:
-    # deal with negative covars
-    # add noise to really low covars to ensure stability
-    # use mahalanobis distance to reject outliers
-    # wrap angles after state updates and innovation
-
-    # y = z - H*x
-    innovation = reading.data - reading.obs_model * self.state
-
-    if self.DEBUG:
-      print "reading:\n",reading.data
-      print "innovation:\n",innovation
-
-    # S = H*P*H' + R
-    innovation_covar = reading.obs_model * self.covar * reading.obs_model.T + reading.covar
-
-    # K = P*H'*S^-1
-    kalman_gain = self.covar * reading.obs_model.T * np.linalg.inv(
-      innovation_covar)
-
-    if self.DEBUG:
-      print "gain:\n", kalman_gain
-      print "innovation_covar:\n", innovation_covar
-      print "innovation: ", innovation
-      print "test: ", self.covar * reading.obs_model.T * (
-        reading.obs_model * self.covar * reading.obs_model.T + reading.covar *
-        0).I
-
-    # x = x + K*y
-    self.state += kalman_gain*innovation
-
-    # print "covar", np.diag(self.covar)
-    #self.state[(roll_vel, yaw_vel, pitch_vel),:] = reading.data
-
-    # Standard form: P = (I - K*H)*P
-    # self.covar = (self.identity - kalman_gain*reading.obs_model) * self.covar
-
-    # Use the Joseph form for numerical stability: P = (I-K*H)*P*(I - K*H)' + K*R*K'
-    aux_mtrx = (self.identity - kalman_gain * reading.obs_model)
-    self.covar = aux_mtrx * self.covar * aux_mtrx.T + kalman_gain * reading.covar * kalman_gain.T
-
-    if self.DEBUG:
-      print "After update"
-      print "state\n", self.state
-      print "covar:\n",self.covar
-
-  def update_scalar(self, reading):
-    # like update but knowing that measurement is a scalar
-    # this avoids matrix inversions and speeds up (surprisingly) drived.py a lot
-
-    # innovation = reading.data - np.matmul(reading.obs_model, self.state)
-    # innovation_covar = np.matmul(np.matmul(reading.obs_model, self.covar), reading.obs_model.T) + reading.covar
-    # kalman_gain = np.matmul(self.covar, reading.obs_model.T)/innovation_covar
-    # self.state += np.matmul(kalman_gain, innovation)
-    # aux_mtrx = self.identity - np.matmul(kalman_gain, reading.obs_model)
-    # self.covar =  np.matmul(aux_mtrx, np.matmul(self.covar, aux_mtrx.T)) + np.matmul(kalman_gain, np.matmul(reading.covar, kalman_gain.T))
-
-    # written without np.matmul
-    es = np.einsum
-    ABC_T = "ij,jk,lk->il"
-    AB_T = "ij,kj->ik"
-    AB = "ij,jk->ik"
-    innovation = reading.data - es(AB, reading.obs_model, self.state)
-    innovation_covar = es(ABC_T, reading.obs_model, self.covar,
-                          reading.obs_model) + reading.covar
-    kalman_gain = es(AB_T, self.covar, reading.obs_model) / innovation_covar
-
-    self.state += es(AB, kalman_gain, innovation)
-    aux_mtrx = self.identity - es(AB, kalman_gain, reading.obs_model)
-    self.covar = es(ABC_T, aux_mtrx, self.covar, aux_mtrx) + \
-                 es(ABC_T, kalman_gain, reading.covar, kalman_gain)
-
-  # Prediction update
-  def predict(self, dt):
-    es = np.einsum
-    ABC_T = "ij,jk,lk->il"
-    AB = "ij,jk->ik"
-
-    # State update
-    transfer_fun, transfer_fun_jacobian = self.calc_transfer_fun(dt)
-
-    # self.state = np.matmul(transfer_fun, self.state)
-    # self.covar = np.matmul(np.matmul(transfer_fun_jacobian, self.covar), transfer_fun_jacobian.T) + self.process_noise * dt
-
-    # x = f(x, u), written in the form x = A(x, u)*x
-    self.state = es(AB, transfer_fun, self.state)
-
-    # P = J*P*J' + Q
-    self.covar = es(ABC_T, transfer_fun_jacobian, self.covar,
-                    transfer_fun_jacobian) + self.process_noise * dt  #!dt
-
-    #! Clip covariance to avoid explosions
-    self.covar = np.clip(self.covar,-1e10,1e10)
-
-  @abc.abstractmethod
-  def calc_transfer_fun(self, dt):
-    """Return a tuple with the transfer function and transfer function jacobian
-    The transfer function and jacobian should both be a numpy matrix of size DIMSxDIMS
-
-    The transfer function matrix A should satisfy the state-update equation
-      x_(k+1) = A * x_k
-
-    The jacobian J is the direct jacobian A*x_k. For linear systems J=A.
-
-    Current implementations calculate A and J as functions of state. Control input
-      can be added trivially by adding a control parameter to predict() and calc_tranfer_update(),
-      and using it during calculation of A and J
-    """
-
-
-class FastEKF1D(EKF):
-  """Fast version of EKF for 1D problems with scalar readings."""
-
-  def __init__(self, dt, var_init, Q):
-    super(FastEKF1D, self).__init__(False)
-    self.state = [0, 0]
-    self.covar = [var_init, var_init, 0]
-
-    # Process Noise
-    self.dtQ0 = dt * Q[0]
-    self.dtQ1 = dt * Q[1]
-
-  def update(self, reading):
-    raise NotImplementedError
-
-  def update_scalar(self, reading):
-    # TODO(mgraczyk): Delete this for speed.
-    # assert np.all(reading.obs_model == [1, 0])
-
-    rcov = reading.covar[0, 0]
-
-    x = self.state
-    S = self.covar
-
-    innovation = reading.data - x[0]
-    innovation_covar = S[0] + rcov
-
-    k0 = S[0] / innovation_covar
-    k1 = S[2] / innovation_covar
-
-    x[0] += k0 * innovation
-    x[1] += k1 * innovation
-
-    mk = 1 - k0
-    S[1] += k1 * (k1 * (S[0] + rcov) - 2 * S[2])
-    S[2] = mk * (S[2] - k1 * S[0]) + rcov * k0 * k1
-    S[0] = mk * mk * S[0] + rcov * k0 * k0
-
-  def predict(self, dt):
-    # State update
-    x = self.state
-
-    x[0] += dt * x[1]
-
-    # P = J*P*J' + Q
-    S = self.covar
-    S[0] += dt * (2 * S[2] + dt * S[1]) + self.dtQ0
-    S[2] += dt * S[1]
-    S[1] += self.dtQ1
-
-    # Clip covariance to avoid explosions
-    S = max(-1e10, min(S, 1e10))
-
-  def calc_transfer_fun(self, dt):
-    tf = np.identity(2)
-    tf[0, 1] = dt
-    tfj = tf
-    return tf, tfj

common/kalman/simple_kalman.py

@@ -1,23 +0,0 @@
-import numpy as np
-
-
-class KF1D:
-  # this EKF assumes constant covariance matrix, so calculations are much simpler
-  # the Kalman gain also needs to be precomputed using the control module
-
-  def __init__(self, x0, A, C, K):
-    self.x = x0
-    self.A = A
-    self.C = C
-    self.K = K
-
-    self.A_K = self.A - np.dot(self.K, self.C)
-
-    # K matrix needs to  be pre-computed as follow:
-    # import control
-    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
-    # self.K = np.transpose(K)
-
-  def update(self, meas):
-    self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
-    return self.x

common/logging_extra.py

@@ -1,14 +1,23 @@
+import io
 import os
 import sys
 import copy
 import json
+import time
+import uuid
 import socket
 import logging
+import traceback
+import numpy as np
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
 
+LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
+
 def json_handler(obj):
+  if isinstance(obj, np.bool_):
+    return bool(obj)
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
   return repr(obj)
@@ -18,7 +27,7 @@ def json_robust_dumps(obj):
 
 class NiceOrderedDict(OrderedDict):
   def __str__(self):
-    return '{'+', '.join("%r: %r" % p for p in self.iteritems())+'}'
+    return json_robust_dumps(self)
 
 class SwagFormatter(logging.Formatter):
   def __init__(self, swaglogger):
@@ -60,10 +69,57 @@ class SwagFormatter(logging.Formatter):
     return record_dict
 
   def format(self, record):
+    if self.swaglogger is None:
+      raise Exception("must set swaglogger before calling format()")
     return json_robust_dumps(self.format_dict(record))
 
-_tmpfunc = lambda: 0
-_srcfile = os.path.normcase(_tmpfunc.__code__.co_filename)
+class SwagLogFileFormatter(SwagFormatter):
+  def fix_kv(self, k, v):
+    # append type to names to preserve legacy naming in logs
+    # avoids overlapping key namespaces with different types
+    # e.g. log.info() creates 'msg' -> 'msg$s'
+    #      log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
+    #      because overlapping namespace 'msg' caused problems
+    if isinstance(v, (str, bytes)):
+      k += "$s"
+    elif isinstance(v, float):
+      k += "$f"
+    elif isinstance(v, bool):
+      k += "$b"
+    elif isinstance(v, int):
+      k += "$i"
+    elif isinstance(v, dict):
+      nv = {}
+      for ik, iv in v.items():
+        ik, iv = self.fix_kv(ik, iv)
+        nv[ik] = iv
+      v = nv
+    elif isinstance(v, list):
+      k += "$a"
+    return k, v
+
+  def format(self, record):
+    if isinstance(record, str):
+      v = json.loads(record)
+    else:
+      v = self.format_dict(record)
+
+    mk, mv = self.fix_kv('msg', v['msg'])
+    del v['msg']
+    v[mk] = mv
+    v['id'] = uuid.uuid4().hex
+
+    return json_robust_dumps(v)
+
+class SwagErrorFilter(logging.Filter):
+  def filter(self, record):
+    return record.levelno < logging.ERROR
+
+def _tmpfunc():
+  return 0
+
+def _srcfile():
+  return os.path.normcase(_tmpfunc.__code__.co_filename)
 
 class SwagLogger(logging.Logger):
   def __init__(self):
@@ -74,28 +130,6 @@ class SwagLogger(logging.Logger):
     self.log_local = local()
     self.log_local.ctx = {}
 
-  def findCaller(self):
-    """
-      Find the stack frame of the caller so that we can note the source
-      file name, line number and function name.
-      """
-    # f = currentframe()
-    f = sys._getframe(3)
-    #On some versions of IronPython, currentframe() returns None if
-    #IronPython isn't run with -X:Frames.
-    if f is not None:
-      f = f.f_back
-    rv = "(unknown file)", 0, "(unknown function)"
-    while hasattr(f, "f_code"):
-      co = f.f_code
-      filename = os.path.normcase(co.co_filename)
-      if filename in (logging._srcfile, _srcfile):
-        f = f.f_back
-        continue
-      rv = (co.co_filename, f.f_lineno, co.co_name)
-      break
-    return rv
-
   def local_ctx(self):
     try:
       return self.log_local.ctx
@@ -122,21 +156,94 @@ class SwagLogger(logging.Logger):
   def bind_global(self, **kwargs):
     self.global_ctx.update(kwargs)
 
-  def event(self, event_name, *args, **kwargs):
+  def event(self, event, *args, **kwargs):
     evt = NiceOrderedDict()
-    evt['event'] = event_name
+    evt['event'] = event
     if args:
       evt['args'] = args
     evt.update(kwargs)
-    self.info(evt)
+    if 'error' in kwargs:
+      self.error(evt)
+    elif 'debug' in kwargs:
+      self.debug(evt)
+    else:
+      self.info(evt)
+
+  def timestamp(self, event_name):
+    if LOG_TIMESTAMPS:
+      t = time.monotonic()
+      tstp = NiceOrderedDict()
+      tstp['timestamp'] = NiceOrderedDict()
+      tstp['timestamp']["event"] = event_name
+      tstp['timestamp']["time"] = t*1e9
+      self.debug(tstp)
+
+  def findCaller(self, stack_info=False, stacklevel=1):
+    """
+    Find the stack frame of the caller so that we can note the source
+    file name, line number and function name.
+    """
+    f = sys._getframe(3)
+    #On some versions of IronPython, currentframe() returns None if
+    #IronPython isn't run with -X:Frames.
+    if f is not None:
+      f = f.f_back
+    orig_f = f
+    while f and stacklevel > 1:
+      f = f.f_back
+      stacklevel -= 1
+    if not f:
+      f = orig_f
+    rv = "(unknown file)", 0, "(unknown function)", None
+    while hasattr(f, "f_code"):
+      co = f.f_code
+      filename = os.path.normcase(co.co_filename)
+
+      if filename == _srcfile:
+        f = f.f_back
+        continue
+      sinfo = None
+      if stack_info:
+        sio = io.StringIO()
+        sio.write('Stack (most recent call last):\n')
+        traceback.print_stack(f, file=sio)
+        sinfo = sio.getvalue()
+        if sinfo[-1] == '\n':
+          sinfo = sinfo[:-1]
+        sio.close()
+      rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
+      break
+    return rv
 
 if __name__ == "__main__":
   log = SwagLogger()
 
+  stdout_handler = logging.StreamHandler(sys.stdout)
+  stdout_handler.setLevel(logging.INFO)
+  stdout_handler.addFilter(SwagErrorFilter())
+  log.addHandler(stdout_handler)
+
+  stderr_handler = logging.StreamHandler(sys.stderr)
+  stderr_handler.setLevel(logging.ERROR)
+  log.addHandler(stderr_handler)
+
   log.info("asdasd %s", "a")
   log.info({'wut': 1})
+  log.warning("warning")
+  log.error("error")
+  log.critical("critical")
+  log.event("test", x="y")
 
   with log.ctx():
+    stdout_handler.setFormatter(SwagFormatter(log))
+    stderr_handler.setFormatter(SwagFormatter(log))
     log.bind(user="some user")
     log.info("in req")
-    log.event("do_req")
+    print("")
+    log.warning("warning")
+    print("")
+    log.error("error")
+    print("")
+    log.critical("critical")
+    print("")
+    log.event("do_req", a=1, b="c")

common/markdown.py

@@ -0,0 +1,45 @@
+HTML_REPLACEMENTS = [
+  (r'&', r'&'),
+  (r'"', r'"'),
+]
+
+def parse_markdown(text: str, tab_length: int = 2) -> str:
+  lines = text.split("\n")
+  output: list[str] = []
+  list_level = 0
+
+  def end_outstanding_lists(level: int, end_level: int) -> int:
+    while level > end_level:
+      level -= 1
+      output.append("</ul>")
+      if level > 0:
+        output.append("</li>")
+    return end_level
+
+  for i, line in enumerate(lines):
+    if i + 1 < len(lines) and lines[i + 1].startswith("==="):  # heading
+      output.append(f"<h1>{line}</h1>")
+    elif line.startswith("==="):
+      pass
+    elif line.lstrip().startswith("* "):  # list
+      line_level = 1 + line.count(" " * tab_length, 0, line.index("*"))
+      if list_level >= line_level:
+        list_level = end_outstanding_lists(list_level, line_level)
+      else:
+        list_level += 1
+        if list_level > 1:
+          output[-1] = output[-1].replace("</li>", "")
+        output.append("<ul>")
+      output.append(f"<li>{line.replace('*', '', 1).lstrip()}</li>")
+    else:
+      list_level = end_outstanding_lists(list_level, 0)
+      if len(line) > 0:
+        output.append(line)
+
+  end_outstanding_lists(list_level, 0)
+  output_str = "\n".join(output) + "\n"
+
+  for (fr, to) in HTML_REPLACEMENTS:
+    output_str = output_str.replace(fr, to)
+
+  return output_str

common/mat.h

@@ -1,8 +1,7 @@
-#ifndef COMMON_MAT_H
-#define COMMON_MAT_H
+#pragma once
 
 typedef struct vec3 {
-	float v[3];
+  float v[3];
 } vec3;
 
 typedef struct vec4 {
@@ -10,14 +9,14 @@ typedef struct vec4 {
 } vec4;
 
 typedef struct mat3 {
-	float v[3*3];
+  float v[3*3];
 } mat3;
 
 typedef struct mat4 {
   float v[4*4];
 } mat4;
 
-static inline mat3 matmul3(const mat3 a, const mat3 b) {
+static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
   mat3 ret = {{0.0}};
   for (int r=0; r<3; r++) {
     for (int c=0; c<3; c++) {
@@ -31,7 +30,7 @@ static inline mat3 matmul3(const mat3 a, const mat3 b) {
   return ret;
 }
 
-static inline vec3 matvecmul3(const mat3 a, const vec3 b) {
+static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
   vec3 ret = {{0.0}};
   for (int r=0; r<3; r++) {
     for (int c=0; c<3; c++) {
@@ -41,7 +40,7 @@ static inline vec3 matvecmul3(const mat3 a, const vec3 b) {
   return ret;
 }
 
-static inline mat4 matmul(const mat4 a, const mat4 b) {
+static inline mat4 matmul(const mat4 &a, const mat4 &b) {
   mat4 ret = {{0.0}};
   for (int r=0; r<4; r++) {
     for (int c=0; c<4; c++) {
@@ -55,7 +54,7 @@ static inline mat4 matmul(const mat4 a, const mat4 b) {
   return ret;
 }
 
-static inline vec4 matvecmul(const mat4 a, const vec4 b) {
+static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
   vec4 ret = {{0.0}};
   for (int r=0; r<4; r++) {
     for (int c=0; c<4; c++) {
@@ -67,7 +66,7 @@ static inline vec4 matvecmul(const mat4 a, const vec4 b) {
 
 // scales the input and output space of a transformation matrix
 // that assumes pixel-center origin.
-static inline mat3 transform_scale_buffer(const mat3 in, float s) {
+static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
   // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
 
   mat3 transform_out = {{
@@ -84,5 +83,3 @@ static inline mat3 transform_scale_buffer(const mat3 in, float s) {
 
   return matmul3(transform_in, matmul3(in, transform_out));
 }
-
-#endif

common/mock/__init__.py

@@ -0,0 +1,50 @@
+"""
+Utilities for generating mock messages for testing.
+example in common/tests/test_mock.py
+"""
+
+
+import functools
+import threading
+from cereal.messaging import PubMaster
+from cereal.services import SERVICE_LIST
+from openpilot.common.mock.generators import generate_livePose
+from openpilot.common.realtime import Ratekeeper
+
+
+MOCK_GENERATOR = {
+  "livePose": generate_livePose
+}
+
+
+def generate_messages_loop(services: list[str], done: threading.Event):
+  pm = PubMaster(services)
+  rk = Ratekeeper(100)
+  i = 0
+  while not done.is_set():
+    for s in services:
+      should_send = i % (100/SERVICE_LIST[s].frequency) == 0
+      if should_send:
+        message = MOCK_GENERATOR[s]()
+        pm.send(s, message)
+    i += 1
+    rk.keep_time()
+
+
+def mock_messages(services: list[str] | str):
+  if isinstance(services, str):
+    services = [services]
+
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      done = threading.Event()
+      t = threading.Thread(target=generate_messages_loop, args=(services, done))
+      t.start()
+      try:
+        return func(*args, **kwargs)
+      finally:
+        done.set()
+        t.join()
+    return wrapper
+  return decorator

common/mock/generators.py

@@ -0,0 +1,14 @@
+from cereal import messaging
+
+
+def generate_livePose():
+  msg = messaging.new_message('livePose')
+  meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
+  msg.livePose.orientationNED = meas
+  msg.livePose.velocityDevice = meas
+  msg.livePose.angularVelocityDevice = meas
+  msg.livePose.accelerationDevice = meas
+  msg.livePose.inputsOK = True
+  msg.livePose.posenetOK = True
+  msg.livePose.sensorsOK = True
+  return msg

common/numpy_fast.py

@@ -1,25 +0,0 @@
-def clip(x, lo, hi):
-  return max(lo, min(hi, x))
-
-
-def interp(x, xp, fp):
-  N = len(xp)
-  if not hasattr(x, '__iter__'):
-    hi = 0
-    while hi < N and x > xp[hi]:
-      hi += 1
-    low = hi - 1
-    return fp[-1] if hi == N and x > xp[low] else (
-      fp[0] if hi == 0 else
-      (x - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]) + fp[low])
-
-  result = []
-  for v in x:
-    hi = 0
-    while hi < N and v > xp[hi]:
-      hi += 1
-    low = hi - 1
-    result.append(fp[-1] if hi == N and v > xp[low] else (fp[
-      0] if hi == 0 else (v - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]
-                                                               ) + fp[low]))
-  return result

common/params.cc

@@ -0,0 +1,242 @@
+#include "common/params.h"
+
+#include <dirent.h>
+#include <sys/file.h>
+
+#include <algorithm>
+#include <cassert>
+#include <csignal>
+#include <unordered_map>
+
+#include "common/params_keys.h"
+#include "common/queue.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+
+namespace {
+
+volatile sig_atomic_t params_do_exit = 0;
+void params_sig_handler(int signal) {
+  params_do_exit = 1;
+}
+
+int fsync_dir(const std::string &path) {
+  int result = -1;
+  int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
+  if (fd >= 0) {
+    result = HANDLE_EINTR(fsync(fd));
+    HANDLE_EINTR(close(fd));
+  }
+  return result;
+}
+
+bool create_params_path(const std::string &param_path, const std::string &key_path) {
+  // Make sure params path exists
+  if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
+    return false;
+  }
+
+  // See if the symlink exists, otherwise create it
+  if (!util::file_exists(key_path)) {
+    // 1) Create temp folder
+    // 2) Symlink it to temp link
+    // 3) Move symlink to <params>/d
+
+    std::string tmp_path = param_path + "/.tmp_XXXXXX";
+    // this should be OK since mkdtemp just replaces characters in place
+    char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
+    if (tmp_dir == NULL) {
+      return false;
+    }
+
+    std::string link_path = std::string(tmp_dir) + ".link";
+    if (symlink(tmp_dir, link_path.c_str()) != 0) {
+      return false;
+    }
+
+    // don't return false if it has been created by other
+    if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
+  std::string params_path = path.empty() ? Path::params() : path;
+  if (!create_params_path(params_path, params_path + prefix)) {
+    throw std::runtime_error(util::string_format(
+        "Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
+        errno, params_path.c_str(), prefix.c_str()));
+  }
+  return params_path;
+}
+
+class FileLock {
+public:
+  FileLock(const std::string &fn) {
+    fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
+    if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
+      LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
+    }
+  }
+  ~FileLock() { close(fd_); }
+
+private:
+  int fd_ = -1;
+};
+
+} // namespace
+
+
+Params::Params(const std::string &path) {
+  params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
+  params_path = ensure_params_path(params_prefix, path);
+}
+
+Params::~Params() {
+  if (future.valid()) {
+    future.wait();
+  }
+  assert(queue.empty());
+}
+
+std::vector<std::string> Params::allKeys() const {
+  std::vector<std::string> ret;
+  for (auto &p : keys) {
+    ret.push_back(p.first);
+  }
+  return ret;
+}
+
+bool Params::checkKey(const std::string &key) {
+  return keys.find(key) != keys.end();
+}
+
+ParamKeyFlag Params::getKeyFlag(const std::string &key) {
+  return static_cast<ParamKeyFlag>(keys[key].flags);
+}
+
+ParamKeyType Params::getKeyType(const std::string &key) {
+  return keys[key].type;
+}
+
+std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
+  return keys[key].default_value;
+}
+
+int Params::put(const char* key, const char* value, size_t value_size) {
+  // Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
+  // 1) Create temp file
+  // 2) Write data to temp file
+  // 3) fsync() the temp file
+  // 4) rename the temp file to the real name
+  // 5) fsync() the containing directory
+  std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
+  int tmp_fd = mkstemp((char*)tmp_path.c_str());
+  if (tmp_fd < 0) return -1;
+
+  int result = -1;
+  do {
+    // Write value to temp.
+    ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
+    if (bytes_written < 0 || (size_t)bytes_written != value_size) {
+      result = -20;
+      break;
+    }
+
+    // fsync to force persist the changes.
+    if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break;
+
+    FileLock file_lock(params_path + "/.lock");
+
+    // Move temp into place.
+    if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
+
+    // fsync parent directory
+    result = fsync_dir(getParamPath());
+  } while (false);
+
+  close(tmp_fd);
+  if (result != 0) {
+    ::unlink(tmp_path.c_str());
+  }
+  return result;
+}
+
+int Params::remove(const std::string &key) {
+  FileLock file_lock(params_path + "/.lock");
+  int result = unlink(getParamPath(key).c_str());
+  if (result != 0) {
+    return result;
+  }
+  return fsync_dir(getParamPath());
+}
+
+std::string Params::get(const std::string &key, bool block) {
+  if (!block) {
+    return util::read_file(getParamPath(key));
+  } else {
+    // blocking read until successful
+    params_do_exit = 0;
+    void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
+    void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
+
+    std::string value;
+    while (!params_do_exit) {
+      if (value = util::read_file(getParamPath(key)); !value.empty()) {
+        break;
+      }
+      util::sleep_for(100);  // 0.1 s
+    }
+
+    std::signal(SIGINT, prev_handler_sigint);
+    std::signal(SIGTERM, prev_handler_sigterm);
+    return value;
+  }
+}
+
+std::map<std::string, std::string> Params::readAll() {
+  FileLock file_lock(params_path + "/.lock");
+  return util::read_files_in_dir(getParamPath());
+}
+
+void Params::clearAll(ParamKeyFlag key_flag) {
+  FileLock file_lock(params_path + "/.lock");
+
+  // 1) delete params of key_flag
+  // 2) delete files that are not defined in the keys.
+  if (DIR *d = opendir(getParamPath().c_str())) {
+    struct dirent *de = NULL;
+    while ((de = readdir(d))) {
+      if (de->d_type != DT_DIR) {
+        auto it = keys.find(de->d_name);
+        if (it == keys.end() || (it->second.flags & key_flag)) {
+          unlink(getParamPath(de->d_name).c_str());
+        }
+      }
+    }
+    closedir(d);
+  }
+
+  fsync_dir(getParamPath());
+}
+
+void Params::putNonBlocking(const std::string &key, const std::string &val) {
+   queue.push(std::make_pair(key, val));
+  // start thread on demand
+  if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
+    future = std::async(std::launch::async, &Params::asyncWriteThread, this);
+  }
+}
+
+void Params::asyncWriteThread() {
+  // TODO: write the latest one if a key has multiple values in the queue.
+  std::pair<std::string, std::string> p;
+  while (queue.try_pop(p, 0)) {
+    // Params::put is Thread-Safe
+    put(p.first, p.second);
+  }
+}

common/params.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include <future>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "common/queue.h"
+
+enum ParamKeyFlag {
+  PERSISTENT = 0x02,
+  CLEAR_ON_MANAGER_START = 0x04,
+  CLEAR_ON_ONROAD_TRANSITION = 0x08,
+  CLEAR_ON_OFFROAD_TRANSITION = 0x10,
+  DONT_LOG = 0x20,
+  DEVELOPMENT_ONLY = 0x40,
+  CLEAR_ON_IGNITION_ON = 0x80,
+  ALL = 0xFFFFFFFF
+};
+
+enum ParamKeyType {
+  STRING = 0, // must be utf-8 decodable
+  BOOL = 1,
+  INT = 2,
+  FLOAT = 3,
+  TIME = 4, // ISO 8601
+  JSON = 5,
+  BYTES = 6
+};
+
+struct ParamKeyAttributes {
+  uint32_t flags;
+  ParamKeyType type;
+  std::optional<std::string> default_value = std::nullopt;
+};
+
+class Params {
+public:
+  explicit Params(const std::string &path = {});
+  ~Params();
+  // Not copyable.
+  Params(const Params&) = delete;
+  Params& operator=(const Params&) = delete;
+
+  std::vector<std::string> allKeys() const;
+  bool checkKey(const std::string &key);
+  ParamKeyFlag getKeyFlag(const std::string &key);
+  ParamKeyType getKeyType(const std::string &key);
+  std::optional<std::string> getKeyDefaultValue(const std::string &key);
+  inline std::string getParamPath(const std::string &key = {}) {
+    return params_path + params_prefix + (key.empty() ? "" : "/" + key);
+  }
+
+  // Delete a value
+  int remove(const std::string &key);
+  void clearAll(ParamKeyFlag flag);
+
+  // helpers for reading values
+  std::string get(const std::string &key, bool block = false);
+  inline bool getBool(const std::string &key, bool block = false) {
+    return get(key, block) == "1";
+  }
+  std::map<std::string, std::string> readAll();
+
+  // helpers for writing values
+  int put(const char *key, const char *val, size_t value_size);
+  inline int put(const std::string &key, const std::string &val) {
+    return put(key.c_str(), val.data(), val.size());
+  }
+  inline int putBool(const std::string &key, bool val) {
+    return put(key.c_str(), val ? "1" : "0", 1);
+  }
+  void putNonBlocking(const std::string &key, const std::string &val);
+  inline void putBoolNonBlocking(const std::string &key, bool val) {
+    putNonBlocking(key, val ? "1" : "0");
+  }
+
+private:
+  void asyncWriteThread();
+
+  std::string params_path;
+  std::string params_prefix;
+
+  // for nonblocking write
+  std::future<void> future;
+  SafeQueue<std::pair<std::string, std::string>> queue;
+};

common/params.py

@@ -1,330 +1,19 @@
-#!/usr/bin/env python
-"""ROS has a parameter server, we have files.
-
-The parameter store is a persistent key value store, implemented as a directory with a writer lock.
-On Android, we store params under params_dir = /data/params. The writer lock is a file
-"<params_dir>/.lock" taken using flock(), and data is stored in a directory symlinked to by
-"<params_dir>/d".
-
-Each key, value pair is stored as a file with named <key> with contents <value>, located in
-  <params_dir>/d/<key>
-
-Readers of a single key can just open("<params_dir>/d/<key>") and read the file contents.
-Readers who want a consistent snapshot of multiple keys should take the lock.
-
-Writers should take the lock before modifying anything. Writers should also leave the DB in a
-consistent state after a crash. The implementation below does this by copying all params to a temp
-directory <params_dir>/<tmp>, then atomically symlinking <params_dir>/<d> to <params_dir>/<tmp>
-before deleting the old <params_dir>/<d> directory.
-
-Writers that only modify a single key can simply take the lock, then swap the corresponding value
-file in place without messing with <params_dir>/d.
-"""
-import time
-import os
-import errno
-import sys
-import shutil
-import fcntl
-import tempfile
-from enum import Enum
-
-def mkdirs_exists_ok(path):
-  try:
-    os.makedirs(path)
-  except OSError:
-    if not os.path.isdir(path):
-      raise
-
-class TxType(Enum):
-  PERSISTANT = 1
-  CLEAR_ON_MANAGER_START = 2
-  CLEAR_ON_CAR_START = 3
-
-class UnknownKeyName(Exception):
-  pass
-
-keys = {
-# written: manager
-# read:    loggerd, uploaderd, baseui
-  "DongleId": TxType.PERSISTANT,
-  "AccessToken": TxType.PERSISTANT,
-  "Version": TxType.PERSISTANT,
-  "GitCommit": TxType.PERSISTANT,
-  "GitBranch": TxType.PERSISTANT,
-  "GitRemote": TxType.PERSISTANT,
-# written: baseui
-# read:    ui, controls
-  "IsMetric": TxType.PERSISTANT,
-  "IsRearViewMirror": TxType.PERSISTANT,
-  "IsFcwEnabled": TxType.PERSISTANT,
-  "HasAcceptedTerms": TxType.PERSISTANT,
-  "IsUploadVideoOverCellularEnabled": TxType.PERSISTANT,
-# written: visiond
-# read:    visiond, controlsd
-  "CalibrationParams": TxType.PERSISTANT,
-# written: visiond
-# read:    visiond, ui
-  "CloudCalibration": TxType.PERSISTANT,
-# written: controlsd
-# read:    radard
-  "CarParams": TxType.CLEAR_ON_CAR_START,
-
-  "Passive": TxType.PERSISTANT,
-  "DoUninstall": TxType.CLEAR_ON_MANAGER_START,
-}
-
-def fsync_dir(path):
-  fd = os.open(path, os.O_RDONLY)
-  try:
-    os.fsync(fd)
-  finally:
-    os.close(fd)
-
-
-class FileLock(object):
-  def __init__(self, path, create):
-    self._path = path
-    self._create = create
-    self._fd = None
-
-  def acquire(self):
-    self._fd = os.open(self._path, os.O_CREAT if self._create else 0)
-    fcntl.flock(self._fd, fcntl.LOCK_EX)
-
-  def release(self):
-    if self._fd is not None:
-      os.close(self._fd)
-      self._fd = None
-
-
-class DBAccessor(object):
-  def __init__(self, path):
-    self._path = path
-    self._vals = None
-
-  def keys(self):
-    self._check_entered()
-    return self._vals.keys()
-
-  def get(self, key):
-    self._check_entered()
-    try:
-      return self._vals[key]
-    except KeyError:
-      return None
-
-  def _get_lock(self, create):
-    lock = FileLock(os.path.join(self._path, ".lock"), create)
-    lock.acquire()
-    return lock
-
-  def _read_values_locked(self):
-    """Callers should hold a lock while calling this method."""
-    vals = {}
-    try:
-      data_path = self._data_path()
-      keys = os.listdir(data_path)
-      for key in keys:
-        with open(os.path.join(data_path, key), "rb") as f:
-          vals[key] = f.read()
-    except (OSError, IOError) as e:
-      # Either the DB hasn't been created yet, or somebody wrote a bug and left the DB in an
-      # inconsistent state. Either way, return empty.
-      if e.errno == errno.ENOENT:
-        return {}
-
-    return vals
-
-  def _data_path(self):
-    return os.path.join(self._path, "d")
-
-  def _check_entered(self):
-    if self._vals is None:
-      raise Exception("Must call __enter__ before using DB")
-
-
-class DBReader(DBAccessor):
-  def __enter__(self):
-    try:
-      lock = self._get_lock(False)
-    except OSError as e:
-      # Do not create lock if it does not exist.
-      if e.errno == errno.ENOENT:
-        self._vals = {}
-        return self
-
-    try:
-      # Read everything.
-      self._vals = self._read_values_locked()
-      return self
-    finally:
-      lock.release()
-
-  def __exit__(self, type, value, traceback): pass
-
-
-class DBWriter(DBAccessor):
-  def __init__(self, path):
-    super(DBWriter, self).__init__(path)
-    self._lock = None
-    self._prev_umask = None
-
-  def put(self, key, value):
-    self._vals[key] = value
-
-  def delete(self, key):
-    self._vals.pop(key, None)
-
-  def __enter__(self):
-    mkdirs_exists_ok(self._path)
-
-    # Make sure we can write and that permissions are correct.
-    self._prev_umask = os.umask(0)
-
-    try:
-      os.chmod(self._path, 0o777)
-      self._lock = self._get_lock(True)
-      self._vals = self._read_values_locked()
-    except:
-      os.umask(self._prev_umask)
-      self._prev_umask = None
-      raise
-
-    return self
-
-  def __exit__(self, type, value, traceback):
-    self._check_entered()
-
-    try:
-      # data_path refers to the externally used path to the params. It is a symlink.
-      # old_data_path is the path currently pointed to by data_path.
-      # tempdir_path is a path where the new params will go, which the new data path will point to.
-      # new_data_path is a temporary symlink that will atomically overwrite data_path.
-      #
-      # The current situation is:
-      #   data_path -> old_data_path
-      # We're going to write params data to tempdir_path
-      #   tempdir_path -> params data
-      # Then point new_data_path to tempdir_path
-      #   new_data_path -> tempdir_path
-      # Then atomically overwrite data_path with new_data_path
-      #   data_path -> tempdir_path
-      old_data_path = None
-      new_data_path = None
-      tempdir_path = tempfile.mkdtemp(prefix=".tmp", dir=self._path)
-
-      try:
-        # Write back all keys.
-        os.chmod(tempdir_path, 0o777)
-        for k, v in self._vals.items():
-          with open(os.path.join(tempdir_path, k), "wb") as f:
-            f.write(v)
-            f.flush()
-            os.fsync(f.fileno())
-        fsync_dir(tempdir_path)
-
-        data_path = self._data_path()
-        try:
-          old_data_path = os.path.join(self._path, os.readlink(data_path))
-        except (OSError, IOError) as e:
-          # NOTE(mgraczyk): If other DB implementations have bugs, this could cause
-          #                 copies to be left behind, but we still want to overwrite.
-          pass
-
-        new_data_path = "{}.link".format(tempdir_path)
-        os.symlink(os.path.basename(tempdir_path), new_data_path)
-        os.rename(new_data_path, data_path)
-        fsync_dir(self._path)
-      finally:
-        # If the rename worked, we can delete the old data. Otherwise delete the new one.
-        success = new_data_path is not None and os.path.exists(data_path) and (
-          os.readlink(data_path) == os.path.basename(tempdir_path))
-
-        if success:
-          if old_data_path is not None:
-            shutil.rmtree(old_data_path)
-        else:
-          shutil.rmtree(tempdir_path)
-
-        # Regardless of what happened above, there should be no link at new_data_path.
-        if new_data_path is not None and os.path.islink(new_data_path):
-          os.remove(new_data_path)
-    finally:
-      os.umask(self._prev_umask)
-      self._prev_umask = None
-
-      # Always release the lock.
-      self._lock.release()
-      self._lock = None
-
-
-
-class JSDB(object):
-  def __init__(self, fn):
-    self._fn = fn
-
-  def begin(self, write=False):
-    if write:
-      return DBWriter(self._fn)
-    else:
-      return DBReader(self._fn)
-
-class Params(object):
-  def __init__(self, db='/data/params'):
-    self.env = JSDB(db)
-
-  def _clear_keys_with_type(self, tx_type):
-    with self.env.begin(write=True) as txn:
-      for key in keys:
-        if keys[key] == tx_type:
-          txn.delete(key)
-
-  def manager_start(self):
-    self._clear_keys_with_type(TxType.CLEAR_ON_MANAGER_START)
-
-  def car_start(self):
-    self._clear_keys_with_type(TxType.CLEAR_ON_CAR_START)
-
-  def delete(self, key):
-    with self.env.begin(write=True) as txn:
-      txn.delete(key)
-
-  def get(self, key, block=False):
-    if key not in keys:
-      raise UnknownKeyName(key)
-
-    while 1:
-      with self.env.begin() as txn:
-        ret = txn.get(key)
-      if not block or ret is not None:
-        break
-      # is polling really the best we can do?
-      time.sleep(0.05)
-    return ret
-
-  def put(self, key, dat):
-    if key not in keys:
-      raise UnknownKeyName(key)
-
-    with self.env.begin(write=True) as txn:
-      txn.put(key, dat)
-    print "set", key
+from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+assert Params
+assert ParamKeyFlag
+assert ParamKeyType
+assert UnknownKeyName
 
 if __name__ == "__main__":
-  params = Params()
-  if len(sys.argv) > 2:
-    params.put(sys.argv[1], sys.argv[2])
-  else:
-    for k in keys:
-      pp = params.get(k)
-      if pp is None:
-        print k, "is None"
-      elif all(ord(c) < 128 and ord(c) >= 32 for c in pp):
-        print k, pp
-      else:
-        print k, pp.encode("hex")
+  import sys
 
-  # Test multiprocess:
-  # seq 0 100000 | xargs -P20 -I{} python common/params.py DongleId {} && sleep 0.05
-  # while python common/params.py DongleId; do sleep 0.05; done
+  params = Params()
+  key = sys.argv[1]
+  assert params.check_key(key), f"unknown param: {key}"
+
+  if len(sys.argv) == 3:
+    val = sys.argv[2]
+    print(f"SET: {key} = {val}")
+    params.put(key, val)
+  elif len(sys.argv) == 2:
+    print(f"GET: {key} = {params.get(key)}")

common/params_keys.h

@@ -0,0 +1,163 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include "cereal/gen/cpp/log.capnp.h"
+
+inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
+    {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
+    {"AdbEnabled", {PERSISTENT, BOOL}},
+    {"AlwaysOnDM", {PERSISTENT, BOOL}},
+    {"ApiCache_Device", {PERSISTENT, STRING}},
+    {"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
+    {"AssistNowToken", {PERSISTENT, STRING}},
+    {"AthenadPid", {PERSISTENT, INT}},
+    {"AthenadUploadQueue", {PERSISTENT, JSON}},
+    {"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
+    {"BootCount", {PERSISTENT, INT}},
+    {"CalibrationParams", {PERSISTENT, BYTES}},
+    {"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
+    {"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
+    {"CarBatteryCapacity", {PERSISTENT, INT}},
+    {"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
+    {"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"CarParamsPersistent", {PERSISTENT, BYTES}},
+    {"CarParamsPrevRoute", {PERSISTENT, BYTES}},
+    {"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
+    {"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"CurrentBootlog", {PERSISTENT, STRING}},
+    {"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"DisableLogging", {PERSISTENT, BOOL, "0"}},
+    {"DisablePowerDown", {PERSISTENT, BOOL}},
+    {"DisableUpdates", {PERSISTENT, BOOL}},
+    {"DisengageOnAccelerator", {PERSISTENT, BOOL, "0"}},
+    {"DongleId", {PERSISTENT, STRING}},
+    {"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY, BOOL}},
+    {"ExperimentalMode", {PERSISTENT, BOOL}},
+    {"ExperimentalModeConfirmed", {PERSISTENT, BOOL}},
+    {"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"ForcePowerDown", {PERSISTENT, BOOL}},
+    {"GitBranch", {PERSISTENT, STRING}},
+    {"GitCommit", {PERSISTENT, STRING}},
+    {"GitCommitDate", {PERSISTENT, STRING}},
+    {"GitDiff", {PERSISTENT, STRING}},
+    {"GithubSshKeys", {PERSISTENT, STRING}},
+    {"GithubUsername", {PERSISTENT, STRING}},
+    {"GitRemote", {PERSISTENT, STRING}},
+    {"GsmApn", {PERSISTENT, STRING}},
+    {"GsmMetered", {PERSISTENT, BOOL, "1"}},
+    {"GsmRoaming", {PERSISTENT, BOOL}},
+    {"HardwareSerial", {PERSISTENT, STRING}},
+    {"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
+    {"InstallDate", {PERSISTENT, TIME}},
+    {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsEngaged", {PERSISTENT, BOOL}},
+    {"IsLdwEnabled", {PERSISTENT, BOOL}},
+    {"IsMetric", {PERSISTENT, BOOL}},
+    {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsOnroad", {PERSISTENT, BOOL}},
+    {"IsRhdDetected", {PERSISTENT, BOOL}},
+    {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LanguageSetting", {PERSISTENT, STRING, "en"}},
+    {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
+    {"LastGPSPosition", {PERSISTENT, STRING}},
+    {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
+    {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
+    {"LastUpdateTime", {PERSISTENT, TIME}},
+    {"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"LiveDelay", {PERSISTENT, BYTES}},
+    {"LiveParameters", {PERSISTENT, JSON}},
+    {"LiveParametersV2", {PERSISTENT, BYTES}},
+    {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
+    {"LocationFilterInitialState", {PERSISTENT, BYTES}},
+    {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LongitudinalPersonality", {PERSISTENT, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
+    {"NetworkMetered", {PERSISTENT, BOOL}},
+    {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
+    {"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
+    {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"OpenpilotEnabledToggle", {PERSISTENT, BOOL, "1"}},
+    {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"PrimeType", {PERSISTENT, INT}},
+    {"RecordAudio", {PERSISTENT, BOOL}},
+    {"RecordAudioFeedback", {PERSISTENT, BOOL, "0"}},
+    {"RecordFront", {PERSISTENT, BOOL}},
+    {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
+    {"SecOCKey", {PERSISTENT | DONT_LOG, STRING}},
+    {"RouteCount", {PERSISTENT, INT, "0"}},
+    {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"SshEnabled", {PERSISTENT, BOOL}},
+    {"TermsVersion", {PERSISTENT, STRING}},
+    {"TrainingVersion", {PERSISTENT, STRING}},
+    {"UbloxAvailable", {PERSISTENT, BOOL}},
+    {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
+    {"UpdaterAvailableBranches", {PERSISTENT, STRING}},
+    {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
+    {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"Version", {PERSISTENT, STRING}},
+    {"dp_dev_last_log", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"dp_dev_reset_conf", {CLEAR_ON_MANAGER_START, BOOL, "0"}},
+    {"dp_dev_is_rhd", {PERSISTENT, BOOL, "0"}},
+    {"dp_dev_monitoring_disabled", {PERSISTENT, BOOL, "0"}},
+    {"dp_dev_beep", {PERSISTENT, BOOL, "0"}},
+    {"dp_lat_alka", {PERSISTENT, BOOL, "0"}},
+    {"dp_ui_display_mode", {PERSISTENT, INT, "0"}},
+    {"dp_dev_model_selected", {PERSISTENT, STRING}},
+    {"dp_dev_model_list", {PERSISTENT, STRING}},
+    {"dp_lat_lca_speed", {PERSISTENT, INT, "20"}},
+    {"dp_lat_lca_auto_sec", {PERSISTENT, FLOAT, "0.0"}},
+    {"dp_dev_go_off_road", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"dp_ui_hide_hud_speed_kph", {PERSISTENT, INT, "0"}},
+    {"dp_lon_ext_radar", {PERSISTENT, BOOL, "0"}},
+    {"dp_lat_road_edge_detection", {PERSISTENT, BOOL, "0"}},
+    {"dp_ui_rainbow", {PERSISTENT, BOOL, "0"}},
+    {"dp_lon_acm", {PERSISTENT, BOOL, "0"}},
+    {"dp_lon_aem", {PERSISTENT, BOOL, "0"}},
+    {"dp_lon_dtsc", {PERSISTENT, BOOL, "0"}},
+    {"dp_dev_audible_alert_mode", {PERSISTENT, INT, "0"}},
+    {"dp_dev_auto_shutdown_in", {PERSISTENT, INT, "-5"}},
+    {"dp_ui_lead", {PERSISTENT, INT, "0"}},
+    {"dp_dev_dashy", {PERSISTENT, INT, "0"}},
+    {"dp_dev_delay_loggerd", {PERSISTENT, INT, "0"}},
+    {"dp_dev_disable_connect", {PERSISTENT, BOOL, "0"}},
+    {"dp_dev_tethering", {PERSISTENT, BOOL, "0"}},
+    {"dp_toyota_door_auto_lock_unlock", {PERSISTENT, BOOL, "0"}},
+    {"dp_toyota_tss1_sng", {PERSISTENT, BOOL, "0"}},
+    {"dp_toyota_stock_lon", {PERSISTENT, BOOL, "0"}},
+    {"dp_vag_a0_sng", {PERSISTENT, BOOL, "0"}},
+    {"dp_vag_pq_steering_patch", {PERSISTENT, BOOL, "0"}},
+    {"dp_vag_avoid_eps_lockout", {PERSISTENT, BOOL, "0"}},
+};

common/params_pyx.pyx

@@ -0,0 +1,196 @@
+# distutils: language = c++
+# cython: language_level = 3
+import builtins
+import datetime
+import json
+from libcpp cimport bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.optional cimport optional
+
+from openpilot.common.swaglog import cloudlog
+
+cdef extern from "common/params.h":
+  cpdef enum ParamKeyFlag:
+    PERSISTENT
+    CLEAR_ON_MANAGER_START
+    CLEAR_ON_ONROAD_TRANSITION
+    CLEAR_ON_OFFROAD_TRANSITION
+    DEVELOPMENT_ONLY
+    CLEAR_ON_IGNITION_ON
+    ALL
+
+  cpdef enum ParamKeyType:
+    STRING
+    BOOL
+    INT
+    FLOAT
+    TIME
+    JSON
+    BYTES
+
+  cdef cppclass c_Params "Params":
+    c_Params(string) except + nogil
+    string get(string, bool) nogil
+    bool getBool(string, bool) nogil
+    int remove(string) nogil
+    int put(string, string) nogil
+    void putNonBlocking(string, string) nogil
+    void putBoolNonBlocking(string, bool) nogil
+    int putBool(string, bool) nogil
+    bool checkKey(string) nogil
+    ParamKeyType getKeyType(string) nogil
+    optional[string] getKeyDefaultValue(string) nogil
+    string getParamPath(string) nogil
+    void clearAll(ParamKeyFlag)
+    vector[string] allKeys()
+
+PYTHON_2_CPP = {
+  (str, STRING): lambda v: v,
+  (builtins.bool, BOOL): lambda v: "1" if v else "0",
+  (int, INT): str,
+  (float, FLOAT): str,
+  (datetime.datetime, TIME): lambda v: v.isoformat(),
+  (dict, JSON): json.dumps,
+  (list, JSON): json.dumps,
+  (bytes, BYTES): lambda v: v,
+}
+CPP_2_PYTHON = {
+  STRING: lambda v: v.decode("utf-8"),
+  BOOL: lambda v: v == b"1",
+  INT: int,
+  FLOAT: float,
+  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
+  JSON: json.loads,
+  BYTES: lambda v: v,
+}
+
+def ensure_bytes(v):
+  return v.encode() if isinstance(v, str) else v
+
+class UnknownKeyName(Exception):
+  pass
+
+cdef class Params:
+  cdef c_Params* p
+  cdef str d
+
+  def __cinit__(self, d=""):
+    cdef string path = <string>d.encode()
+    with nogil:
+      self.p = new c_Params(path)
+    self.d = d
+
+  def __reduce__(self):
+    return (type(self), (self.d,))
+
+  def __dealloc__(self):
+    del self.p
+
+  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
+    self.p.clearAll(tx_flag)
+
+  def check_key(self, key):
+    key = ensure_bytes(key)
+    if not self.p.checkKey(key):
+      raise UnknownKeyName(key)
+    return key
+
+  def python2cpp(self, proposed_type, expected_type, value, key):
+    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
+    if cast:
+      return cast(value)
+    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
+
+  def _cpp2python(self, t, value, default, key):
+    if value is None:
+      return None
+    try:
+      return CPP_2_PYTHON[t](value)
+    except (KeyError, TypeError, ValueError):
+      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
+      return self._cpp2python(t, default, None, key)
+
+  def get(self, key, bool block=False, bool return_default=False):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    cdef string val
+    with nogil:
+      val = self.p.get(k, block)
+
+    default_val = (default.value() if default.has_value() else None) if return_default else None
+    if val == b"":
+      if block:
+        # If we got no value while running in blocked mode
+        # it means we got an interrupt while waiting
+        raise KeyboardInterrupt
+      else:
+        return self._cpp2python(t, default_val, None, key)
+    return self._cpp2python(t, val, default_val, key)
+
+  def get_bool(self, key, bool block=False):
+    cdef string k = self.check_key(key)
+    cdef bool r
+    with nogil:
+      r = self.p.getBool(k, block)
+    return r
+
+  def _put_cast(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
+
+  def put(self, key, dat):
+    """
+    Warning: This function blocks until the param is written to disk!
+    In very rare cases this can take over a second, and your code will hang.
+    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
+    in general try to avoid writing params as much as possible.
+    """
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = self._put_cast(key, dat)
+    with nogil:
+      self.p.put(k, dat_bytes)
+
+  def put_bool(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBool(k, val)
+
+  def put_nonblocking(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = self._put_cast(key, dat)
+    with nogil:
+      self.p.putNonBlocking(k, dat_bytes)
+
+  def put_bool_nonblocking(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBoolNonBlocking(k, val)
+
+  def remove(self, key):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.remove(k)
+
+  def get_param_path(self, key=""):
+    cdef string key_bytes = ensure_bytes(key)
+    return self.p.getParamPath(key_bytes).decode("utf-8")
+
+  def get_type(self, key):
+    return self.p.getKeyType(self.check_key(key))
+
+  def all_keys(self):
+    return self.p.allKeys()
+
+  def get_default_value(self, key):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
+
+  def cpp2python(self, key, value):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return self._cpp2python(t, value, None, key)

common/pid.py

@@ -0,0 +1,64 @@
+import numpy as np
+from numbers import Number
+
+class PIDController:
+  def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p = k_p
+    self._k_i = k_i
+    self._k_d = k_d
+    self.k_f = k_f   # feedforward gain
+    if isinstance(self._k_p, Number):
+      self._k_p = [[0], [self._k_p]]
+    if isinstance(self._k_i, Number):
+      self._k_i = [[0], [self._k_i]]
+    if isinstance(self._k_d, Number):
+      self._k_d = [[0], [self._k_d]]
+
+    self.set_limits(pos_limit, neg_limit)
+
+    self.i_dt = 1.0 / rate
+    self.speed = 0.0
+
+    self.reset()
+
+  @property
+  def k_p(self):
+    return np.interp(self.speed, self._k_p[0], self._k_p[1])
+
+  @property
+  def k_i(self):
+    return np.interp(self.speed, self._k_i[0], self._k_i[1])
+
+  @property
+  def k_d(self):
+    return np.interp(self.speed, self._k_d[0], self._k_d[1])
+
+  def reset(self):
+    self.p = 0.0
+    self.i = 0.0
+    self.d = 0.0
+    self.f = 0.0
+    self.control = 0
+
+  def set_limits(self, pos_limit, neg_limit):
+    self.pos_limit = pos_limit
+    self.neg_limit = neg_limit
+
+  def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
+    self.speed = speed
+    self.p = self.k_p * float(error)
+    self.d = self.k_d * error_rate
+    self.f = self.k_f * feedforward
+
+    if not freeze_integrator:
+      i = self.i + self.k_i * self.i_dt * error
+
+      # Don't allow windup if already clipping
+      test_control = self.p + i + self.d + self.f
+      i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
+      i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
+      self.i = np.clip(i, i_lowerbound, i_upperbound)
+
+    control = self.p + self.i + self.d + self.f
+    self.control = np.clip(control, self.neg_limit, self.pos_limit)
+    return self.control

common/prefix.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cassert>
+#include <string>
+
+#include "common/params.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+
+class OpenpilotPrefix {
+public:
+  OpenpilotPrefix(std::string prefix = {}) {
+    if (prefix.empty()) {
+      prefix = util::random_string(15);
+    }
+    msgq_path = Path::shm_path() + "/" + prefix;
+    bool ret = util::create_directories(msgq_path, 0777);
+    assert(ret);
+    setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
+  }
+
+  ~OpenpilotPrefix() {
+    auto param_path = Params().getParamPath();
+    if (util::file_exists(param_path)) {
+      std::string real_path = util::readlink(param_path);
+      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
+      unlink(param_path.c_str());
+    }
+    if (getenv("COMMA_CACHE") == nullptr) {
+      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
+    }
+    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
+    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
+    unsetenv("OPENPILOT_PREFIX");
+  }
+
+private:
+  std::string msgq_path;
+};

common/prefix.py

@@ -0,0 +1,59 @@
+import os
+import shutil
+import uuid
+
+
+from openpilot.common.params import Params
+from openpilot.system.hardware import PC
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+
+class OpenpilotPrefix:
+  def __init__(self, prefix: str = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+    self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
+    self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
+    self.create_dirs_on_enter = create_dirs_on_enter
+    self.clean_dirs_on_exit = clean_dirs_on_exit
+    self.shared_download_cache = shared_download_cache
+
+  def __enter__(self):
+    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
+    os.environ['OPENPILOT_PREFIX'] = self.prefix
+
+    if self.create_dirs_on_enter:
+      self.create_dirs()
+
+    if self.shared_download_cache:
+      os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
+
+    return self
+
+  def __exit__(self, exc_type, exc_obj, exc_tb):
+    if self.clean_dirs_on_exit:
+      self.clean_dirs()
+    try:
+      del os.environ['OPENPILOT_PREFIX']
+      if self.original_prefix is not None:
+        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
+    except KeyError:
+      pass
+    return False
+
+  def create_dirs(self):
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
+
+  def clean_dirs(self):
+    symlink_path = Params().get_param_path()
+    if os.path.exists(symlink_path):
+      shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
+      os.remove(symlink_path)
+    shutil.rmtree(self.msgq_path, ignore_errors=True)
+    if PC:
+      shutil.rmtree(Paths.log_root(), ignore_errors=True)
+    if not os.environ.get("COMMA_CACHE", False):
+      shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
+    shutil.rmtree(Paths.comma_home(), ignore_errors=True)

common/profiler.py

@@ -1,47 +0,0 @@
-import time
-
-class Profiler(object):
-  def __init__(self, enabled=False):
-    self.enabled = enabled
-    self.cp = {}
-    self.cp_ignored = []
-    self.iter = 0
-    self.start_time = time.clock()
-    self.last_time = self.start_time
-    self.tot = 0.
-
-  def reset(self, enabled=False):
-    self.enabled = enabled
-    self.cp = {}
-    self.cp_ignored = []
-    self.iter = 0
-    self.start_time = time.clock()
-    self.last_time = self.start_time
-
-  def checkpoint(self, name, ignore=False):
-    # ignore flag needed when benchmarking threads with ratekeeper
-    if not self.enabled:
-      return
-    tt = time.clock()
-    if name not in self.cp:
-      self.cp[name] = 0.
-      if ignore:
-        self.cp_ignored.append(name)
-    self.cp[name] += tt - self.last_time
-    if not ignore:
-      self.tot += tt - self.last_time
-    self.last_time = tt
-
-  def display(self):
-    if not self.enabled:
-      return
-    self.iter += 1
-    print "******* Profiling *******"
-    for n in self.cp:
-      ms = self.cp[n]
-      if n in self.cp_ignored:
-        print "%30s: %7.2f   perc: %1.0f" % (n, ms*1000.0, ms/self.tot*100), "  IGNORED"
-      else:
-        print "%30s: %7.2f   perc: %1.0f" % (n, ms*1000.0, ms/self.tot*100)
-    print "Iter clock: %2.6f   TOTAL: %2.2f" % (self.tot/self.iter, self.tot)
-

common/queue.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+template <class T>
+class SafeQueue {
+public:
+  SafeQueue() = default;
+
+  void push(const T& v) {
+    {
+      std::unique_lock lk(m);
+      q.push(v);
+    }
+    cv.notify_one();
+  }
+
+  T pop() {
+    std::unique_lock lk(m);
+    cv.wait(lk, [this] { return !q.empty(); });
+    T v = q.front();
+    q.pop();
+    return v;
+  }
+
+  bool try_pop(T& v, int timeout_ms = 0) {
+    std::unique_lock lk(m);
+    if (!cv.wait_for(lk, std::chrono::milliseconds(timeout_ms), [this] { return !q.empty(); })) {
+      return false;
+    }
+    v = q.front();
+    q.pop();
+    return true;
+  }
+
+  bool empty() const {
+    std::scoped_lock lk(m);
+    return q.empty();
+  }
+
+  size_t size() const {
+    std::scoped_lock lk(m);
+    return q.size();
+  }
+
+private:
+  mutable std::mutex m;
+  std::condition_variable cv;
+  std::queue<T> q;
+};

common/ratekeeper.cc

@@ -0,0 +1,40 @@
+#include "common/ratekeeper.h"
+
+#include <algorithm>
+
+#include "common/swaglog.h"
+#include "common/timing.h"
+#include "common/util.h"
+
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
+    : name(name),
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
+  interval = 1 / rate;
+  last_monitor_time = seconds_since_boot();
+  next_frame_time = last_monitor_time + interval;
+}
+
+bool RateKeeper::keepTime() {
+  bool lagged = monitorTime();
+  if (remaining_ > 0) {
+    util::sleep_for(remaining_ * 1000);
+  }
+  return lagged;
+}
+
+bool RateKeeper::monitorTime() {
+  ++frame_;
+  last_monitor_time = seconds_since_boot();
+  remaining_ = next_frame_time - last_monitor_time;
+
+  bool lagged = remaining_ < 0;
+  if (lagged) {
+    if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
+      LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
+    }
+    next_frame_time = last_monitor_time + interval;
+  } else {
+    next_frame_time += interval;
+  }
+  return lagged;
+}