fix tested branch detection

This reverts commit 9ff1fb48de.

.gitignore

@@ -1,11 +1,16 @@
+venv/
 .DS_Store
 .tags
 .ipynb_checkpoints
 .idea
+.overlay_init
+.overlay_consistent
 .sconsign.dblite
+.vscode
 model2.png
 a.out
 
+*.dylib
 *.DSYM
 *.d
 *.pyc
@@ -13,23 +18,51 @@ a.out
 .*.swp
 .*.swo
 .*.un~
+*.tmp
 *.o
+*.o-*
+*.os
+*.os-*
 *.so
 *.a
 *.clb
 *.class
 *.pyxbldc
 *.vcd
-lane.cpp
-loc*.cpp
 config.json
 clcache
 
+persist
 board/obj/
 selfdrive/boardd/boardd
 selfdrive/logcatd/logcatd
+selfdrive/mapd/default_speeds_by_region.json
 selfdrive/proclogd/proclogd
-selfdrive/ui/ui
+selfdrive/ui/_ui
+selfdrive/test/longitudinal_maneuvers/out
+selfdrive/visiond/visiond
+selfdrive/loggerd/loggerd
+selfdrive/sensord/_gpsd
+selfdrive/sensord/_sensord
+selfdrive/camerad/camerad
+selfdrive/modeld/_modeld
+selfdrive/modeld/_dmonitoringmodeld
 /src/
 
 one
+openpilot
+notebooks
+xx
+panda_jungle
+apks
+openpilot-apks
+
+.coverage*
+coverage.xml
+htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt

.travis.yml

@@ -1,12 +0,0 @@
-sudo: required
-
-services:
-  - docker
-
-install:
-  - docker build -t tmppilot -f Dockerfile.openpilot .
-
-script:
-  - docker run --rm
-      -v "$(pwd)"/selfdrive/test/tests/plant/out:/tmp/openpilot/selfdrive/test/tests/plant/out
-      tmppilot /bin/sh -c 'cd /tmp/openpilot/selfdrive/test/tests/plant && OPTEST=1 ./test_longitudinal.py'

CONTRIBUTING.md

@@ -2,17 +2,50 @@
 
 Our software is open source so you can solve your own problems without needing help from others. And if you solve a problem and are so kind, you can upstream it for the rest of the world to use.
 
-Most open source development activity is coordinated through our [slack](https://slack.comma.ai). A lot of documentation is available on our [medium](https://medium.com/@comma_ai/)
+Most open source development activity is coordinated through our [Discord](https://discord.comma.ai). A lot of documentation is available on our [medium](https://medium.com/@comma_ai/).
 
 ## Getting Started
 
- * Join our slack [slack.comma.ai](https://slack.comma.ai)
+ * Join our [Discord](https://discord.comma.ai)
  * Make sure you have a [GitHub account](https://github.com/signup/free)
- * Fork the repository on GitHub
+ * Fork [our repositories](https://github.com/commaai) on GitHub
+
+## Testing
+
+### Local Testing
+
+You can test your changes on your machine by running `run_docker_tests.sh`. This will run some automated tests in docker against your code.
+
+### Automated Testing
+
+All PRs and commits are automatically checked by Github Actions. Check out `.github/workflows/` for what Github Actions runs. Any new tests sould be added to Github Actions.
+
+### Code Style and Linting
+
+Code is automatically checked for style by Github Actions as part of the automated tests. You can also run these tests yourself by running `pre-commit run --all`.
 
 ## Car Ports (openpilot)
 
-We've released a guide for porting to Toyota cars [here](https://medium.com/@comma_ai/openpilot-port-guide-for-toyota-models-e5467f4b5fe6)
+We've released a [Model Port guide](https://medium.com/@comma_ai/openpilot-port-guide-for-toyota-models-e5467f4b5fe6) for porting to Toyota/Lexus models.
 
-If you port openpilot to a substantially new car, you might be eligible for a bounty. See our bounties at [comma.ai/bounties.html](https://comma.ai/bounties.html)
+If you port openpilot to a substantially new car brand, see this more generic [Brand Port guide](https://medium.com/@comma_ai/how-to-write-a-car-port-for-openpilot-7ce0785eda84). You might also be eligible for a bounty. See our bounties at [comma.ai/bounties.html](https://comma.ai/bounties.html)
 
+## Pull Requests
+
+Pull requests should be against the master branch. Before running master on in-car hardware, you'll need to clone the submodules too. That can be done by recursively cloning the repository:
+```
+git clone https://github.com/commaai/openpilot.git --recursive
+```
+Or alternatively, when on the master branch:
+```
+git submodule init
+git submodule update
+```
+The reasons for having submodules on a dedicated repository and our new development philosophy can be found in our [post about externalization](https://medium.com/@comma_ai/a-2020-theme-externalization-13b33326d8b3).
+Modules that are in seperate repositories include:
+* apks
+* cereal
+* laika
+* opendbc
+* panda
+* rednose

Dockerfile.openpilot

@@ -1,19 +0,0 @@
-FROM ubuntu:16.04
-ENV PYTHONUNBUFFERED 1
-
-RUN apt-get update && apt-get install -y build-essential clang vim screen wget bzip2 git libglib2.0-0 python-pip capnproto libcapnp-dev libzmq5-dev libffi-dev libusb-1.0-0
-RUN pip install numpy==1.11.2 scipy==0.18.1 matplotlib
-
-COPY requirements_openpilot.txt /tmp/
-RUN pip install -r /tmp/requirements_openpilot.txt
-
-ENV PYTHONPATH /tmp/openpilot:$PYTHONPATH
-
-COPY ./common /tmp/openpilot/common
-COPY ./cereal /tmp/openpilot/cereal
-COPY ./opendbc /tmp/openpilot/opendbc
-COPY ./selfdrive /tmp/openpilot/selfdrive
-COPY ./phonelibs /tmp/openpilot/phonelibs
-COPY ./pyextra /tmp/openpilot/pyextra
-
-RUN mkdir -p /tmp/openpilot/selfdrive/test/out

Jenkinsfile

@@ -0,0 +1,89 @@
+pipeline {
+  agent {
+    docker {
+      image 'python:3.7.3'
+      args '--user=root'
+    }
+  }
+  environment {
+    COMMA_JWT = credentials('athena-test-jwt')
+  }
+
+  stages {
+
+    stage('Release Build') {
+      when {
+        branch 'devel-staging'
+      }
+      steps {
+        lock(resource: "", label: 'eon-build', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
+          timeout(time: 60, unit: 'MINUTES') {
+            dir(path: 'selfdrive/test') {
+              sh 'pip install paramiko'
+              sh 'python phone_ci.py "cd release && PUSH=1 ./build_release2.sh"'
+            }
+          }
+        }
+      }
+    }
+
+    stage('On-device Tests') {
+      when {
+        not {
+          anyOf {
+            branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging'
+          }
+        }
+      }
+
+      parallel {
+
+        stage('Build') {
+          environment {
+            CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
+          }
+
+          steps {
+            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
+              timeout(time: 60, unit: 'MINUTES') {
+                dir(path: 'selfdrive/test') {
+                  sh 'pip install paramiko'
+                  sh 'python phone_ci.py "cd release && ./build_devel.sh"'
+                }
+              }
+            }
+          }
+        }
+
+        stage('Replay Tests') {
+          steps {
+            lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
+              timeout(time: 60, unit: 'MINUTES') {
+                dir(path: 'selfdrive/test') {
+                  sh 'pip install paramiko'
+                  sh 'python phone_ci.py "cd selfdrive/test/process_replay && ./camera_replay.py"'
+                }
+              }
+            }
+          }
+        }
+
+        stage('HW Tests') {
+          steps {
+            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
+              timeout(time: 60, unit: 'MINUTES') {
+                dir(path: 'selfdrive/test') {
+                  sh 'pip install paramiko'
+                  sh 'python phone_ci.py "SCONS_CACHE=1 scons -j3 cereal/ && \
+                                          nosetests -s selfdrive/test/test_sounds.py && \
+                                          nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"'
+                }
+              }
+            }
+          }
+        }
+
+      }
+    }
+  }
+}

Makefile

@@ -1,9 +0,0 @@
-
-code_dir := $(shell pwd)
-
-# TODO: Add a global build system
-
-.PHONY: all
-all:
-	cd selfdrive && PYTHONPATH=$(code_dir) PREPAREONLY=1 ./manager.py
-

README.md

@@ -1,133 +1,307 @@
-[![](https://i.imgur.com/RTQYufz.jpg)](#)
+[![](https://i.imgur.com/UelUjKAh.png)](#)
 
-Welcome to openpilot
-======
+Table of Contents
+=======================
 
-[openpilot](http://github.com/commaai/openpilot) is an open source driving agent. Currently it performs the functions of Adaptive Cruise Control (ACC) and Lane Keeping Assist System (LKAS) for Hondas, Acuras, Toyotas, and a Chevy. It's about on par with Tesla Autopilot and GM Super Cruise, and better than [all other manufacturers](http://www.thedrive.com/tech/5707/the-war-for-autonomous-driving-part-iii-us-vs-germany-vs-japan).
+* [What is openpilot?](#what-is-openpilot)
+* [Integration with Stock Features](#integration-with-stock-features)
+* [Supported Hardware](#supported-hardware)
+* [Supported Cars](#supported-cars)
+* [Community Maintained Cars and Features](#community-maintained-cars-and-features)
+* [Installation Instructions](#installation-instructions)
+* [Limitations of openpilot ALC and LDW](#limitations-of-openpilot-alc-and-ldw)
+* [Limitations of openpilot ACC and FCW](#limitations-of-openpilot-acc-and-fcw)
+* [Limitations of openpilot DM](#limitations-of-openpilot-dm)
+* [User Data and comma Account](#user-data-and-comma-account)
+* [Safety and Testing](#safety-and-testing)
+* [Testing on PC](#testing-on-pc)
+* [Community and Contributing](#community-and-contributing)
+* [Directory Structure](#directory-structure)
+* [Licensing](#licensing)
 
-The openpilot codebase has been written to be concise and enable rapid prototyping. We look forward to your contributions - improving real vehicle automation has never been easier.
+---
 
-Community
+What is openpilot?
 ------
 
-openpilot is supported by [comma.ai](https://comma.ai/).
-
-We have a [Twitter you should follow](https://twitter.com/comma_ai).
-
-Also, we have a 3500+ person [community on slack](https://slack.comma.ai).
+[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW) and Lane Departure Warning (LDW) for a growing variety of supported [car makes, models and model years](#supported-cars). In addition, while openpilot is engaged, a camera based Driver Monitoring (DM) feature alerts distracted and asleep drivers.
 
 <table>
   <tr>
-    <td><a href="https://www.youtube.com/watch?v=9TDi0BHgXyo" title="YouTube" rel="noopener"><img src="https://i.imgur.com/gBTo7yB.png"></a></td>
-    <td><a href="https://www.youtube.com/watch?v=1zCtj3ckGFo" title="YouTube" rel="noopener"><img src="https://i.imgur.com/gNhhcep.png"></a></td>
-    <td><a href="https://www.youtube.com/watch?v=Qd2mjkBIRx0" title="YouTube" rel="noopener"><img src="https://i.imgur.com/tFnSexp.png"></a></td>
-    <td><a href="https://www.youtube.com/watch?v=ju12vlBm59E" title="YouTube" rel="noopener"><img src="https://i.imgur.com/3BKiJVy.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=mgAbfr42oI8" title="YouTube" rel="noopener"><img src="https://i.imgur.com/kAtT6Ei.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=394rJKeh76k" title="YouTube" rel="noopener"><img src="https://i.imgur.com/lTt8cS2.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=1iNOc3cq8cs" title="YouTube" rel="noopener"><img src="https://i.imgur.com/ANnuSpe.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=Vr6NgrB-zHw" title="YouTube" rel="noopener"><img src="https://i.imgur.com/Qypanuq.png"></a></td>
   </tr>
   <tr>
-    <td><a href="https://www.youtube.com/watch?v=Z5VY5FzgNt4" title="YouTube" rel="noopener"><img src="https://i.imgur.com/3I9XOK2.png"></a></td>
-    <td><a href="https://www.youtube.com/watch?v=blnhZC7OmMg" title="YouTube" rel="noopener"><img src="https://i.imgur.com/f9IgX6s.png"></a></td>
-    <td><a href="https://www.youtube.com/watch?v=iRkz7FuJsA8" title="YouTube" rel="noopener"><img src="https://i.imgur.com/Vo5Zvmn.png"></a></td>
-    <td><a href="https://www.youtube.com/watch?v=IHjEqAKDqjM" title="YouTube" rel="noopener"><img src="https://i.imgur.com/V9Zd81n.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=Ug41KIKF0oo" title="YouTube" rel="noopener"><img src="https://i.imgur.com/3caZ7xM.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=NVR_CdG1FRg" title="YouTube" rel="noopener"><img src="https://i.imgur.com/bAZOwql.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=tkEvIdzdfUE" title="YouTube" rel="noopener"><img src="https://i.imgur.com/EFINEzG.png"></a></td>
+    <td><a href="https://www.youtube.com/watch?v=_P-N1ewNne4" title="YouTube" rel="noopener"><img src="https://i.imgur.com/gAyAq22.png"></a></td>
   </tr>
 </table>
 
-Hardware
+Integration with Stock Features
 ------
 
-Right now openpilot supports the [EON Dashcam DevKit](https://comma.ai/shop/products/eon-dashcam-devkit). We'd like to support other platforms as well.
+In all supported cars:
+* Stock Lane Keep Assist (LKA) and stock ALC are replaced by openpilot ALC, which only functions when openpilot is engaged by the user.
+* Stock LDW is replaced by openpilot LDW.
 
-Install openpilot on a neo device by entering ``https://openpilot.comma.ai`` during NEOS setup.
+Additionally, on specific supported cars (see ACC column in [supported cars](#supported-cars)):
+* Stock ACC is replaced by openpilot ACC.
+* openpilot FCW operates in addition to stock FCW.
+
+openpilot should preserve all other vehicle's stock features, including, but are not limited to: FCW, Automatic Emergency Braking (AEB), auto high-beam, blind spot warning, and side collision warning.
+
+Supported Hardware
+------
+
+At the moment, openpilot supports the [EON DevKit](https://comma.ai/shop/products/eon-dashcam-devkit) and the [comma two](https://comma.ai/shop/products/comma-two-devkit). A [car harness](https://comma.ai/shop/products/car-harness) is recommended to connect the EON or comma two to the car. For experimental purposes, openpilot can also run on an Ubuntu computer with external [webcams](https://github.com/commaai/openpilot/tree/master/tools/webcam).
 
 Supported Cars
 ------
 
-| Make    | Model                  | Supported Package    | Lateral | Longitudinal | No Accel Below  | No Steer Below |
-| ------- | ---------------------- | -------------------- | ------- | ------------ | --------------  | -------------- |
-| Acura   | ILX 2016               | AcuraWatch Plus      | Yes     | Yes          | 25mph*          | 12mph          |
-| Acura   | ILX 2017               | AcuraWatch Plus      | Yes     | Yes          | 25mph*          | 12mph          |
-| Acura   | RDX 2018               | AcuraWatch Plus      | Yes     | Yes          | 25mph*          | 12mph          |
-| GM      | Volt 2017              | Driver Confidence II | Yes     | Yes          | 0mph            | 0mph           |
-| GM      | Volt 2018              | Driver Confidence II | Yes     | Yes          | 0mph            | 0mph           |
-| Honda   | Accord 2018            | All                  | Yes     | Stock        | 0mph            | 3mph           |
-| Honda   | Civic 2016 *(Sedan)*   | Honda Sensing        | Yes     | Yes          | 0mph            | 12mph          |
-| Honda   | Civic 2017 *(Sedan)*   | Honda Sensing        | Yes     | Yes          | 0mph            | 12mph          |
-| Honda   | Civic 2017 *(Hatch)*   | Honda Sensing        | Yes     | Stock        | 0mph            | 12mph          |
-| Honda   | Civic 2018 *(Sedan)*   | Honda Sensing        | Yes     | Yes          | 0mph            | 12mph          |
-| Honda   | CR-V 2015              | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | CR-V 2016              | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | CR-V 2017              | Honda Sensing        | Yes     | Stock        | 0mph            | 12mph          |
-| Honda   | CR-V 2018              | Honda Sensing        | Yes     | Stock        | 0mph            | 12mph          |
-| Honda   | Odyssey 2017           | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | Odyssey 2018           | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | Pilot 2017             | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | Pilot 2018             | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | Ridgeline 2017         | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Honda   | Ridgeline 2018         | Honda Sensing        | Yes     | Yes          | 25mph*          | 12mph          |
-| Lexus   | RX Hybrid 2017         | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Lexus   | RX Hybrid 2018         | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Corolla 2017           | All                  | Yes     | Yes          | 20mph           | 0mph           |
-| Toyota  | Corolla 2018           | All                  | Yes     | Yes          | 20mph           | 0mph           |
-| Toyota  | Prius 2016             | TSS-P                | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Prius 2017             | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Prius 2018             | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Prius Prime 2017       | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Prius Prime 2018       | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Rav4 2016              | TSS-P                | Yes     | Yes          | 20mph           | 0mph           |
-| Toyota  | Rav4 2017              | All                  | Yes     | Yes          | 20mph           | 0mph           |
-| Toyota  | Rav4 2018              | All                  | Yes     | Yes          | 20mph           | 0mph           |
-| Toyota  | Rav4 Hybrid 2017       | All                  | Yes     | Yes          | 0mph            | 0mph           |
-| Toyota  | Rav4 Hybrid 2018       | All                  | Yes     | Yes          | 0mph            | 0mph           |
+| Make      | Model (US Market Reference)   | Supported Package | ACC              | No ACC accel below | No ALC below      |
+| ----------| ------------------------------| ------------------| -----------------| -------------------| ------------------|
+| Acura     | ILX 2016-18                   | AcuraWatch Plus   | openpilot        | 25mph<sup>1</sup>  | 25mph             |
+| Acura     | RDX 2016-18                   | AcuraWatch Plus   | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Accord 2018-19                | All               | Stock            | 0mph               | 3mph              |
+| Honda     | Accord Hybrid 2018-20         | All               | Stock            | 0mph               | 3mph              |
+| Honda     | Civic Hatchback 2017-19       | Honda Sensing     | Stock            | 0mph               | 12mph             |
+| Honda     | Civic Sedan/Coupe 2016-18     | Honda Sensing     | openpilot        | 0mph               | 12mph             |
+| Honda     | Civic Sedan/Coupe 2019-20     | Honda Sensing     | Stock            | 0mph               | 2mph<sup>2</sup>  |
+| Honda     | CR-V 2015-16                  | Touring           | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | CR-V 2017-20                  | Honda Sensing     | Stock            | 0mph               | 12mph             |
+| Honda     | CR-V Hybrid 2017-2019         | Honda Sensing     | Stock            | 0mph               | 12mph             |
+| Honda     | Fit 2018-19                   | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | HR-V 2019                     | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Insight 2019                  | Honda Sensing     | Stock            | 0mph               | 3mph              |
+| Honda     | Odyssey 2018-20               | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 0mph              |
+| Honda     | Passport 2019                 | All               | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Pilot 2016-18                 | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Pilot 2019                    | All               | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Ridgeline 2017-20             | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Hyundai   | Sonata 2020                   | All               | Stock            | 0mph               | 0mph              |
+| Lexus     | CT Hybrid 2017-18             | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Lexus     | ES 2019                       | All               | openpilot        | 0mph               | 0mph              |
+| Lexus     | ES Hybrid 2019                | All               | openpilot        | 0mph               | 0mph              |
+| Lexus     | IS 2017-2019                  | All               | Stock            | 22mph              | 0mph              |
+| Lexus     | IS Hybrid 2017                | All               | Stock            | 0mph               | 0mph              |
+| Lexus     | NX Hybrid 2018                | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Lexus     | RX 2016-17                    | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Lexus     | RX 2020                       | All               | openpilot        | 0mph               | 0mph              |
+| Lexus     | RX Hybrid 2016-19             | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Lexus     | RX Hybrid 2020                | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Avalon 2016                   | TSS-P             | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Avalon 2017-18                | All               | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Camry 2018-20                 | All               | Stock            | 0mph<sup>4</sup>   | 0mph              |
+| Toyota    | Camry Hybrid 2018-19          | All               | Stock            | 0mph<sup>4</sup>   | 0mph              |
+| Toyota    | C-HR 2017-19                  | All               | Stock            | 0mph               | 0mph              |
+| Toyota    | C-HR Hybrid 2017-19           | All               | Stock            | 0mph               | 0mph              |
+| Toyota    | Corolla 2017-19               | All               | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Corolla 2020                  | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Corolla Hatchback 2019-20     | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Corolla Hybrid 2020           | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Highlander 2017-19            | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Highlander Hybrid 2017-19     | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Highlander 2020               | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Highlander Hybrid 2020        | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Prius 2016                    | TSS-P             | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Prius 2017-20                 | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Prius Prime 2017-20           | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Rav4 2016                     | TSS-P             | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Rav4 2017-18                  | All               | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Rav4 2019-20                  | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Rav4 Hybrid 2016              | TSS-P             | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Rav4 Hybrid 2017-18           | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Rav4 Hybrid 2019-20           | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Sienna 2018-20                | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
 
+<sup>1</sup>[Comma Pedal](https://github.com/commaai/openpilot/wiki/comma-pedal) is used to provide stop-and-go capability to some of the openpilot-supported cars that don't currently support stop-and-go. ***NOTE: The Comma Pedal is not officially supported by [comma](https://comma.ai).*** <br />
+<sup>2</sup>2019 Honda Civic 1.6L Diesel Sedan does not have ALC below 12mph. <br />
+<sup>3</sup>When disconnecting the Driver Support Unit (DSU), openpilot ACC will replace stock ACC. ***NOTE: disconnecting the DSU disables Automatic Emergency Braking (AEB).*** <br />
+<sup>4</sup>28mph for Camry 4CYL L, 4CYL LE and 4CYL SE which don't have Full-Speed Range Dynamic Radar Cruise Control. <br />
 
-*[Comma Pedal](https://community.comma.ai/wiki/index.php/Comma_Pedal) is used to provide stop-and-go capability to some of the openpilot-supported cars that don't currently support stop-and-go. Here is how to [build a Comma Pedal](https://medium.com/@jfrux/comma-pedal-building-with-macrofab-6328bea791e8). ***NOTE: The Comma Pedal is not officially supported by [comma.ai](https://comma.ai)***
-
-Community Maintained Cars
+Community Maintained Cars and Features
 ------
 
-| Make    | Model                  | Supported Package    | Lateral | Longitudinal | No Accel Below  | No Steer Below	|
-| ------- | -----------------------| -------------------- | ------- | ------------ | -----------     | ------------ |
-| Tesla   | Model S 2012           | All                  | Yes     | Not yet      | Not applicable  | 0mph         |
-| Tesla   | Model S 2013           | All                  | Yes     | Not yet      | Not applicable  | 0mph         |
+| Make      | Model (US Market Reference)   | Supported Package | ACC              | No ACC accel below | No ALC below |
+| ----------| ------------------------------| ------------------| -----------------| -------------------| -------------|
+| Buick     | Regal 2018<sup>1</sup>        | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| Cadillac  | ATS 2018<sup>1</sup>          | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| Chevrolet | Malibu 2017<sup>1</sup>       | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| Chevrolet | Volt 2017-18<sup>1</sup>      | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| Chrysler  | Pacifica 2017-18              | Adaptive Cruise   | Stock            | 0mph               | 9mph         |
+| Chrysler  | Pacifica 2020                 | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
+| Chrysler  | Pacifica Hybrid 2017-18       | Adaptive Cruise   | Stock            | 0mph               | 9mph         |
+| Chrysler  | Pacifica Hybrid 2019-20       | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
+| Genesis   | G80 2018                      | All               | Stock            | 0mph               | 0mph         |
+| Genesis   | G90 2018                      | All               | Stock            | 0mph               | 0mph         |
+| GMC       | Acadia Denali 2018<sup>2</sup>| Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| Holden    | Astra 2017<sup>1</sup>        | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| Hyundai   | Elantra 2017-19               | SCC + LKAS        | Stock            | 19mph              | 34mph        |
+| Hyundai   | Genesis 2015-16               | SCC + LKAS        | Stock            | 19mph              | 37mph        |
+| Hyundai   | Ioniq Electric Premium SE 2020| SCC + LKAS        | Stock            | 0mph               | 32mph        |
+| Hyundai   | Ioniq Electric Limited 2019   | SCC + LKAS        | Stock            | 0mph               | 32mph        |
+| Hyundai   | Kona 2017-19                  | SCC + LKAS        | Stock            | 22mph              | 0mph         |
+| Hyundai   | Kona EV 2019                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Hyundai   | Palisade 2020                 | All               | Stock            | 0mph               | 0mph         |
+| Hyundai   | Santa Fe 2019                 | All               | Stock            | 0mph               | 0mph         |
+| Hyundai   | Sonata 2019                   | All               | Stock            | 0mph               | 0mph         |
+| Jeep      | Grand Cherokee 2016-18        | Adaptive Cruise   | Stock            | 0mph               | 9mph         |
+| Jeep      | Grand Cherokee 2019-20        | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
+| Kia       | Forte 2018-19                 | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Kia       | Optima 2017                   | SCC + LKAS/LDWS   | Stock            | 0mph               | 32mph        |
+| Kia       | Optima 2019                   | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Kia       | Sorento 2018                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Kia       | Stinger 2018                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Nissan    | Leaf 2018-19<sup>2</sup>      | Propilot          | Stock            | 0mph               | 0mph         |
+| Nissan    | Rogue 2019<sup>2</sup>        | Propilot          | Stock            | 0mph               | 0mph         |
+| Nissan    | X-Trail 2017<sup>2</sup>      | Propilot          | Stock            | 0mph               | 0mph         |
+| Subaru    | Ascent 2019                   | EyeSight          | Stock            | 0mph               | 0mph         |
+| Subaru    | Crosstrek 2018-19             | EyeSight          | Stock            | 0mph               | 0mph         |
+| Subaru    | Forester 2019                 | EyeSight          | Stock            | 0mph               | 0mph         |
+| Subaru    | Impreza 2017-19               | EyeSight          | Stock            | 0mph               | 0mph         |
+| Volkswagen| Golf 2015-19                  | Driver Assistance | Stock            | 0mph               | 0mph         |
 
-[[Tesla Pull Request]](https://github.com/commaai/openpilot/pull/246)
+<sup>1</sup>Requires an [OBD-II car harness](https://comma.ai/shop/products/comma-car-harness) and [community built giraffe](https://github.com/commaai/openpilot/wiki/GM). ***NOTE: disconnecting the ASCM disables Automatic Emergency Braking (AEB).*** <br />
+<sup>2</sup>Requires a custom connector for the developer [car harness](https://comma.ai/shop/products/car-harness) <br />
 
-*Community Maintained Cars are not confirmed by comma.ai to meet our safety model. Be extra cautious using them.
+Although it's not upstream, there's a community of people getting openpilot to run on Tesla's [here](https://tinkla.us/)
 
-In Progress Cars
-------
-- All TSS-P Toyota with Steering Assist.
-  - 'Full Speed Range Dynamic Radar Cruise Control' is required to enable stop-and-go. Only the Prius, Camry and C-HR have this option.
-  - Even though the Tundra, Sequoia and the Land Cruiser have TSS-P, they don't have Steering Assist and are not supported.
-- All LSS-P Lexus with Steering Assist or Lane Keep Assist.
-  - 'All-Speed Range Dynamic Radar Cruise Control' is required to enable stop-and-go. Only the GS, GSH, F, RX, RXH, LX, NX, NXH, LC, LCH, LS, LSH have this option.
-  - Even though the LX have TSS-P, it does not have Steering Assist and is not supported.
+Community Maintained Cars and Features are not verified by comma to meet our [safety model](SAFETY.md). Be extra cautious using them. They are only available after enabling the toggle in `Settings->Developer->Enable Community Features`.
 
-How can I add support for my car?
+To promote a car from community maintained, it must meet a few requirements. We must own one from the brand, we must sell the harness for it, has full ISO26262 in both panda and openpilot, there must be a path forward for longitudinal control, it must have AEB still enabled, and it must support fingerprinting 2.0
+
+Installation Instructions
 ------
 
-If your car has adaptive cruise control and lane keep assist, you are in luck. Using a [panda](https://comma.ai/shop/products/panda-obd-ii-dongle/) and [cabana](https://community.comma.ai/cabana/), you can understand how to make your car drive by wire.
+Install openpilot on an EON or comma two by entering ``https://openpilot.comma.ai`` during the installer setup.
 
-We've written a [porting guide](https://medium.com/@comma_ai/openpilot-port-guide-for-toyota-models-e5467f4b5fe6) for Toyota that might help you after you have the basics figured out.
+Follow these [video instructions](https://youtu.be/lcjqxCymins) to properly mount the device on the windshield. Note: openpilot features an automatic pose calibration routine and openpilot performance should not be affected by small pitch and yaw misalignments caused by imprecise device mounting.
 
-- BMW, Audi, Volvo, and Mercedes all use [FlexRay](https://en.wikipedia.org/wiki/FlexRay) and are unlikely to be supported any time soon.
-- We put time into a Ford port, but the steering has a 10 second cutout limitation that makes it unusable.
-- The 2016-2017 Honda Accord use a custom signaling protocol for steering that's unlikely to ever be upstreamed.
+Before placing the device on your windshield, check the state and local laws and ordinances where you drive. Some state laws prohibit or restrict the placement of objects on the windshield of a motor vehicle.
 
-Directory structure
+You will be able to engage openpilot after reviewing the onboarding screens and finishing the calibration procedure.
+
+Limitations of openpilot ALC and LDW
+------
+
+openpilot ALC and openpilot LDW do not automatically drive the vehicle or reduce the amount of attention that must be paid to operate your vehicle. The driver must always keep control of the steering wheel and be ready to correct the openpilot ALC action at all times.
+
+While changing lanes, openpilot is not capable of looking next to you or checking your blind spot. Only nudge the wheel to initiate a lane change after you have confirmed it's safe to do so.
+
+Many factors can impact the performance of openpilot ALC and openpilot LDW, causing them to be unable to function as intended. These include, but are not limited to:
+
+* Poor visibility (heavy rain, snow, fog, etc.) or weather conditions that may interfere with sensor operation.
+* The road facing camera is obstructed, covered or damaged by mud, ice, snow, etc.
+* Obstruction caused by applying excessive paint or adhesive products (such as wraps, stickers, rubber coating, etc.) onto the vehicle.
+* The device is mounted incorrectly.
+* When in sharp curves, like on-off ramps, intersections etc...; openpilot is designed to be limited in the amount of steering torque it can produce.
+* In the presence of restricted lanes or construction zones.
+* When driving on highly banked roads or in presence of strong cross-wind.
+* Extremely hot or cold temperatures.
+* Bright light (due to oncoming headlights, direct sunlight, etc.).
+* Driving on hills, narrow, or winding roads.
+
+The list above does not represent an exhaustive list of situations that may interfere with proper operation of openpilot components. It is the driver's responsibility to be in control of the vehicle at all times.
+
+Limitations of openpilot ACC and FCW
+------
+
+openpilot ACC and openpilot FCW are not systems that allow careless or inattentive driving. It is still necessary for the driver to pay close attention to the vehicle’s surroundings and to be ready to re-take control of the gas and the brake at all times.
+
+Many factors can impact the performance of openpilot ACC and openpilot FCW, causing them to be unable to function as intended. These include, but are not limited to:
+
+* Poor visibility (heavy rain, snow, fog, etc.) or weather conditions that may interfere with sensor operation.
+* The road facing camera or radar are obstructed, covered, or damaged by mud, ice, snow, etc.
+* Obstruction caused by applying excessive paint or adhesive products (such as wraps, stickers, rubber coating, etc.) onto the vehicle.
+* The device is mounted incorrectly.
+* Approaching a toll booth, a bridge or a large metal plate.
+* When driving on roads with pedestrians, cyclists, etc...
+* In presence of traffic signs or stop lights, which are not detected by openpilot at this time.
+* When the posted speed limit is below the user selected set speed. openpilot does not detect speed limits at this time.
+* In presence of vehicles in the same lane that are not moving.
+* When abrupt braking maneuvers are required. openpilot is designed to be limited in the amount of deceleration and acceleration that it can produce.
+* When surrounding vehicles perform close cut-ins from neighbor lanes.
+* Driving on hills, narrow, or winding roads.
+* Extremely hot or cold temperatures.
+* Bright light (due to oncoming headlights, direct sunlight, etc.).
+* Interference from other equipment that generates radar waves.
+
+The list above does not represent an exhaustive list of situations that may interfere with proper operation of openpilot components. It is the driver's responsibility to be in control of the vehicle at all times.
+
+Limitations of openpilot DM
+------
+
+openpilot DM should not be considered an exact measurement of the alertness of the driver.
+
+Many factors can impact the performance of openpilot DM, causing it to be unable to function as intended. These include, but are not limited to:
+
+* Low light conditions, such as driving at night or in dark tunnels.
+* Bright light (due to oncoming headlights, direct sunlight, etc.).
+* The driver's face is partially or completely outside field of view of the driver facing camera.
+* Right hand driving vehicles.
+* The driver facing camera is obstructed, covered, or damaged.
+
+The list above does not represent an exhaustive list of situations that may interfere with proper operation of openpilot components. A driver should not rely on openpilot DM to assess their level of attention.
+
+User Data and comma Account
+------
+
+By default, openpilot uploads the driving data to our servers. You can also access your data by pairing with the comma connect app ([iOS](https://apps.apple.com/us/app/comma-connect/id1456551889), [Android](https://play.google.com/store/apps/details?id=ai.comma.connect&hl=en_US)). We use your data to train better models and improve openpilot for everyone.
+
+openpilot is open source software: the user is free to disable data collection if they wish to do so.
+
+openpilot logs the road facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
+The driver facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
+
+By using openpilot, you agree to [our Privacy Policy](https://my.comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
+
+Safety and Testing
+----
+
+* openpilot observes ISO26262 guidelines, see [SAFETY.md](SAFETY.md) for more detail.
+* openpilot has software in the loop [tests](.github/workflows/test.yaml) that run on every commit.
+* The safety model code lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
+* panda has software in the loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
+* Internally, we have a hardware in the loop Jenkins test suite that builds and unit tests the various processes.
+* panda has additional hardware in the loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
+* We run the latest openpilot in a testing closet containing 10 EONs continuously replaying routes.
+
+Testing on PC
+------
+
+Check out the tools directory in master: lots of tools you can use to replay driving data, test and develop openpilot from your pc.
+
+Community and Contributing
+------
+
+openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot). Bug fixes and new car ports are encouraged.
+
+You can add support for your car by following guides we have written for [Brand](https://medium.com/@comma_ai/how-to-write-a-car-port-for-openpilot-7ce0785eda84) and [Model](https://medium.com/@comma_ai/openpilot-port-guide-for-toyota-models-e5467f4b5fe6) ports. Generally, a car with adaptive cruise control and lane keep assist is a good candidate. [Join our Discord](https://discord.comma.ai) to discuss car ports: most car makes have a dedicated channel.
+
+Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs/).
+
+And [follow us on Twitter](https://twitter.com/comma_ai).
+
+Directory Structure
 ------
     .
     ├── apk                 # The apk files used for the UI
-    ├── cereal              # The messaging spec used for all logs on EON
+    ├── cereal              # The messaging spec and libs used for all logs
     ├── common              # Library like functionality we've developed here
     ├── installer/updater   # Manages auto-updates of openpilot
     ├── opendbc             # Files showing how to interpret data from cars
-    ├── panda               # Code used to communicate on CAN and LIN
-    ├── phonelibs           # Libraries used on EON
-    ├── pyextra             # Libraries used on EON
+    ├── panda               # Code used to communicate on CAN
+    ├── phonelibs           # Libraries used on NEOS devices
+    ├── pyextra             # Libraries used on NEOS devices
     └── selfdrive           # Code needed to drive the car
-        ├── assets          # Fonts and images for UI
+        ├── assets          # Fonts, images, and sounds for UI
+        ├── athena          # Allows communication with the app
         ├── boardd          # Daemon to talk to the board
-        ├── can             # Helpers for parsing CAN messages
+        ├── camerad         # Driver to capture images from the camera sensors
         ├── car             # Car specific code to read states and control actuators
         ├── common          # Shared C/C++ code for the daemons
         ├── controls        # Perception, planning and controls
@@ -135,56 +309,31 @@ Directory structure
         ├── locationd       # Soon to be home of precise location
         ├── logcatd         # Android logcat as a service
         ├── loggerd         # Logger and uploader of car data
+        ├── modeld          # Driving and monitoring model runners
         ├── proclogd        # Logs information from proc
         ├── sensord         # IMU / GPS interface code
-        ├── test            # Car simulator running code through virtual maneuvers
-        ├── ui              # The UI
-        └── visiond         # Embedded vision pipeline
+        ├── test            # Unit tests, system tests and a car simulator
+        └── ui              # The UI
 
-To understand how the services interact, see `selfdrive/service_list.yaml`
-
-User Data / chffr Account / Crash Reporting
-------
-
-By default openpilot creates an account and includes a client for chffr, our dashcam app. We use your data to train better models and improve openpilot for everyone.
-
-It's open source software, so you are free to disable it if you wish.
-
-It logs the road facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The user facing camera is only logged if you explicitly opt-in in settings.
-It does not log the microphone.
-
-By using it, you agree to [our privacy policy](https://community.comma.ai/privacy.html). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma.ai. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma.ai for the use of this data.
-
-Testing on PC
-------
-
-There is rudimentary infrastructure to run a basic simulation and generate a report of openpilot's behavior in different scenarios.
-
-```bash
-# Requires working docker
-./run_docker_tests.sh
-```
-
-The resulting plots are displayed in `selfdrive/test/tests/plant/out/longitudinal/index.html`
-
-More extensive testing infrastructure and simulation environments are coming soon.
-
-Contributing
-------
-
-We welcome both pull requests and issues on
-[github](http://github.com/commaai/openpilot). Bug fixes and new car support encouraged.
-
-Want to get paid to work on openpilot? [comma.ai is hiring](https://comma.ai/jobs/)
+To understand how the services interact, see `cereal/service_list.yaml`.
 
 Licensing
 ------
 
 openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
-Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
+Any user of this software shall indemnify and hold harmless comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
 
 **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
 YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
 NO WARRANTY EXPRESSED OR IMPLIED.**
+
+---
+
+<img src="https://d1qb2nb5cznatu.cloudfront.net/startups/i/1061157-bc7e9bf3b246ece7322e6ffe653f6af8-medium_jpg.jpg?buster=1458363130" width="75"></img> <img src="https://cdn-images-1.medium.com/max/1600/1*C87EjxGeMPrkTuVRVWVg4w.png" width="225"></img>
+
+[![openpilot tests](https://github.com/commaai/openpilot/workflows/openpilot%20tests/badge.svg?event=push)](https://github.com/commaai/openpilot/actions)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/alerts/)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:python)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:cpp)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)

README_chffrplus.md

@@ -1,36 +0,0 @@
-Welcome to chffrplus
-======
-
-[chffrplus](https://github.com/commaai/chffrplus) is an open source dashcam.
-
-This is the shipping reference software for the comma EON Dashcam DevKit. It keeps many of the niceities of [openpilot](https://github.com/commaai/openpilot), like high quality sensors, great camera, and good autostart and stop. Though unlike openpilot, it cannot control your car. chffrplus can interface with your car through a [panda](https://shop.comma.ai/products/panda-obd-ii-dongle), but just like our dashcam app [chffr](https://getchffr.com/), it is read only.
-
-It integrates with the rest of the comma ecosystem, so you can view your drives on the [chffr](https://getchffr.com/) app for Android or iOS, and reverse engineer your car with [cabana](https://community.comma.ai/cabana/?demo=1).
-
-
-Hardware
-------
-
-Right now chffrplus supports the [EON Dashcam DevKit](https://shop.comma.ai/products/eon-dashcam-devkit) for hardware to run on.
-
-Install chffrplus on a EON device by entering ``https://chffrplus.comma.ai`` during NEOS setup.
-
-
-User Data / chffr Account / Crash Reporting
-------
-
-By default chffrplus creates an account and includes a client for chffr, our dashcam app.
-
-It's open source software, so you are free to disable it if you wish. 
-
-It logs the road facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-It does not log the user facing camera or the microphone.
-
-By using it, you agree to [our privacy policy](https://beta.comma.ai/privacy.html). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma.ai. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma.ai for the use of this data.
-
-
-Licensing
-------
-
-chffrplus is released under the MIT license.
-

RELEASES.md

@@ -1,3 +1,302 @@
+Version 0.7.7 (2020-07-20)
+========================
+ * White panda is no longer supported, upgrade to comma two or black panda
+ * Improved vehicle model estimation using high precision localizer
+ * Improved thermal management on comma two
+ * Improved autofocus for road-facing camera
+ * Improved noise performance for driver-facing camera
+ * Block lane change start using blindspot monitor on select Toyota, Hyundai, and Subaru
+ * Fix GM ignition detection
+ * Code cleanup and smaller release sizes
+ * Hyundai Sonata 2020 promoted to officially supported car
+ * Hyundai Ioniq Electric Limited 2019 and Ioniq SE 2020 support thanks to baldwalker!
+ * Subaru Forester 2019 and Ascent 2019 support thanks to martinl!
+
+Version 0.7.6.1 (2020-06-16)
+========================
+* Hotfix: update kernel on some comma twos (orders #8570-#8680)
+
+Version 0.7.6 (2020-06-05)
+========================
+* White panda is deprecated, upgrade to comma two or black panda
+* 2017 Nissan X-Trail, 2018-19 Leaf and 2019 Rogue support thanks to avolmensky!
+* 2017 Mazda CX-5 support in dashcam mode thanks to Jafaral!
+* Huge CPU savings in modeld by using thneed!
+* Lots of code cleanup and refactors
+
+Version 0.7.5 (2020-05-13)
+========================
+* Right-Hand Drive support for both driving and driver monitoring!
+* New driving model: improved at sharp turns and lead speed estimation
+* New driver monitoring model: overall improvement on comma two
+* Driver camera preview in settings to improve mounting position
+* Added support for many Hyundai, Kia, Genesis models thanks to xx979xx!
+* Improved lateral tuning for 2020 Toyota Rav 4 (hybrid)
+
+Version 0.7.4 (2020-03-20)
+========================
+ * New driving model: improved lane changes and lead car detection
+ * Improved driver monitoring model: improve eye detection
+ * Improved calibration stability
+ * Improved lateral control on some 2019 and 2020 Toyota Prius
+ * Improved lateral control on VW Golf: 20% more steering torque
+ * Fixed bug where some 2017 and 2018 Toyota C-HR would use the wrong steering angle sensor
+ * Support for Honda Insight thanks to theantihero!
+ * Code cleanup in car abstraction layers and ui
+
+Version 0.7.3 (2020-02-21)
+========================
+ * Support for 2020 Highlander thanks to che220!
+ * Support for 2018 Lexus NX 300h thanks to kengggg!
+ * Speed up ECU firmware query
+ * Fix bug where manager would sometimes hang after shutting down the car
+
+Version 0.7.2 (2020-02-07)
+========================
+ * ECU firmware version based fingerprinting for Honda & Toyota
+ * New driving model: improved path prediction during turns and lane changes and better lead speed tracking
+ * Improve driver monitoring under extreme lighting and add low accuracy alert
+ * Support for 2019 Rav4 Hybrid thanks to illumiN8i!
+ * Support for 2016, 2017 and 2020 Lexus RX thanks to illumiN8i!
+ * Support for 2020 Chrysler Pacifica Hybrid thanks to adhintz!
+
+Version 0.7.1 (2020-01-20)
+========================
+ * comma two support!
+ * Lane Change Assist above 45 mph!
+ * Replace zmq with custom messaging library, msgq!
+ * Supercombo model: calibration and driving models are combined for better lead estimate
+ * More robust updater thanks to jyoung8607! Requires NEOS update
+ * Improve low speed ACC tuning
+
+Version 0.7 (2019-12-13)
+========================
+ * Move to SCons build system!
+ * Add Lane Departure Warning (LDW) for all supported vehicles!
+ * NEOS update: increase wifi speed thanks to jyoung8607!
+ * Adaptive driver monitoring based on scene
+ * New driving model trained end-to-end: improve lane lines and lead detection
+ * Smarter torque limit alerts for all cars
+ * Improve GM longitudinal control: proper computations for 15Hz radar
+ * Move GM port, Toyota with DSU removed, comma pedal in community features; toggle switch required
+ * Remove upload over cellular toggle: only upload qlog and qcamera files if not on wifi
+ * Refactor Panda code towards ISO26262 and SIL2 compliancy
+ * Forward stock FCW for Honda Nidec
+ * Volkswagen port now standard: comma Harness intercepts stock camera
+
+Version 0.6.6 (2019-11-05)
+========================
+ * Volkswagen support thanks to jyoung8607!
+ * Toyota Corolla Hybrid with TSS 2.0 support thanks to u8511049!
+ * Lexus ES with TSS 2.0 support thanks to energee!
+ * Fix GM ignition detection and lock safety mode not required anymore
+ * Log panda firmware and dongle ID thanks to martinl!
+ * New driving model: improve path prediction and lead detection
+ * New driver monitoring model, 4x smaller and running on DSP
+ * Display an alert and don't start openpilot if panda has wrong firmware
+ * Fix bug preventing EON from terminating processes after a drive
+ * Remove support for Toyota giraffe without the 120Ohm resistor
+
+Version 0.6.5 (2019-10-07)
+========================
+ * NEOS update: upgrade to Python3 and new installer!
+ * comma Harness support!
+ * New driving model: improve path prediction
+ * New driver monitoring model: more accurate face and eye detection
+ * Redesign offroad screen to display updates and alerts
+ * Increase maximum allowed acceleration
+ * Prevent car 12V battery drain by cutting off EON charge after 3 days of no drive
+ * Lexus CT Hybrid support thanks to thomaspich!
+ * Louder chime for critical alerts
+ * Add toggle to switch to dashcam mode
+ * Fix "invalid vehicle params" error on DSU-less Toyota
+
+Version 0.6.4 (2019-09-08)
+========================
+ * Forward stock AEB for Honda Nidec
+ * Improve lane centering on banked roads
+ * Always-on forward collision warning
+ * Always-on driver monitoring, except for right hand drive countries
+ * Driver monitoring learns the user's normal driving position
+ * Honda Fit support thanks to energee!
+ * Lexus IS support
+
+Version 0.6.3 (2019-08-12)
+========================
+ * Alert sounds from EON: requires NEOS update
+ * Improve driver monitoring: eye tracking and improved awareness logic
+ * Improve path prediction with new driving model
+ * Improve lane positioning with wide lanes and exits
+ * Improve lateral control on RAV4
+ * Slow down for turns using model
+ * Open sourced regression test to verify outputs against reference logs
+ * Open sourced regression test to sanity check all car models
+
+Version 0.6.2 (2019-07-29)
+========================
+ * New driving model!
+ * Improve lane tracking with double lines
+ * Strongly improve stationary vehicle detection
+ * Strongly reduce cases of braking due to false leads
+ * Better lead tracking around turns
+ * Improve cut-in prediction by using neural network
+ * Improve lateral control on Toyota Camry and C-HR thanks to zorrobyte!
+ * Fix unintended openpilot disengagements on Jeep thanks to adhintz!
+ * Fix delayed transition to offroad when car is turned off
+
+Version 0.6.1 (2019-07-21)
+========================
+ * Remote SSH with comma prime and [ssh.comma.ai](https://ssh.comma.ai)
+ * Panda code Misra-c2012 compliance, tested against cppcheck coverage
+ * Lockout openpilot after 3 terminal alerts for driver distracted or unresponsive
+ * Toyota Sienna support thanks to wocsor!
+
+Version 0.6 (2019-07-01)
+========================
+ * New model, with double the pixels and ten times the temporal context!
+ * Car should not take exits when in the right lane
+ * openpilot uses only ~65% of the CPU (down from 75%)
+ * Routes visible in connect/explorer after only 0.2% is uploaded (qlogs)
+ * loggerd and sensord are open source, every line of openpilot is now open
+ * Panda safety code is MISRA compliant and ships with a signed version on release2
+ * New NEOS is 500MB smaller and has a reproducible usr/pipenv
+ * Lexus ES Hybrid support thanks to wocsor!
+ * Improve tuning for supported Toyota with TSS 2.0
+ * Various other stability improvements
+
+Version 0.5.13 (2019-05-31)
+==========================
+ * Reduce panda power consumption by 70%, down to 80mW, when car is off (not for GM)
+ * Reduce EON power consumption by 40%, down to 1100mW, when car is off
+ * Reduce CPU utilization by 20% and improve stability
+ * Temporarily remove mapd functionalities to improve stability
+ * Add openpilot record-only mode for unsupported cars
+ * Synchronize controlsd to boardd to reduce latency
+ * Remove panda support for Subaru giraffe
+
+Version 0.5.12 (2019-05-16)
+==========================
+ * Improve lateral control for the Prius and Prius Prime
+ * Compress logs before writing to disk
+ * Remove old driving data when storage reaches 90% full
+ * Fix small offset in following distance
+ * Various small CPU optimizations
+ * Improve offroad power consumption: require NEOS Update
+ * Add default speed limits for Estonia thanks to martinl!
+ * Subaru Crosstrek support thanks to martinl!
+ * Toyota Avalon support thanks to njbrown09!
+ * Toyota Rav4 with TSS 2.0 support thanks to wocsor!
+ * Toyota Corolla with TSS 2.0 support thanks to wocsor!
+
+Version 0.5.11 (2019-04-17)
+========================
+ * Add support for Subaru
+ * Reduce panda power consumption by 60% when car is off
+ * Fix controlsd lag every 6 minutes. This would sometimes cause disengagements
+ * Fix bug in controls with new angle-offset learner in MPC
+ * Reduce cpu consumption of ubloxd by rewriting it in C++
+ * Improve driver monitoring model and face detection
+ * Improve performance of visiond and ui
+ * Honda Passport 2019 support
+ * Lexus RX Hybrid 2019 support thanks to schomems!
+ * Improve road selection heuristic in mapd
+ * Add Lane Departure Warning to dashboard for Toyota thanks to arne182
+
+Version 0.5.10 (2019-03-19)
+========================
+ * Self-tuning vehicle parameters: steering offset, tire stiffness and steering ratio
+ * Improve longitudinal control at low speed when lead vehicle harshly decelerates
+ * Fix panda bug going unexpectedly in DCP mode when EON is connected
+ * Reduce white panda power consumption by 500mW when EON is disconnected by turning off WIFI
+ * New Driver Monitoring Model
+ * Support QR codes for login using comma connect
+ * Refactor comma pedal FW and use CRC-8 checksum algorithm for safety. Reflashing pedal is required.
+   Please see `#hw-pedal` on [discord](discord.comma.ai) for assistance updating comma pedal.
+ * Additional speed limit rules for Germany thanks to arne182
+ * Allow negative speed limit offsets
+
+Version 0.5.9 (2019-02-10)
+========================
+ * Improve calibration using a dedicated neural network
+ * Abstract planner in its own process to remove lags in controls process
+ * Improve speed limits with country/region defaults by road type
+ * Reduce mapd data usage with gzip thanks to eFiniLan
+ * Zip log files in the background to reduce disk usage
+ * Kia Optima support thanks to emmertex!
+ * Buick Regal 2018 support thanks to HOYS!
+ * Comma pedal support for Toyota thanks to wocsor! Note: tuning needed and not maintained by comma
+ * Chrysler Pacifica and Jeep Grand Cherokee support thanks to adhintz!
+
+Version 0.5.8 (2019-01-17)
+========================
+ * Open sourced visiond
+ * Auto-slowdown for upcoming turns
+ * Chrysler/Jeep/Fiat support thanks to adhintz!
+ * Honda Civic 2019 support thanks to csouers!
+ * Improve use of car display in Toyota thanks to arne182!
+ * No data upload when connected to Android or iOS hotspots and "Enable Upload Over Cellular" setting is off
+ * EON stops charging when 12V battery drops below 11.8V
+
+Version 0.5.7 (2018-12-06)
+========================
+ * Speed limit from OpenStreetMap added to UI
+ * Highlight speed limit when speed exceeds road speed limit plus a delta
+ * Option to limit openpilot max speed to road speed limit plus a delta
+ * Cadillac ATS support thanks to vntarasov!
+ * GMC Acadia support thanks to CryptoKylan!
+ * Decrease GPU power consumption
+ * NEOSv8 autoupdate
+
+Version 0.5.6 (2018-11-16)
+========================
+ * Refresh settings layout and add feature descriptions
+ * In Honda, keep stock camera on for logging and extra stock features; new openpilot giraffe setting is 0111!
+ * In Toyota, option to keep stock camera on for logging and extra stock features (e.g. AHB); 120Ohm resistor required on giraffe.
+ * Improve camera calibration stability
+ * More tuning to Honda positive accelerations
+ * Reduce brake pump use on Hondas
+ * Chevrolet Malibu support thanks to tylergets!
+ * Holden Astra support thanks to AlexHill!
+
+Version 0.5.5 (2018-10-20)
+========================
+ * Increase allowed Honda positive accelerations
+ * Fix sporadic unexpected braking when passing semi-trucks in Toyota
+ * Fix gear reading bug in Hyundai Elantra thanks to emmertex!
+
+Version 0.5.4 (2018-09-25)
+========================
+ * New Driving Model
+ * New Driver Monitoring Model
+ * Improve longitudinal mpc in mid-low speed braking
+ * Honda Accord hybrid support thanks to energee!
+ * Ship mpc binaries and sensibly reduce build time
+ * Calibration more stable
+ * More Hyundai and Kia cars supported thanks to emmertex!
+ * Various GM Volt improvements thanks to vntarasov!
+
+Version 0.5.3 (2018-09-03)
+========================
+ * Hyundai Santa Fe support!
+ * Honda Pilot 2019 support thanks to energee!
+ * Toyota Highlander support thanks to daehahn!
+ * Improve steering tuning for Honda Odyssey
+
+Version 0.5.2 (2018-08-16)
+========================
+ * New calibration: more accurate, a lot faster, open source!
+ * Enable orbd
+ * Add little endian support to CAN packer
+ * Fix fingerprint for Honda Accord 1.5T
+ * Improve driver monitoring model
+
+Version 0.5.1 (2018-08-01)
+========================
+ * Fix radar error on Civic sedan 2018
+ * Improve thermal management logic
+ * Alpha Toyota C-HR and Camry support!
+ * Auto-switch Driver Monitoring to 3 min counter when inaccurate
+
 Version 0.5 (2018-07-11)
 ========================
  * Driver Monitoring (beta) option in settings!

SAFETY.md

@@ -1,73 +1,34 @@
 openpilot Safety
 ======
 
-openpilot is an Adaptive Cruise Control (ACC) and Lane Keeping Assist (LKA) system. 
-Like other ACC and LKA systems, openpilot requires the driver to be alert and to 
-pay attention at all times. We repeat, **driver alertness is necessary, but not 
-sufficient, for openpilot to be used safely**.
+openpilot is an Adaptive Cruise Control (ACC) and Automated Lane Centering (ALC) system.
+Like other ACC and ALC systems, openpilot is a failsafe passive system and it requires the
+driver to be alert and to pay attention at all times.
 
-Even with an attentive driver, we must make further efforts for the system to be
-safe. We have designed openpilot with two other safety considerations.
+In order to enforce driver alertness, openpilot includes a driver monitoring feature
+that alerts the driver when distracted.
 
-1. The driver must always be capable to immediately retake manual control of the vehicle, 
+However, even with an attentive driver, we must make further efforts for the system to be
+safe. We repeat, **driver alertness is necessary, but not sufficient, for openpilot to be
+used safely** and openpilot is provided with no warranty of fitness for any purpose.
+
+openpilot is developed in good faith to be compliant with FMVSS requirements and to follow
+industry standards of safety for Level 2 Driver Assistance Systems. In particular, we observe
+ISO26262 guidelines, including those from [pertinent documents](https://www.nhtsa.gov/sites/nhtsa.dot.gov/files/documents/13498a_812_573_alcsystemreport.pdf)
+released by NHTSA. In addition, we impose strict coding guidelines (like [MISRA C : 2012](https://www.misra.org.uk/MISRAHome/MISRAC2012/tabid/196/Default.aspx))
+on parts of openpilot that are safety relevant. We also perform software-in-the-loop,
+hardware-in-the-loop and in-vehicle tests before each software release.
+
+Following Hazard and Risk Analysis and FMEA, at a very high level, we have designed openpilot
+ensuring two main safety requirements.
+
+1. The driver must always be capable to immediately retake manual control of the vehicle,
    by stepping on either pedal or by pressing the cancel button.
 2. The vehicle must not alter its trajectory too quickly for the driver to safely
    react. This means that while the system is engaged, the actuators are constrained
    to operate within reasonable limits.
 
-Following are details of the car specific safety implementations:
+For vehicle specific implementation of the safety concept, refer to `panda/board/safety/`.
 
-Honda/Acura
-------
-
-  - While the system is engaged, gas, brake and steer limits are subject to the same limits used by
-    the stock system.
-
-  - Without an interceptor, the gas is controlled by the Powertrain Control Module (PCM). 
-    The PCM limits acceleration to what is reasonable for a cruise control system.  With an
-    interceptor, the gas is clipped to 60%.
-
-  - The brake is controlled by the 0x1FA CAN message. This message allows full
-    braking, although the board and the software clip it to 1/4th of the max.
-    This is around .3g of braking.
-
-  - Steering is controlled by the 0xE4 CAN message. The Electronic Power Steering (EPS) 
-    controller in the car limits the torque to a very small amount, so regardless of the 
-    message, the controller cannot jerk the wheel.
-
-  - Brake and gas pedal pressed signals are contained in the 0x17C CAN message. A rising edge of
-    either signal triggers a disengagement, which is enforced by the board and in software. The
-    green led on the board signifies if the board is allowing control messages.
-
-  - Honda CAN uses both a counter and a checksum to ensure integrity and prevent
-    replay of the same message.
-
-Toyota/Lexus
-------
-
-  - While the system is engaged, gas, brake and steer limits are subject to the same limits used by
-    the stock system.
-
-  - With the stock Driving Support Unit (DSU) enabled, the acceleration is controlled 
-    by the stock system and is subject to the stock adaptive cruise control limits. Without the
-    stock DSU connected, the acceleration command is controlled by the 0x343 CAN message and its
-    value is limited by the board and the software to between .3g of deceleration and .15g of
-    acceleration. The acceleration command is ignored by the Engine Control Module (ECM) while the
-    cruise control system is disengaged.
-
-  - Steering torque is controlled through the 0x2E4 CAN message and it's limited by the board and in
-    software to a value of -1500 and 1500. In addition, the vehicle EPS unit will not respond to
-    commands outside these limits.  A steering torque rate limit is enforced by the board and in
-    software so that the commanded steering torque must rise from 0 to max value no faster than
-    1.5s. Commanded steering torque is limited by the board and in software to be no more than 350
-    units above the actual EPS generated motor torque to ensure limited differences between
-    commanded and actual torques.
-
-  - Brake and gas pedal pressed signals are contained in the 0x224 and 0x1D2 CAN messages,
-    respectively. A rising edge of either signal triggers a disengagement, which is enforced by the
-    board and in software. Additionally, the cruise control system disengages on the rising edge of
-    the brake pedal pressed signal.
-
-  - The cruise control system state is contained in the 0x1D2 message. No control messages are
-    allowed if the cruise control system is not active. This is enforced by the software and the
-    board. The green led on the board signifies if the board is allowing control messages.
+**Extra note**: comma.ai strongly discourages the use of openpilot forks with safety code either missing or
+  not fully meeting the above requirements.

SConstruct

@@ -0,0 +1,264 @@
+import os
+import shutil
+import subprocess
+import sys
+import platform
+
+AddOption('--test',
+          action='store_true',
+          help='build test files')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
+if arch == "aarch64" and not os.path.isdir("/system"):
+  arch = "larch64"
+
+webcam = bool(ARGUMENTS.get("use_webcam", 0))
+QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None
+
+if arch == "aarch64" or arch == "larch64":
+  lenv = {
+    "LD_LIBRARY_PATH": '/data/data/com.termux/files/usr/lib',
+    "PATH": os.environ['PATH'],
+  }
+
+  if arch == "aarch64":
+    # android
+    lenv["ANDROID_DATA"] = os.environ['ANDROID_DATA']
+    lenv["ANDROID_ROOT"] = os.environ['ANDROID_ROOT']
+
+  cpppath = [
+    "#phonelibs/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/lib",
+    "/data/data/com.termux/files/usr/lib",
+    "/system/vendor/lib64",
+    "/system/comma/usr/lib",
+    "#phonelibs/nanovg",
+  ]
+
+  if arch == "larch64":
+    libpath += [
+      "#phonelibs/snpe/larch64",
+      "#phonelibs/libyuv/larch64/lib",
+      "/usr/lib/aarch64-linux-gnu"
+    ]
+    cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+    cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+    rpath = ["/usr/local/lib"]
+  else:
+    libpath += [
+      "#phonelibs/snpe/aarch64",
+      "#phonelibs/libyuv/lib"
+    ]
+    cflags = ["-DQCOM", "-mcpu=cortex-a57"]
+    cxxflags = ["-DQCOM", "-mcpu=cortex-a57"]
+    rpath = ["/system/vendor/lib64"]
+
+    if QCOM_REPLAY:
+      cflags += ["-DQCOM_REPLAY"]
+      cxxflags += ["-DQCOM_REPLAY"]
+
+else:
+  cflags = []
+  cxxflags = []
+
+  lenv = {
+    "PATH": "#external/bin:" + os.environ['PATH'],
+  }
+  cpppath = [
+    "#external/tensorflow/include",
+  ]
+
+  if arch == "Darwin":
+    libpath = [
+      "#phonelibs/libyuv/mac/lib",
+      "#cereal",
+      "#selfdrive/common",
+      "/usr/local/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+  else:
+    libpath = [
+      "#phonelibs/snpe/x86_64-linux-clang",
+      "#phonelibs/libyuv/x64/lib",
+      "#external/tensorflow/lib",
+      "#cereal",
+      "#selfdrive/common",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+  rpath = [
+    "external/tensorflow/lib",
+    "cereal",
+    "selfdrive/common"
+  ]
+
+  # allows shared libraries to work globally
+  rpath = [os.path.join(os.getcwd(), x) for x in rpath]
+
+if GetOption('asan'):
+  ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags_asan = ["-fsanitize=address"]
+else:
+  ccflags_asan = []
+  ldflags_asan = []
+
+# change pythonpath to this
+lenv["PYTHONPATH"] = Dir("#").path
+
+env = Environment(
+  ENV=lenv,
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Wunused",
+    "-Werror",
+    "-Wno-deprecated-register",
+    "-Wno-inconsistent-missing-override",
+  ] + cflags + ccflags_asan,
+
+  CPPPATH=cpppath + [
+    "#",
+    "#selfdrive",
+    "#phonelibs/bzip2",
+    "#phonelibs/libyuv/include",
+    "#phonelibs/openmax/include",
+    "#phonelibs/json11",
+    "#phonelibs/curl/include",
+    #"#phonelibs/opencv/include", # use opencv4 instead
+    "#phonelibs/libgralloc/include",
+    "#phonelibs/android_frameworks_native/include",
+    "#phonelibs/android_hardware_libhardware/include",
+    "#phonelibs/android_system_core/include",
+    "#phonelibs/linux/include",
+    "#phonelibs/snpe/include",
+    "#phonelibs/nanovg",
+    "#selfdrive/common",
+    "#selfdrive/camerad",
+    "#selfdrive/camerad/include",
+    "#selfdrive/loggerd/include",
+    "#selfdrive/modeld",
+    "#cereal/messaging",
+    "#cereal",
+    "#opendbc/can",
+  ],
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags_asan,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++14"] + cxxflags,
+  LIBPATH=libpath + [
+    "#cereal",
+    "#selfdrive/common",
+    "#phonelibs",
+  ]
+)
+
+if os.environ.get('SCONS_CACHE'):
+  cache_dir = '/tmp/scons_cache'
+
+  if os.getenv('CI'):
+    branch = os.getenv('GIT_BRANCH')
+
+    if QCOM_REPLAY:
+      cache_dir = '/tmp/scons_cache_qcom_replay'
+    elif branch is not None and branch != 'master':
+      cache_dir_branch = '/tmp/scons_cache_' + branch
+      if not os.path.isdir(cache_dir_branch) and os.path.isdir(cache_dir):
+        shutil.copytree(cache_dir, cache_dir_branch)
+      cache_dir = cache_dir_branch
+  CacheDir(cache_dir)
+
+node_interval = 5
+node_count = 0
+def progress_function(node):
+  global node_count
+  node_count += node_interval
+  sys.stderr.write("progress: %d\n" % node_count)
+
+if os.environ.get('SCONS_PROGRESS'):
+  Progress(progress_function, interval=node_interval)
+
+SHARED = False
+
+def abspath(x):
+  if arch == 'aarch64':
+    pth = os.path.join("/data/pythonpath", x[0].path)
+    env.Depends(pth, x)
+    return File(pth)
+  else:
+    # rpath works elsewhere
+    return x[0].path.rsplit("/", 1)[1][:-3]
+
+# still needed for apks
+zmq = 'zmq'
+Export('env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY')
+
+# cereal and messaging are shared with the system
+SConscript(['cereal/SConscript'])
+if SHARED:
+  cereal = abspath([File('cereal/libcereal_shared.so')])
+  messaging = abspath([File('cereal/libmessaging_shared.so')])
+else:
+  cereal = [File('#cereal/libcereal.a')]
+  messaging = [File('#cereal/libmessaging.a')]
+Export('cereal', 'messaging')
+
+SConscript(['selfdrive/common/SConscript'])
+Import('_common', '_visionipc', '_gpucommon', '_gpu_libs')
+
+if SHARED:
+  common, visionipc, gpucommon = abspath(common), abspath(visionipc), abspath(gpucommon)
+else:
+  common = [_common, 'json11']
+  visionipc = _visionipc
+  gpucommon = [_gpucommon] + _gpu_libs
+
+Export('common', 'visionipc', 'gpucommon')
+
+SConscript(['opendbc/can/SConscript'])
+
+SConscript(['common/SConscript'])
+SConscript(['common/kalman/SConscript'])
+SConscript(['common/transformations/SConscript'])
+SConscript(['phonelibs/SConscript'])
+
+SConscript(['selfdrive/camerad/SConscript'])
+SConscript(['selfdrive/modeld/SConscript'])
+
+SConscript(['selfdrive/controls/lib/cluster/SConscript'])
+SConscript(['selfdrive/controls/lib/lateral_mpc/SConscript'])
+SConscript(['selfdrive/controls/lib/longitudinal_mpc/SConscript'])
+SConscript(['selfdrive/controls/lib/longitudinal_mpc_model/SConscript'])
+
+SConscript(['selfdrive/boardd/SConscript'])
+SConscript(['selfdrive/proclogd/SConscript'])
+
+SConscript(['selfdrive/ui/SConscript'])
+SConscript(['selfdrive/loggerd/SConscript'])
+
+SConscript(['selfdrive/locationd/SConscript'])
+SConscript(['selfdrive/locationd/models/SConscript'])
+
+if arch == "aarch64":
+  SConscript(['selfdrive/logcatd/SConscript'])
+  SConscript(['selfdrive/sensord/SConscript'])
+  SConscript(['selfdrive/clocksd/SConscript'])
+else:
+  SConscript(['tools/lib/index_log/SConscript'])

cereal/.gitignore

@@ -1,3 +1,14 @@
 gen
 node_modules
 package-lock.json
+*.pyc
+__pycache__
+.*.swp
+.*.swo
+libcereal*.a
+libmessaging.*
+libmessaging_shared.*
+services.h
+.sconsign.dblite
+libcereal_shared.*
+.mypy_cache/

cereal/Makefile

@@ -1,59 +0,0 @@
-PWD := $(shell pwd)
-
-SRCS := log.capnp car.capnp
-
-GENS := gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++
-JS := gen/js/car.capnp.js gen/js/log.capnp.js
-
-UNAME_M ?= $(shell uname -m)
-
-# only generate C++ for docker tests
-ifneq ($(OPTEST),1)
-	GENS += gen/c/car.capnp.c gen/c/log.capnp.c gen/c/include/c++.capnp.h gen/c/include/java.capnp.h
-
-	ifeq ($(UNAME_M),x86_64)
-		GENS += gen/java/Car.java gen/java/Log.java
-	endif
-
-endif
-
-ifeq ($(UNAME_M),aarch64)
-	CAPNPC=PATH=$(PWD)/../phonelibs/capnp-cpp/aarch64/bin/:$$PATH capnpc
-else
-	CAPNPC=capnpc
-endif
-
-.PHONY: all
-all: $(GENS)
-js: $(JS)
-
-.PHONY: clean
-clean:
-	rm -rf gen
-	rm -rf node_modules
-	rm -rf package-lock.json
-
-gen/c/%.capnp.c: %.capnp
-	@echo "[ CAPNPC C ] $@"
-	mkdir -p gen/c/
-	$(CAPNPC) '$<' -o c:gen/c/
-
-gen/js/%.capnp.js: %.capnp
-	@echo "[ CAPNPC JavaScript ] $@"
-	mkdir -p gen/js/
-	sh ./generate_javascript.sh
-
-gen/cpp/%.capnp.c++: %.capnp
-	@echo "[ CAPNPC C++ ] $@"
-	mkdir -p gen/cpp/
-	$(CAPNPC) '$<' -o c++:gen/cpp/
-
-gen/java/Car.java gen/java/Log.java: $(SRCS)
-	@echo "[ CAPNPC java ] $@"
-	mkdir -p gen/java/
-	$(CAPNPC) $^ -o java:gen/java
-
-# c-capnproto needs some empty headers
-gen/c/include/c++.capnp.h gen/c/include/java.capnp.h:
-	mkdir -p gen/c/include
-	touch '$@'

cereal/SConscript

@@ -0,0 +1,65 @@
+Import('env', 'arch', 'zmq')
+
+gen_dir = Dir('gen')
+messaging_dir = Dir('messaging')
+
+# TODO: remove src-prefix and cereal from command string. can we set working directory?
+env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
+env.Command(
+  ['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'],
+  ['car.capnp', 'log.capnp'],
+  'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/')
+import shutil
+if shutil.which('capnpc-java'):
+  env.Command(
+    ['gen/java/Car.java', 'gen/java/Log.java'],
+    ['car.capnp', 'log.capnp'],
+    'capnpc $SOURCES --src-prefix=cereal -o java:' + gen_dir.path + '/java/')
+
+# TODO: remove non shared cereal and messaging
+cereal_objects = env.SharedObject([
+    'gen/cpp/car.capnp.c++',
+    'gen/cpp/log.capnp.c++',
+  ])
+
+env.Library('cereal', cereal_objects)
+env.SharedLibrary('cereal_shared', cereal_objects)
+
+cereal_dir = Dir('.')
+services_h = env.Command(
+  ['services.h'],
+  ['service_list.yaml', 'services.py'],
+  'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+
+messaging_objects = env.SharedObject([
+  'messaging/messaging.cc',
+  'messaging/impl_zmq.cc',
+  'messaging/impl_msgq.cc',
+  'messaging/msgq.cc',
+  'messaging/socketmaster.cc',
+])
+
+messaging_lib = env.Library('messaging', messaging_objects)
+Depends('messaging/impl_zmq.cc', services_h)
+
+# note, this rebuilds the deps shared, zmq is statically linked to make APK happy
+# TODO: get APK to load system zmq to remove the static link
+if arch == "aarch64":
+  zmq_static = FindFile("libzmq.a", "/usr/lib")
+  shared_lib_shared_lib = [zmq_static, 'm', 'stdc++', "gnustl_shared", "kj", "capnp"]
+  env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
+
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
+Depends('messaging/bridge.cc', services_h)
+
+# different target?
+#env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=[messaging_lib, 'zmq'])
+
+
+env.Command(['messaging/messaging_pyx.so'],
+  [messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'],
+  "cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace")
+
+
+if GetOption('test'):
+  env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib])

cereal/__init__.py

@@ -1,3 +1,4 @@
+# pylint: skip-file
 import os
 import capnp
 

cereal/car.capnp

@@ -13,16 +13,16 @@ struct CarEvent @0x9b1657f34caf3ad3 {
   name @0 :EventName;
   enable @1 :Bool;
   noEntry @2 :Bool;
-  warning @3 :Bool;
+  warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
   userDisable @4 :Bool;
   softDisable @5 :Bool;
   immediateDisable @6 :Bool;
   preEnable @7 :Bool;
-  permanent @8 :Bool;
+  permanent @8 :Bool; # alerts presented regardless of openpilot state
 
   enum EventName @0xbaa8c5d505f727de {
     # TODO: copy from error list
-    commIssue @0;
+    canError @0;
     steerUnavailable @1;
     brakeUnavailable @2;
     gasUnavailable @3;
@@ -37,32 +37,81 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     buttonEnable @12;
     pedalPressed @13;
     cruiseDisabled @14;
-    radarCommIssue @15;
-    dataNeeded @16;
+    radarCanError @15;
+    dataNeededDEPRECATED @16;
     speedTooLow @17;
     outOfSpace @18;
     overheat @19;
-    calibrationInProgress @20;
+    calibrationIncomplete @20;
     calibrationInvalid @21;
     controlsMismatch @22;
     pcmEnable @23;
     pcmDisable @24;
     noTarget @25;
     radarFault @26;
-    modelCommIssue @27;
+    modelCommIssueDEPRECATED @27;
     brakeHold @28;
     parkBrake @29;
     manualRestart @30;
     lowSpeedLockout @31;
     plannerError @32;
-    ipasOverride @33;
+    ipasOverrideDEPRECATED @33;
     debugAlert @34;
     steerTempUnavailableMute @35;
     resumeRequired @36;
     preDriverDistracted @37;
     promptDriverDistracted @38;
     driverDistracted @39;
-    geofence @40;
+    geofenceDEPRECATED @40;
+    driverMonitorOnDEPRECATED @41;
+    driverMonitorOffDEPRECATED @42;
+    preDriverUnresponsive @43;
+    promptDriverUnresponsive @44;
+    driverUnresponsive @45;
+    belowSteerSpeed @46;
+    calibrationProgressDEPRECATED @47;
+    lowBattery @48;
+    invalidGiraffeHondaDEPRECATED @49;
+    vehicleModelInvalid @50;
+    controlsFailed @51;
+    sensorDataInvalid @52;
+    commIssue @53;
+    tooDistracted @54;
+    posenetInvalid @55;
+    soundsUnavailable @56;
+    preLaneChangeLeft @57;
+    preLaneChangeRight @58;
+    laneChange @59;
+    invalidGiraffeToyota @60;
+    internetConnectivityNeeded @61;
+    communityFeatureDisallowed @62;
+    lowMemory @63;
+    stockAeb @64;
+    ldw @65;
+    carUnrecognized @66;
+    radarCommIssue @67;
+    driverMonitorLowAcc @68;
+    invalidLkasSetting @69;
+    speedTooHigh @70;
+    laneChangeBlocked @71;
+    relayMalfunction @72;
+    gasPressed @73;
+    stockFcw @74;
+    startup @75;
+    startupNoCar @76;
+    startupNoControl @77;
+    startupMaster @78;
+    fcw @79;
+    steerSaturated @80;
+    whitePandaUnsupported @81;
+    startupWhitePanda @82;
+    canErrorPersistentDEPRECATED @83;
+    belowEngageSpeed @84;
+    noGps @85;
+    focusRecoverActive @86;
+    wrongCruiseMode @87;
+    neosUpdateRequired @88;
+    modeldLagging @89;
   }
 }
 
@@ -91,10 +140,17 @@ struct CarState {
   brakeLights @19 :Bool;
 
   # steering wheel
-  steeringAngle @7 :Float32;   # deg
-  steeringRate @15 :Float32;   # deg/s
-  steeringTorque @8 :Float32;  # TODO: standardize units
-  steeringPressed @9 :Bool;    # if the user is using the steering wheel
+  steeringAngle @7 :Float32;       # deg
+  steeringRate @15 :Float32;       # deg/s
+  steeringTorque @8 :Float32;      # TODO: standardize units
+  steeringTorqueEps @27 :Float32;  # TODO: standardize units
+  steeringPressed @9 :Bool;        # if the user is using the steering wheel
+  steeringRateLimited @29 :Bool;   # if the torque is limited by the rate limiter
+  steerWarning @35 :Bool;          # temporary steer unavailble
+  steerError @36 :Bool;            # permanent steer error
+  stockAeb @30 :Bool;
+  stockFcw @31 :Bool;
+  espDisabled @32 :Bool;
 
   # cruise state
   cruiseState @10 :CruiseState;
@@ -111,10 +167,18 @@ struct CarState {
   # lock info
   doorOpen @24 :Bool;
   seatbeltUnlatched @25 :Bool;
+  canValid @26 :Bool;
+
+  # clutch (manual transmission only)
+  clutchPressed @28 :Bool;
 
   # which packets this state came from
   canMonoTimes @12: List(UInt64);
 
+  # blindspot sensors
+  leftBlindspot @33 :Bool; # Is there something blocking the left lane change
+  rightBlindspot @34 :Bool; # Is there something blocking the right lane change
+
   struct WheelSpeeds {
     # optional wheel speeds
     fl @0 :Float32;
@@ -129,6 +193,7 @@ struct CarState {
     available @2 :Bool;
     speedOffset @3 :Float32;
     standstill @4 :Bool;
+    nonAdaptive @5 :Bool;
   }
 
   enum GearShifter {
@@ -140,9 +205,10 @@ struct CarState {
     sport @5;
     low @6;
     brake @7;
+    eco @8;
+    manumatic @9;
   }
 
-
   # send on change
   struct ButtonEvent {
     pressed @0 :Bool;
@@ -158,13 +224,16 @@ struct CarState {
       altButton1 @6;
       altButton2 @7;
       altButton3 @8;
+      setCruise @9;
+      resumeCruise @10;
+      gapAdjustCruise @11;
     }
   }
 }
 
 # ******* radar state @ 20hz *******
 
-struct RadarState {
+struct RadarData @0x888ad6581cf0aacb {
   errors @0 :List(Error);
   points @1 :List(RadarPoint);
 
@@ -172,7 +241,7 @@ struct RadarState {
   canMonoTimes @2 :List(UInt64);
 
   enum Error {
-    commIssue @0;
+    canError @0;
     fault @1;
     wrongConfig @2;
   }
@@ -235,6 +304,10 @@ struct CarControl {
     leadVisible @3: Bool;
     visualAlert @4: VisualAlert;
     audibleAlert @5: AudibleAlert;
+    rightLaneVisible @6: Bool;
+    leftLaneVisible @7: Bool;
+    rightLaneDepart @8: Bool;
+    leftLaneDepart @9: Bool;
 
     enum VisualAlert {
       # these are the choices from the Honda
@@ -246,19 +319,21 @@ struct CarControl {
       wrongGear @4;
       seatbeltUnbuckled @5;
       speedTooHigh @6;
+      ldw @7;
     }
 
     enum AudibleAlert {
       # these are the choices from the Honda
       # map as good as you can for your car
       none @0;
-      beepSingle @1;
-      beepTriple @2;
-      beepRepeated @3;
-      chimeSingle @4;
-      chimeDouble @5;
-      chimeRepeated @6;
-      chimeContinuous @7;
+      chimeEngage @1;
+      chimeDisengage @2;
+      chimeError @3;
+      chimeWarning1 @4;
+      chimeWarning2 @5;
+      chimeWarningRepeat @6;
+      chimePrompt @7;
+      chimeWarning2Repeat @8;
     }
   }
 }
@@ -267,84 +342,191 @@ struct CarControl {
 
 struct CarParams {
   carName @0 :Text;
-  radarNameDEPRECATED @1 :Text;
-  carFingerprint @2 :Text;
+  carFingerprint @1 :Text;
 
-  enableSteerDEPRECATED @3 :Bool;
-  enableGasInterceptor @4 :Bool;
-  enableBrakeDEPRECATED @5 :Bool;
-  enableCruise @6 :Bool;
-  enableCamera @26 :Bool;
-  enableDsu @27 :Bool; # driving support unit
-  enableApgs @28 :Bool; # advanced parking guidance system
+  enableGasInterceptor @2 :Bool;
+  enableCruise @3 :Bool;
+  enableCamera @4 :Bool;
+  enableDsu @5 :Bool; # driving support unit
+  enableApgs @6 :Bool; # advanced parking guidance system
 
-  minEnableSpeed @17 :Float32;
-  safetyModel @18 :Int16;
-  safetyParam @41 :Int16;
+  minEnableSpeed @7 :Float32;
+  minSteerSpeed @8 :Float32;
+  safetyModel @9 :SafetyModel;
+  safetyModelPassive @42 :SafetyModel = silent;
+  safetyParam @10 :Int16;
 
-  steerMaxBP @19 :List(Float32);
-  steerMaxV @20 :List(Float32);
-  gasMaxBP @21 :List(Float32);
-  gasMaxV @22 :List(Float32);
-  brakeMaxBP @23 :List(Float32);
-  brakeMaxV @24 :List(Float32);
+  steerMaxBP @11 :List(Float32);
+  steerMaxV @12 :List(Float32);
+  gasMaxBP @13 :List(Float32);
+  gasMaxV @14 :List(Float32);
+  brakeMaxBP @15 :List(Float32);
+  brakeMaxV @16 :List(Float32);
 
-  longPidDeadzoneBP @32 :List(Float32);
-  longPidDeadzoneV @33 :List(Float32);
+  # things about the car in the manual
+  mass @17 :Float32;             # [kg] running weight
+  wheelbase @18 :Float32;        # [m] distance from rear to front axle
+  centerToFront @19 :Float32;   # [m] GC distance to front axle
+  steerRatio @20 :Float32;       # [] ratio between front wheels and steering wheel angles
+  steerRatioRear @21 :Float32;  # [] rear steering ratio wrt front steering (usually 0)
 
-  enum SafetyModels {
-    # does NOT match board setting
-    noOutput @0;
-    honda @1;
+  # things we can derive
+  rotationalInertia @22 :Float32;    # [kg*m2] body rotational inertia
+  tireStiffnessFront @23 :Float32;   # [N/rad] front tire coeff of stiff
+  tireStiffnessRear @24 :Float32;    # [N/rad] rear tire coeff of stiff
+
+  longitudinalTuning @25 :LongitudinalPIDTuning;
+  lateralParams @48 :LateralParams;
+  lateralTuning :union {
+    pid @26 :LateralPIDTuning;
+    indi @27 :LateralINDITuning;
+    lqr @40 :LateralLQRTuning;
+  }
+
+  steerLimitAlert @28 :Bool;
+  steerLimitTimer @47 :Float32;  # time before steerLimitAlert is issued
+
+  vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state
+  directAccelControl @30 :Bool; # Does the car have direct accel control or just gas/brake
+  stoppingControl @31 :Bool; # Does the car allows full control even at lows speeds when stopping
+  startAccel @32 :Float32; # Required acceleraton to overcome creep braking
+  steerRateCost @33 :Float32; # Lateral MPC cost on steering rate
+  steerControlType @34 :SteerControlType;
+  radarOffCan @35 :Bool; # True when radar objects aren't visible on CAN
+
+  steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds
+  openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control?
+  carVin @38 :Text; # VIN number queried during fingerprinting
+  isPandaBlack @39: Bool;
+  dashcamOnly @41: Bool;
+  transmissionType @43 :TransmissionType;
+  carFw @44 :List(CarFw);
+  radarTimeStep @45: Float32 = 0.05;  # time delta between radar updates, 20Hz is very standard
+  communityFeature @46: Bool;  # true if a community maintained feature is detected
+  fingerprintSource @49: FingerprintSource;
+  networkLocation @50 :NetworkLocation;  # Where Panda/C2 is integrated into the car's CAN network
+
+  struct LateralParams {
+    torqueBP @0 :List(Int32);
+    torqueV @1 :List(Int32);
+  }
+
+  struct LateralPIDTuning {
+    kpBP @0 :List(Float32);
+    kpV @1 :List(Float32);
+    kiBP @2 :List(Float32);
+    kiV @3 :List(Float32);
+    kf @4 :Float32;
+  }
+
+  struct LongitudinalPIDTuning {
+    kpBP @0 :List(Float32);
+    kpV @1 :List(Float32);
+    kiBP @2 :List(Float32);
+    kiV @3 :List(Float32);
+    deadzoneBP @4 :List(Float32);
+    deadzoneV @5 :List(Float32);
+  }
+
+  struct LateralINDITuning {
+    outerLoopGain @0 :Float32;
+    innerLoopGain @1 :Float32;
+    timeConstant @2 :Float32;
+    actuatorEffectiveness @3 :Float32;
+  }
+
+  struct LateralLQRTuning {
+    scale @0 :Float32;
+    ki @1 :Float32;
+    dcGain @2 :Float32;
+
+    # State space system
+    a @3 :List(Float32);
+    b @4 :List(Float32);
+    c @5 :List(Float32);
+
+    k @6 :List(Float32);  # LQR gain
+    l @7 :List(Float32);  # Kalman gain
+  }
+
+  enum SafetyModel {
+    silent @0;
+    hondaNidec @1;
     toyota @2;
     elm327 @3;
     gm @4;
-    hondaBosch @5;
+    hondaBoschGiraffe @5;
     ford @6;
     cadillac @7;
+    hyundai @8;
+    chrysler @9;
+    tesla @10;
+    subaru @11;
+    gmPassive @12;
+    mazda @13;
+    nissan @14;
+    volkswagen @15;
+    toyotaIpas @16;
+    allOutput @17;
+    gmAscm @18;
+    noOutput @19;  # like silent but without silent CAN TXs
+    hondaBoschHarness @20;
+    volkswagenPq @21;
+    subaruLegacy @22;  # pre-Global platform
+    hyundaiLegacy @23;
   }
 
-  # things about the car in the manual
-  mass @7 :Float32;             # [kg] running weight
-  wheelbase @8 :Float32;        # [m] distance from rear to front axle
-  centerToFront @9 :Float32;   # [m] GC distance to front axle
-  steerRatio @10 :Float32;       # [] ratio between front wheels and steering wheel angles
-  steerRatioRear @11 :Float32;  # [] rear steering ratio wrt front steering (usually 0)
-
-  # things we can derive
-  rotationalInertia @12 :Float32;    # [kg*m2] body rotational inertia
-  tireStiffnessFront @13 :Float32;   # [N/rad] front tire coeff of stiff
-  tireStiffnessRear @14 :Float32;    # [N/rad] rear tire coeff of stiff
-
-  # Kp and Ki for the lateral control
-  steerKpBP @42 :List(Float32);
-  steerKpV @43 :List(Float32);
-  steerKiBP @44 :List(Float32);
-  steerKiV @45 :List(Float32);
-  steerKpDEPRECATED @15 :Float32;
-  steerKiDEPRECATED @16 :Float32;
-  steerKf @25 :Float32;
-
-  # Kp and Ki for the longitudinal control
-  longitudinalKpBP @36 :List(Float32);
-  longitudinalKpV @37 :List(Float32);
-  longitudinalKiBP @38 :List(Float32);
-  longitudinalKiV @39 :List(Float32);
-
-  steerLimitAlert @29 :Bool;
-
-  vEgoStopping @30 :Float32; # Speed at which the car goes into stopping state
-  directAccelControl @31 :Bool; # Does the car have direct accel control or just gas/brake
-  stoppingControl @34 :Bool; # Does the car allows full control even at lows speeds when stopping
-  startAccel @35 :Float32; # Required acceleraton to overcome creep braking
-  steerRateCost @40 :Float32; # Lateral MPC cost on steering rate
-  steerControlType @46 :SteerControlType;
-  radarOffCan @47 :Bool; # True when radar objects aren't visible on CAN
-
-  steerActuatorDelay @48 :Float32; # Steering wheel actuator delay in seconds
-
   enum SteerControlType {
     torque @0;
     angle @1;
   }
+
+  enum TransmissionType {
+    unknown @0;
+    automatic @1;  # Traditional auto, including DSG
+    manual @2;  # True "stick shift" only
+    direct @3;  # Electric vehicle or other direct drive
+  }
+
+  struct CarFw {
+    ecu @0 :Ecu;
+    fwVersion @1 :Data;
+    address @2: UInt32;
+    subAddress @3: UInt8;
+  }
+
+  enum Ecu {
+    eps @0;
+    esp @1;
+    fwdRadar @2;
+    fwdCamera @3;
+    engine @4;
+    unknown @5;
+    transmission @8; # Transmission Control Module
+    srs @9; # airbag
+    gateway @10; # can gateway
+    hud @11; # heads up display
+    combinationMeter @12; # instrument cluster
+
+    # Toyota only
+    dsu @6;
+    apgs @7;
+
+    # Honda only
+    vsa @13; # Vehicle Stability Assist
+    programmedFuelInjection @14;
+    electricBrakeBooster @15;
+    shiftByWire @16;
+  }
+
+  enum FingerprintSource {
+    can @0;
+    fw @1;
+    fixed @2;
+  }
+
+  enum NetworkLocation {
+    fwdCamera @0;  # Standard/default integration at LKAS camera
+    gateway @1;    # Integration at vehicle's CAN gateway
+  }
 }

cereal/log.capnp

@@ -21,6 +21,8 @@ struct Map(Key, Value) {
 
 struct InitData {
   kernelArgs @0 :List(Text);
+  kernelVersion @15 :Text;
+
   gctx @1 :Text;
   dongleId @2 :Text;
 
@@ -32,6 +34,7 @@ struct InitData {
 
   androidBuildInfo @5 :AndroidBuildInfo;
   androidSensors @6 :List(AndroidSensor);
+  androidProperties @16 :Map(Text, Text);
   chffrAndroidExtra @7 :ChffrAndroidExtra;
   iosBuildInfo @14 :IosBuildInfo;
 
@@ -39,6 +42,7 @@ struct InitData {
 
   dirty @9 :Bool;
   passive @12 :Bool;
+  params @17 :Map(Text, Text);
 
   enum DeviceType {
     unknown @0;
@@ -122,6 +126,11 @@ struct FrameData {
   lensErr @13 :Float32;
   lensTruePos @14 :Float32;
   image @6 :Data;
+  gainFrac @15 :Float32;
+  focusVal @16 :List(Int16);
+  focusConf @17 :List(UInt8);
+  sharpnessScore @18 :List(UInt16);
+  recoverState @19 :Int32;
 
   frameType @7 :FrameType;
   timestampSof @8 :UInt64;
@@ -133,6 +142,7 @@ struct FrameData {
     unknown @0;
     neo @1;
     chffrAndroid @2;
+    front @3;
   }
 
   struct AndroidCaptureResult {
@@ -146,6 +156,12 @@ struct FrameData {
   }
 }
 
+struct Thumbnail {
+  frameId @0 :UInt32;
+  timestampEof @1 :UInt64;
+  thumbnail @2 :Data;
+}
+
 struct GPSNMEAData {
   timestamp @0 :Int64;
   localWallTime @1 :UInt64;
@@ -183,6 +199,10 @@ struct SensorEventData {
     iOS @1;
     fiber @2;
     velodyne @3;  # Velodyne IMU
+    bno055 @4;    # Bosch accelerometer
+    lsm6ds3 @5;   # accelerometer (c2)
+    bmp280 @6;    # barometer (c2)
+    mmc3416x @7;  # magnetometer (c2)
   }
 }
 
@@ -254,18 +274,29 @@ struct ThermalData {
   mem @4 :UInt16;
   gpu @5 :UInt16;
   bat @6 :UInt32;
+  pa0 @21 :UInt16;
 
   # not thermal
   freeSpace @7 :Float32;
   batteryPercent @8 :Int16;
   batteryStatus @9 :Text;
+  batteryCurrent @15 :Int32;
+  batteryVoltage @16 :Int32;
   usbOnline @12 :Bool;
+  networkType @22 :NetworkType;
+  offroadPowerUsage @23 :UInt32;  # Power usage since going offroad in uWh
+  networkStrength @24 :NetworkStrength;
 
   fanSpeed @10 :UInt16;
   started @11 :Bool;
   startedTs @13 :UInt64;
 
   thermalStatus @14 :ThermalStatus;
+  chargingError @17 :Bool;
+  chargingDisabled @18 :Bool;
+
+  memUsedPercent @19 :Int8;
+  cpuPerc @20 :Int8;
 
   enum ThermalStatus {
     green @0;   # all processes run
@@ -273,17 +304,94 @@ struct ThermalData {
     red @2;     # no engage, will disengage
     danger @3;  # immediate process shutdown
   }
+
+  enum NetworkType {
+    none @0;
+    wifi @1;
+    cell2G @2;
+    cell3G @3;
+    cell4G @4;
+    cell5G @5;
+  }
+
+  enum NetworkStrength {
+    unknown @0;
+    poor @1;
+    moderate @2;
+    good @3;
+    great @4;
+  }
 }
 
 struct HealthData {
   # from can health
   voltage @0 :UInt32;
   current @1 :UInt32;
-  started @2 :Bool;
+  ignitionLine @2 :Bool;
   controlsAllowed @3 :Bool;
   gasInterceptorDetected @4 :Bool;
-  startedSignalDetected @5 :Bool;
-  isGreyPanda @6 :Bool;
+  startedSignalDetectedDeprecated @5 :Bool;
+  hasGps @6 :Bool;
+  canSendErrs @7 :UInt32;
+  canFwdErrs @8 :UInt32;
+  canRxErrs @19 :UInt32;
+  gmlanSendErrs @9 :UInt32;
+  hwType @10 :HwType;
+  fanSpeedRpm @11 :UInt16;
+  usbPowerMode @12 :UsbPowerMode;
+  ignitionCan @13 :Bool;
+  safetyModel @14 :Car.CarParams.SafetyModel;
+  faultStatus @15 :FaultStatus;
+  powerSaveEnabled @16 :Bool;
+  uptime @17 :UInt32;
+  faults @18 :List(FaultType);
+
+  enum FaultStatus {
+    none @0;
+    faultTemp @1;
+    faultPerm @2;
+  }
+
+  enum FaultType {
+    relayMalfunction @0;
+    unusedInterruptHandled @1;
+    interruptRateCan1 @2;
+    interruptRateCan2 @3;
+    interruptRateCan3 @4;
+    interruptRateTach @5;
+    interruptRateGmlan @6;
+    interruptRateInterrupts @7;
+    interruptRateSpiDma @8;
+    interruptRateSpiCs @9;
+    interruptRateUart1 @10;
+    interruptRateUart2 @11;
+    interruptRateUart3 @12;
+    interruptRateUart5 @13;
+    interruptRateUartDma @14;
+    interruptRateUsb @15;
+    interruptRateTim1 @16;
+    interruptRateTim3 @17;
+    registerDivergent @18;
+    interruptRateKlineInit @19;
+    # Update max fault type in boardd when adding faults
+  }
+
+  enum HwType {
+    unknown @0;
+    whitePanda @1;
+    greyPanda @2;
+    blackPanda @3;
+    pedal @4;
+    uno @5;
+    dos @6;
+  }
+
+  enum UsbPowerMode {
+    none @0;
+    client @1;
+    cdp @2;
+    dcp @3;
+  }
 }
 
 struct LiveUI {
@@ -293,12 +401,12 @@ struct LiveUI {
   awarenessStatus @3 :Float32;
 }
 
-struct Live20Data {
+struct RadarState @0x9a185389d6fdd05f {
   canMonoTimes @10 :List(UInt64);
   mdMonoTime @6 :UInt64;
   ftMonoTimeDEPRECATED @7 :UInt64;
-  l100MonoTime @11 :UInt64;
-  radarErrors @12 :List(Car.RadarState.Error);
+  controlsStateMonoTime @11 :UInt64;
+  radarErrors @12 :List(Car.RadarData.Error);
 
   # all deprecated
   warpMatrixDEPRECATED @0 :List(Float32);
@@ -324,18 +432,27 @@ struct Live20Data {
     aLeadK @9 :Float32;
     fcw @10 :Bool;
     status @11 :Bool;
+    aLeadTau @12 :Float32;
+    modelProb @13 :Float32;
+    radar @14 :Bool;
   }
 }
 
 struct LiveCalibrationData {
+  # deprecated
   warpMatrix @0 :List(Float32);
+  # camera_frame_from_model_frame
   warpMatrix2 @5 :List(Float32);
+  warpMatrixBig @6 :List(Float32);
   calStatus @1 :Int8;
   calCycle @2 :Int32;
   calPerc @3 :Int8;
 
-  # Maps car space to normalized image space.
+  # view_frame_from_road_frame
+  # ui's is inversed needs new
   extrinsicMatrix @4 :List(Float32);
+  # the direction of travel vector in device frame
+  rpyCalib @7 :List(Float32);
 }
 
 struct LiveTracks {
@@ -351,14 +468,15 @@ struct LiveTracks {
   oncoming @9 :Bool;
 }
 
-struct Live100Data {
+struct ControlsState @0x97ff69c53601abf1 {
   canMonoTimeDEPRECATED @16 :UInt64;
   canMonoTimes @21 :List(UInt64);
-  l20MonoTimeDEPRECATED @17 :UInt64;
+  radarStateMonoTimeDEPRECATED @17 :UInt64;
   mdMonoTimeDEPRECATED @18 :UInt64;
   planMonoTime @28 :UInt64;
+  pathPlanMonoTime @50 :UInt64;
 
-  state @31 :ControlState;
+  state @31 :OpenpilotState;
   vEgo @0 :Float32;
   vEgoRaw @32 :Float32;
   aEgoDEPRECATED @1 :Float32;
@@ -370,9 +488,9 @@ struct Live100Data {
   ufAccelCmd @33 :Float32;
   yActualDEPRECATED @6 :Float32;
   yDesDEPRECATED @7 :Float32;
-  upSteer @8 :Float32;
-  uiSteer @9 :Float32;
-  ufSteer @34 :Float32;
+  upSteerDEPRECATED @8 :Float32;
+  uiSteerDEPRECATED @9 :Float32;
+  ufSteerDEPRECATED @34 :Float32;
   aTargetMinDEPRECATED @10 :Float32;
   aTargetMaxDEPRECATED @11 :Float32;
   aTarget @35 :Float32;
@@ -382,6 +500,9 @@ struct Live100Data {
   curvature @37 :Float32;       # path curvature from vehicle model
   hudLeadDEPRECATED @14 :Int32;
   cumLagMs @15 :Float32;
+  startMonoTime @48 :UInt64;
+  mapValid @49 :Bool;
+  forceDecel @51 :Bool;
 
   enabled @19 :Bool;
   active @36 :Bool;
@@ -395,12 +516,29 @@ struct Live100Data {
   alertStatus @38 :AlertStatus;
   alertSize @39 :AlertSize;
   alertBlinkingRate @42 :Float32;
+  alertType @44 :Text;
+  alertSoundDEPRECATED @45 :Text;
+  alertSound @56 :Car.CarControl.HUDControl.AudibleAlert;
   awarenessStatus @26 :Float32;
-  angleOffset @27 :Float32;
+  angleModelBiasDEPRECATED @27 :Float32;
   gpsPlannerActive @40 :Bool;
   engageable @41 :Bool;  # can OP be engaged?
+  driverMonitoringOn @43 :Bool;
 
-  enum ControlState {
+  # maps
+  vCurvature @46 :Float32;
+  decelForTurn @47 :Bool;
+
+  decelForModel @54 :Bool;
+  canErrorCounter @57 :UInt32;
+
+  lateralControlState :union {
+    indiState @52 :LateralINDIState;
+    pidState @53 :LateralPIDState;
+    lqrState @55 :LateralLQRState;
+  }
+
+  enum OpenpilotState @0xdbe58b96d2d1ac61 {
     disabled @0;
     preEnabled @1;
     enabled @2;
@@ -427,6 +565,40 @@ struct Live100Data {
     full @3;    # full screen
   }
 
+  struct LateralINDIState {
+    active @0 :Bool;
+    steerAngle @1 :Float32;
+    steerRate @2 :Float32;
+    steerAccel @3 :Float32;
+    rateSetPoint @4 :Float32;
+    accelSetPoint @5 :Float32;
+    accelError @6 :Float32;
+    delayedOutput @7 :Float32;
+    delta @8 :Float32;
+    output @9 :Float32;
+    saturated @10 :Bool;
+  }
+
+  struct LateralPIDState {
+    active @0 :Bool;
+    steerAngle @1 :Float32;
+    steerRate @2 :Float32;
+    angleError @3 :Float32;
+    p @4 :Float32;
+    i @5 :Float32;
+    f @6 :Float32;
+    output @7 :Float32;
+    saturated @8 :Bool;
+   }
+
+  struct LateralLQRState {
+    active @0 :Bool;
+    steerAngle @1 :Float32;
+    i @2 :Float32;
+    output @3 :Float32;
+    lqrOutput @4 :Float32;
+    saturated @5 :Bool;
+   }
 }
 
 struct LiveEventData {
@@ -436,6 +608,9 @@ struct LiveEventData {
 
 struct ModelData {
   frameId @0 :UInt32;
+  frameAge @12 :UInt32;
+  frameDropPerc @13 :Float32;
+  timestampEof @9 :UInt64;
 
   path @1 :PathData;
   leftLane @2 :PathData;
@@ -444,17 +619,30 @@ struct ModelData {
   freePath @6 :List(Float32);
 
   settings @5 :ModelSettings;
+  leadFuture @7 :LeadData;
+  speed @8 :List(Float32);
+  meta @10 :MetaData;
+  longitudinal @11 :LongitudinalData;
 
   struct PathData {
     points @0 :List(Float32);
     prob @1 :Float32;
     std @2 :Float32;
+    stds @3 :List(Float32);
+    poly @4 :List(Float32);
+    validLen @5 :Float32;
   }
 
   struct LeadData {
     dist @0 :Float32;
     prob @1 :Float32;
     std @2 :Float32;
+    relVel @3 :Float32;
+    relVelStd @4 :Float32;
+    relY @5 :Float32;
+    relYStd @6 :Float32;
+    relA @7 :Float32;
+    relAStd @8 :Float32;
   }
 
   struct ModelSettings {
@@ -466,6 +654,21 @@ struct ModelData {
     yuvCorrection @5 :List(Float32);
     inputTransform @6 :List(Float32);
   }
+
+  struct MetaData {
+    engagedProb @0 :Float32;
+    desirePrediction @1 :List(Float32);
+    brakeDisengageProb @2 :Float32;
+    gasDisengageProb @3 :Float32;
+    steerOverrideProb @4 :Float32;
+    desireState @5 :List(Float32);
+  }
+
+  struct LongitudinalData {
+    distances @2 :List(Float32);
+    speeds @0 :List(Float32);
+    accelerations @1 :List(Float32);
+  }
 }
 
 struct CalibrationFeatures {
@@ -516,16 +719,18 @@ struct LogRotate {
 
 struct Plan {
   mdMonoTime @9 :UInt64;
-  l20MonoTime @10 :UInt64;
-  events @13 :List(Car.CarEvent);
+  radarStateMonoTime @10 :UInt64;
+  commIssue @31 :Bool;
+
+  eventsDEPRECATED @13 :List(Car.CarEvent);
 
   # lateral, 3rd order polynomial
-  lateralValid @0 :Bool;
-  dPoly @1 :List(Float32);
-  laneWidth @11 :Float32;
+  lateralValidDEPRECATED @0 :Bool;
+  dPolyDEPRECATED @1 :List(Float32);
+  laneWidthDEPRECATED @11 :Float32;
 
   # longitudinal
-  longitudinalValid @2 :Bool;
+  longitudinalValidDEPRECATED @2 :Bool;
   vCruise @16 :Float32;
   aCruise @17 :Float32;
   vTarget @3 :Float32;
@@ -534,8 +739,14 @@ struct Plan {
   aTargetMinDEPRECATED @4 :Float32;
   aTargetMaxDEPRECATED @5 :Float32;
   aTarget @18 :Float32;
+
+  vStart @26 :Float32;
+  aStart @27 :Float32;
+
   jerkFactor @6 :Float32;
   hasLead @7 :Bool;
+  hasLeftLaneDEPRECATED @23 :Bool;
+  hasRightLaneDEPRECATED @24 :Bool;
   fcw @8 :Bool;
   longitudinalPlanSource @15 :LongitudinalPlanSource;
 
@@ -544,6 +755,16 @@ struct Plan {
 
   gpsPlannerActive @19 :Bool;
 
+  # maps
+  vCurvature @21 :Float32;
+  decelForTurn @22 :Bool;
+  mapValid @25 :Bool;
+  radarValid @28 :Bool;
+  radarCanError @30 :Bool;
+
+  processingDelay @29 :Float32;
+
+
   struct GpsTrajectory {
     x @0 :List(Float32);
     y @1 :List(Float32);
@@ -554,6 +775,106 @@ struct Plan {
     mpc1 @1;
     mpc2 @2;
     mpc3 @3;
+    model @4;
+  }
+}
+
+struct PathPlan {
+  laneWidth @0 :Float32;
+
+  dPoly @1 :List(Float32);
+  cPoly @2 :List(Float32);
+  cProb @3 :Float32;
+  lPoly @4 :List(Float32);
+  lProb @5 :Float32;
+  rPoly @6 :List(Float32);
+  rProb @7 :Float32;
+
+  angleSteers @8 :Float32; # deg
+  rateSteers @13 :Float32; # deg/s
+  mpcSolutionValid @9 :Bool;
+  paramsValid @10 :Bool;
+  modelValidDEPRECATED @12 :Bool;
+  angleOffset @11 :Float32;
+  sensorValid @14 :Bool;
+  commIssue @15 :Bool;
+  posenetValid @16 :Bool;
+  desire @17 :Desire;
+  laneChangeState @18 :LaneChangeState;
+  laneChangeDirection @19 :LaneChangeDirection;
+
+  enum Desire {
+    none @0;
+    turnLeft @1;
+    turnRight @2;
+    laneChangeLeft @3;
+    laneChangeRight @4;
+    keepLeft @5;
+    keepRight @6;
+  }
+
+  enum LaneChangeState {
+    off @0;
+    preLaneChange @1;
+    laneChangeStarting @2;
+    laneChangeFinishing @3;
+  }
+
+  enum LaneChangeDirection {
+    none @0;
+    left @1;
+    right @2;
+  }
+}
+
+struct LiveLocationKalman {
+
+  # More info on reference frames:
+  # https://github.com/commaai/openpilot/tree/master/common/transformations
+
+  positionECEF @0 : Measurement;
+  positionGeodetic @1 : Measurement;
+  velocityECEF @2 : Measurement;
+  velocityNED @3 : Measurement;
+  velocityDevice @4 : Measurement;
+  accelerationDevice @5: Measurement;
+
+
+  # These angles are all eulers and roll, pitch, yaw
+  # orientationECEF transforms to rot matrix: ecef_from_device
+  orientationECEF @6 : Measurement;
+  calibratedOrientationECEF @20 : Measurement;
+  orientationNED @7 : Measurement;
+  angularVelocityDevice @8 : Measurement;
+
+  # orientationNEDCalibrated transforms to rot matrix: NED_from_calibrated
+  orientationNEDCalibrated @9 : Measurement;
+
+  # Calibrated frame is simply device frame
+  # aligned with the vehicle
+  velocityCalibrated @10 : Measurement;
+  accelerationCalibrated @11 : Measurement;
+  angularVelocityCalibrated @12 : Measurement;
+
+  gpsWeek @13 :Int32;
+  gpsTimeOfWeek @14 :Float64;
+  status @15 :Status;
+  unixTimestampMillis @16 :Int64;
+  inputsOK @17 :Bool = true;
+  posenetOK @18 :Bool = true;
+  gpsOK @19 :Bool = true;
+  sensorsOK @21 :Bool = true;
+
+  enum Status {
+    uninitialized @0;
+    uncalibrated @1;
+    valid @2;
+  }
+
+  struct Measurement {
+    value @0 : List(Float64);
+    std @1 : List(Float64);
+    valid @2 : Bool;
   }
 }
 
@@ -1200,6 +1521,7 @@ struct UbloxGnss {
     measurementReport @0 :MeasurementReport;
     ephemeris @1 :Ephemeris;
     ionoData @2 :IonoData;
+    hwStatus @3 :HwStatus;
   }
 
   struct MeasurementReport {
@@ -1243,6 +1565,7 @@ struct UbloxGnss {
       carrierPhaseStdev @10 :Float32;
       # doppler standard deviation in Hz
       dopplerStdev @11 :Float32;
+      sigId @12 :UInt8;
 
       struct TrackingStatus {
         # pseudorange valid
@@ -1322,8 +1645,29 @@ struct UbloxGnss {
     healthValid @5 :Bool;
     ionoCoeffsValid @6 :Bool;
   }
-}
 
+  struct HwStatus {
+    noisePerMS @0 :UInt16;
+    agcCnt @1 :UInt16;
+    aStatus @2 :AntennaSupervisorState;
+    aPower @3 :AntennaPowerStatus;
+    jamInd @4 :UInt8;
+
+    enum AntennaSupervisorState {
+      init @0;
+      dontknow @1;
+      ok @2;
+      short @3;
+      open @4;
+    }
+
+    enum AntennaPowerStatus {
+      off @0;
+      on @1;
+      dontknow @2;
+    }
+  }
+}
 
 struct Clocks {
   bootTimeNanos @0 :UInt64;
@@ -1460,11 +1804,14 @@ struct UiLayoutState {
   activeApp @0 :App;
   sidebarCollapsed @1 :Bool;
   mapEnabled @2 :Bool;
+  mockEngaged @3 :Bool;
 
   enum App {
     home @0;
     music @1;
     nav @2;
+    settings @3;
+    none @4;
   }
 }
 
@@ -1530,14 +1877,112 @@ struct OrbKeyFrame {
   descriptors @3 :Data;
 }
 
-struct DriverMonitoring {
+struct DriverState {
   frameId @0 :UInt32;
-  descriptor @1 :List(Float32);
+  descriptorDEPRECATED @1 :List(Float32);
+  stdDEPRECATED @2 :Float32;
+  faceOrientation @3 :List(Float32);
+  facePosition @4 :List(Float32);
+  faceProb @5 :Float32;
+  leftEyeProb @6 :Float32;
+  rightEyeProb @7 :Float32;
+  leftBlinkProb @8 :Float32;
+  rightBlinkProb @9 :Float32;
+  irPwrDEPRECATED @10 :Float32;
+  faceOrientationStd @11 :List(Float32);
+  facePositionStd @12 :List(Float32);
+  sgProb @13 :Float32;
+}
+
+struct DMonitoringState {
+  # TODO: deprecate old fields in controlsState
+  events @0 :List(Car.CarEvent);
+  faceDetected @1 :Bool;
+  isDistracted @2 :Bool;
+  awarenessStatus @3 :Float32;
+  isRHD @4 :Bool;
+  rhdChecked @5 :Bool;
+  posePitchOffset @6 :Float32;
+  posePitchValidCount @7 :UInt32;
+  poseYawOffset @8 :Float32;
+  poseYawValidCount @9 :UInt32;
+  stepChange @10 :Float32;
+  awarenessActive @11 :Float32;
+  awarenessPassive @12 :Float32;
+  isLowStd @13 :Bool;
+  hiStdCount @14 :UInt32;
+  isPreview @15 :Bool;
+}
+
+struct Boot {
+  wallTimeNanos @0 :UInt64;
+  lastKmsg @1 :Data;
+  lastPmsg @2 :Data;
+}
+
+struct LiveParametersData {
+  valid @0 :Bool;
+  gyroBias @1 :Float32;
+  angleOffset @2 :Float32;
+  angleOffsetAverage @3 :Float32;
+  stiffnessFactor @4 :Float32;
+  steerRatio @5 :Float32;
+  sensorValid @6 :Bool;
+  yawRate @7 :Float32;
+  posenetSpeed @8 :Float32;
+  posenetValid @9 :Bool;
+}
+
+struct LiveMapData {
+  speedLimitValid @0 :Bool;
+  speedLimit @1 :Float32;
+  speedAdvisoryValid @12 :Bool;
+  speedAdvisory @13 :Float32;
+  speedLimitAheadValid @14 :Bool;
+  speedLimitAhead @15 :Float32;
+  speedLimitAheadDistance @16 :Float32;
+  curvatureValid @2 :Bool;
+  curvature @3 :Float32;
+  wayId @4 :UInt64;
+  roadX @5 :List(Float32);
+  roadY @6 :List(Float32);
+  lastGps @7: GpsLocationData;
+  roadCurvatureX @8 :List(Float32);
+  roadCurvature @9 :List(Float32);
+  distToTurn @10 :Float32;
+  mapValid @11 :Bool;
+}
+
+struct CameraOdometry {
+  frameId @4 :UInt32;
+  timestampEof @5 :UInt64;
+  trans @0 :List(Float32); # m/s in device frame
+  rot @1 :List(Float32); # rad/s in device frame
+  transStd @2 :List(Float32); # std m/s in device frame
+  rotStd @3 :List(Float32); # std rad/s in device frame
+}
+
+struct KalmanOdometry {
+  trans @0 :List(Float32); # m/s in device frame
+  rot @1 :List(Float32); # rad/s in device frame
+  transStd @2 :List(Float32); # std m/s in device frame
+  rotStd @3 :List(Float32); # std rad/s in device frame
+}
+
+struct Sentinel {
+  enum SentinelType {
+    endOfSegment @0;
+    endOfRoute @1;
+    startOfSegment @2;
+    startOfRoute @3;
+  }
+  type @0 :SentinelType;
 }
 
 struct Event {
   # in nanoseconds?
   logMonoTime @0 :UInt64;
+  valid @67 :Bool = true;
 
   union {
     initData @1 :InitData;
@@ -1546,20 +1991,20 @@ struct Event {
     sensorEventDEPRECATED @4 :SensorEventData;
     can @5 :List(CanData);
     thermal @6 :ThermalData;
-    live100 @7 :Live100Data;
+    controlsState @7 :ControlsState;
     liveEventDEPRECATED @8 :List(LiveEventData);
     model @9 :ModelData;
     features @10 :CalibrationFeatures;
     sensorEvents @11 :List(SensorEventData);
     health @12 :HealthData;
-    live20 @13 :Live20Data;
+    radarState @13 :RadarState;
     liveUIDEPRECATED @14 :LiveUI;
     encodeIdx @15 :EncodeIndex;
     liveTracks @16 :List(LiveTracks);
     sendcan @17 :List(CanData);
     logMessage @18 :Text;
     liveCalibration @19 :LiveCalibrationData;
-    androidLogEntry @20 :AndroidLogEntry;
+    androidLog @20 :AndroidLogEntry;
     gpsLocation @21 :GpsLocationData;
     carState @22 :Car.CarState;
     carControl @23 :Car.CarControl;
@@ -1590,7 +2035,7 @@ struct Event {
     gpsLocationExternal @48 :GpsLocationData;
     location @49 :LiveLocationData;
     uiNavigationEvent @50 :UiNavigationEvent;
-    liveLocationKalman @51 :LiveLocationData;
+    liveLocationKalmanDEPRECATED @51 :LiveLocationData;
     testJoystick @52 :Joystick;
     orbOdometry @53 :OrbOdometry;
     orbFeatures @54 :OrbFeatures;
@@ -1598,6 +2043,19 @@ struct Event {
     orbKeyFrame @56 :OrbKeyFrame;
     uiLayoutState @57 :UiLayoutState;
     orbFeaturesSummary @58 :OrbFeaturesSummary;
-    driverMonitoring @59 :DriverMonitoring;
+    driverState @59 :DriverState;
+    boot @60 :Boot;
+    liveParameters @61 :LiveParametersData;
+    liveMapData @62 :LiveMapData;
+    cameraOdometry @63 :CameraOdometry;
+    pathPlan @64 :PathPlan;
+    kalmanOdometry @65 :KalmanOdometry;
+    thumbnail @66: Thumbnail;
+    carEvents @68: List(Car.CarEvent);
+    carParams @69: Car.CarParams;
+    frontFrame @70: FrameData;
+    dMonitoringState @71: DMonitoringState;
+    liveLocationKalman @72 :LiveLocationKalman;
+    sentinel @73 :Sentinel;
   }
 }

cereal/messaging/.gitignore

@@ -0,0 +1,10 @@
+demo
+bridge
+test_runner
+*.o
+*.os
+*.d
+*.a
+*.so
+messaging_pyx.cpp
+build/

cereal/messaging/__init__.py

@@ -0,0 +1,225 @@
+# must be build with scons
+from .messaging_pyx import Context, Poller, SubSocket, PubSocket  # pylint: disable=no-name-in-module, import-error
+from .messaging_pyx import MultiplePublishersError, MessagingError  # pylint: disable=no-name-in-module, import-error
+import capnp
+
+from cereal import log
+from cereal.services import service_list
+
+assert MultiplePublishersError
+assert MessagingError
+
+# sec_since_boot is faster, but allow to run standalone too
+try:
+  from common.realtime import sec_since_boot
+except ImportError:
+  import time
+  sec_since_boot = time.time
+  print("Warning, using python time.time() instead of faster sec_since_boot")
+
+context = Context()
+
+def new_message(service=None, size=None):
+  dat = log.Event.new_message()
+  dat.logMonoTime = int(sec_since_boot() * 1e9)
+  dat.valid = True
+  if service is not None:
+    if size is None:
+      dat.init(service)
+    else:
+      dat.init(service, size)
+  return dat
+
+def pub_sock(endpoint):
+  sock = PubSocket()
+  sock.connect(context, endpoint)
+  return sock
+
+def sub_sock(endpoint, poller=None, addr="127.0.0.1", conflate=False, timeout=None):
+  sock = SubSocket()
+  addr = addr.encode('utf8')
+  sock.connect(context, endpoint, addr, conflate)
+
+  if timeout is not None:
+    sock.setTimeout(timeout)
+
+  if poller is not None:
+    poller.registerSocket(sock)
+  return sock
+
+
+def drain_sock_raw(sock, wait_for_one=False):
+  """Receive all message currently available on the queue"""
+  ret = []
+  while 1:
+    if wait_for_one and len(ret) == 0:
+      dat = sock.receive()
+    else:
+      dat = sock.receive(non_blocking=True)
+
+    if dat is None:
+      break
+
+    ret.append(dat)
+
+  return ret
+
+def drain_sock(sock, wait_for_one=False):
+  """Receive all message currently available on the queue"""
+  ret = []
+  while 1:
+    if wait_for_one and len(ret) == 0:
+      dat = sock.receive()
+    else:
+      dat = sock.receive(non_blocking=True)
+
+    if dat is None:  # Timeout hit
+      break
+
+    dat = log.Event.from_bytes(dat)
+    ret.append(dat)
+
+  return ret
+
+
+# TODO: print when we drop packets?
+def recv_sock(sock, wait=False):
+  """Same as drain sock, but only returns latest message. Consider using conflate instead."""
+  dat = None
+
+  while 1:
+    if wait and dat is None:
+      rcv = sock.receive()
+    else:
+      rcv = sock.receive(non_blocking=True)
+
+    if rcv is None:  # Timeout hit
+      break
+
+    dat = rcv
+
+  if dat is not None:
+    dat = log.Event.from_bytes(dat)
+
+  return dat
+
+def recv_one(sock):
+  dat = sock.receive()
+  if dat is not None:
+    dat = log.Event.from_bytes(dat)
+  return dat
+
+def recv_one_or_none(sock):
+  dat = sock.receive(non_blocking=True)
+  if dat is not None:
+    dat = log.Event.from_bytes(dat)
+  return dat
+
+def recv_one_retry(sock):
+  """Keep receiving until we get a message"""
+  while True:
+    dat = sock.receive()
+    if dat is not None:
+      return log.Event.from_bytes(dat)
+
+# TODO: This does not belong in messaging
+def get_one_can(logcan):
+  while True:
+    can = recv_one_retry(logcan)
+    if len(can.can) > 0:
+      return can
+
+class SubMaster():
+  def __init__(self, services, ignore_alive=None, addr="127.0.0.1"):
+    self.poller = Poller()
+    self.frame = -1
+    self.updated = {s: False for s in services}
+    self.rcv_time = {s: 0. for s in services}
+    self.rcv_frame = {s: 0 for s in services}
+    self.alive = {s: False for s in services}
+    self.sock = {}
+    self.freq = {}
+    self.data = {}
+    self.logMonoTime = {}
+    self.valid = {}
+
+    if ignore_alive is not None:
+      self.ignore_alive = ignore_alive
+    else:
+      self.ignore_alive = []
+
+    for s in services:
+      if addr is not None:
+        self.sock[s] = sub_sock(s, poller=self.poller, addr=addr, conflate=True)
+      self.freq[s] = service_list[s].frequency
+
+      try:
+        data = new_message(s)
+      except capnp.lib.capnp.KjException:  # pylint: disable=c-extension-no-member
+        # lists
+        data = new_message(s, 0)
+
+      self.data[s] = getattr(data, s)
+      self.logMonoTime[s] = 0
+      self.valid[s] = data.valid
+
+  def __getitem__(self, s):
+    return self.data[s]
+
+  def update(self, timeout=1000):
+    msgs = []
+    for sock in self.poller.poll(timeout):
+      msgs.append(recv_one_or_none(sock))
+    self.update_msgs(sec_since_boot(), msgs)
+
+  def update_msgs(self, cur_time, msgs):
+    # TODO: add optional input that specify the service to wait for
+    self.frame += 1
+    self.updated = dict.fromkeys(self.updated, False)
+    for msg in msgs:
+      if msg is None:
+        continue
+
+      s = msg.which()
+      self.updated[s] = True
+      self.rcv_time[s] = cur_time
+      self.rcv_frame[s] = self.frame
+      self.data[s] = getattr(msg, s)
+      self.logMonoTime[s] = msg.logMonoTime
+      self.valid[s] = msg.valid
+
+    for s in self.data:
+      # arbitrary small number to avoid float comparison. If freq is 0, we can skip the check
+      if self.freq[s] > 1e-5:
+        # alive if delay is within 10x the expected frequency
+        self.alive[s] = (cur_time - self.rcv_time[s]) < (10. / self.freq[s])
+      else:
+        self.alive[s] = True
+
+  def all_alive(self, service_list=None):
+    if service_list is None:  # check all
+      service_list = self.alive.keys()
+    return all(self.alive[s] for s in service_list if s not in self.ignore_alive)
+
+  def all_valid(self, service_list=None):
+    if service_list is None:  # check all
+      service_list = self.valid.keys()
+    return all(self.valid[s] for s in service_list)
+
+  def all_alive_and_valid(self, service_list=None):
+    if service_list is None:  # check all
+      service_list = self.alive.keys()
+    return self.all_alive(service_list=service_list) and self.all_valid(service_list=service_list)
+
+
+class PubMaster():
+  def __init__(self, services):
+    self.sock = {}
+    for s in services:
+      self.sock[s] = pub_sock(s)
+
+  def send(self, s, dat):
+    # accept either bytes or capnp builder
+    if not isinstance(dat, bytes):
+      dat = dat.to_bytes()
+    self.sock[s].send(dat)

cereal/messaging/bridge.cc

@@ -0,0 +1,64 @@
+#include <iostream>
+#include <string>
+#include <cassert>
+#include <csignal>
+#include <map>
+
+typedef void (*sighandler_t)(int sig);
+
+#include "services.h"
+
+#include "impl_msgq.hpp"
+#include "impl_zmq.hpp"
+
+void sigpipe_handler(int sig) {
+  assert(sig == SIGPIPE);
+  std::cout << "SIGPIPE received" << std::endl;
+}
+
+static std::vector<std::string> get_services() {
+  std::vector<std::string> name_list;
+
+  for (const auto& it : services) {
+    std::string name = it.name;
+    if (name == "plusFrame" || name == "uiLayoutState") continue;
+    name_list.push_back(name);
+  }
+
+  return name_list;
+}
+
+
+int main(void){
+  signal(SIGPIPE, (sighandler_t)sigpipe_handler);
+
+  auto endpoints = get_services();
+
+  std::map<SubSocket*, PubSocket*> sub2pub;
+
+  Context *zmq_context = new ZMQContext();
+  Context *msgq_context = new MSGQContext();
+  Poller *poller = new MSGQPoller();
+
+  for (auto endpoint: endpoints){
+    SubSocket * msgq_sock = new MSGQSubSocket();
+    msgq_sock->connect(msgq_context, endpoint, "127.0.0.1", false);
+    poller->registerSocket(msgq_sock);
+
+    PubSocket * zmq_sock = new ZMQPubSocket();
+    zmq_sock->connect(zmq_context, endpoint);
+
+    sub2pub[msgq_sock] = zmq_sock;
+  }
+
+
+  while (true){
+    for (auto sub_sock : poller->poll(100)){
+      Message * msg = sub_sock->receive();
+      if (msg == NULL) continue;
+      sub2pub[sub_sock]->sendMessage(msg);
+      delete msg;
+    }
+  }
+  return 0;
+}

cereal/messaging/impl_msgq.cc

@@ -0,0 +1,199 @@
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <cstdlib>
+#include <csignal>
+#include <cerrno>
+
+
+#include "impl_msgq.hpp"
+
+volatile sig_atomic_t msgq_do_exit = 0;
+
+void sig_handler(int signal) {
+  assert(signal == SIGINT || signal == SIGTERM);
+  msgq_do_exit = 1;
+}
+
+
+MSGQContext::MSGQContext() {
+}
+
+MSGQContext::~MSGQContext() {
+}
+
+void MSGQMessage::init(size_t sz) {
+  size = sz;
+  data = new char[size];
+}
+
+void MSGQMessage::init(char * d, size_t sz) {
+  size = sz;
+  data = new char[size];
+  memcpy(data, d, size);
+}
+
+void MSGQMessage::takeOwnership(char * d, size_t sz) {
+  size = sz;
+  data = d;
+}
+
+void MSGQMessage::close() {
+  if (size > 0){
+    delete[] data;
+  }
+  size = 0;
+}
+
+MSGQMessage::~MSGQMessage() {
+  this->close();
+}
+
+
+int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate){
+  assert(context);
+  assert(address == "127.0.0.1");
+
+  q = new msgq_queue_t;
+  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  if (r != 0){
+    return r;
+  }
+
+  msgq_init_subscriber(q);
+
+  if (conflate){
+    q->read_conflate = true;
+  }
+
+  timeout = -1;
+
+  return 0;
+}
+
+
+Message * MSGQSubSocket::receive(bool non_blocking){
+  msgq_do_exit = 0;
+
+  void (*prev_handler_sigint)(int);
+  void (*prev_handler_sigterm)(int);
+  if (!non_blocking){
+    prev_handler_sigint = std::signal(SIGINT, sig_handler);
+    prev_handler_sigterm = std::signal(SIGTERM, sig_handler);
+  }
+
+  msgq_msg_t msg;
+
+  MSGQMessage *r = NULL;
+
+  int rc = msgq_msg_recv(&msg, q);
+
+  // Hack to implement blocking read with a poller. Don't use this
+  while (!non_blocking && rc == 0 && msgq_do_exit == 0){
+    msgq_pollitem_t items[1];
+    items[0].q = q;
+
+    int t = (timeout != -1) ? timeout : 100;
+
+    int n = msgq_poll(items, 1, t);
+    rc = msgq_msg_recv(&msg, q);
+
+    // The poll indicated a message was ready, but the receive failed. Try again
+    if (n == 1 && rc == 0){
+      continue;
+    }
+
+    if (timeout != -1){
+      break;
+    }
+  }
+
+
+  if (!non_blocking){
+    std::signal(SIGINT, prev_handler_sigint);
+    std::signal(SIGTERM, prev_handler_sigterm);
+  }
+
+  errno = msgq_do_exit ? EINTR : 0;
+
+  if (rc > 0){
+    if (msgq_do_exit){
+      msgq_msg_close(&msg); // Free unused message on exit
+    } else {
+      r = new MSGQMessage;
+      r->takeOwnership(msg.data, msg.size);
+    }
+  }
+
+  return (Message*)r;
+}
+
+void MSGQSubSocket::setTimeout(int t){
+  timeout = t;
+}
+
+MSGQSubSocket::~MSGQSubSocket(){
+  if (q != NULL){
+    msgq_close_queue(q);
+    delete q;
+  }
+}
+
+int MSGQPubSocket::connect(Context *context, std::string endpoint){
+  assert(context);
+
+  q = new msgq_queue_t;
+  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  if (r != 0){
+    return r;
+  }
+
+  msgq_init_publisher(q);
+
+  return 0;
+}
+
+int MSGQPubSocket::sendMessage(Message *message){
+  msgq_msg_t msg;
+  msg.data = message->getData();
+  msg.size = message->getSize();
+
+  return msgq_msg_send(&msg, q);
+}
+
+int MSGQPubSocket::send(char *data, size_t size){
+  msgq_msg_t msg;
+  msg.data = data;
+  msg.size = size;
+
+  return msgq_msg_send(&msg, q);
+}
+
+MSGQPubSocket::~MSGQPubSocket(){
+  if (q != NULL){
+    msgq_close_queue(q);
+    delete q;
+  }
+}
+
+
+void MSGQPoller::registerSocket(SubSocket * socket){
+  assert(num_polls + 1 < MAX_POLLERS);
+  polls[num_polls].q = (msgq_queue_t*)socket->getRawSocket();
+
+  sockets.push_back(socket);
+  num_polls++;
+}
+
+std::vector<SubSocket*> MSGQPoller::poll(int timeout){
+  std::vector<SubSocket*> r;
+
+  msgq_poll(polls, num_polls, timeout);
+  for (size_t i = 0; i < num_polls; i++){
+    if (polls[i].revents){
+      r.push_back(sockets[i]);
+    }
+  }
+
+  return r;
+}

cereal/messaging/impl_msgq.hpp

@@ -0,0 +1,64 @@
+#pragma once
+#include "messaging.hpp"
+#include "msgq.hpp"
+#include <zmq.h>
+#include <string>
+
+#define MAX_POLLERS 128
+
+class MSGQContext : public Context {
+private:
+  void * context = NULL;
+public:
+  MSGQContext();
+  void * getRawContext() {return context;}
+  ~MSGQContext();
+};
+
+class MSGQMessage : public Message {
+private:
+  char * data;
+  size_t size;
+public:
+  void init(size_t size);
+  void init(char *data, size_t size);
+  void takeOwnership(char *data, size_t size);
+  size_t getSize(){return size;}
+  char * getData(){return data;}
+  void close();
+  ~MSGQMessage();
+};
+
+class MSGQSubSocket : public SubSocket {
+private:
+  msgq_queue_t * q = NULL;
+  int timeout;
+public:
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false);
+  void setTimeout(int timeout);
+  void * getRawSocket() {return (void*)q;}
+  Message *receive(bool non_blocking=false);
+  ~MSGQSubSocket();
+};
+
+class MSGQPubSocket : public PubSocket {
+private:
+  msgq_queue_t * q = NULL;
+public:
+  int connect(Context *context, std::string endpoint);
+  int sendMessage(Message *message);
+  int send(char *data, size_t size);
+  ~MSGQPubSocket();
+};
+
+class MSGQPoller : public Poller {
+private:
+  std::vector<SubSocket*> sockets;
+  msgq_pollitem_t polls[MAX_POLLERS];
+  size_t num_polls = 0;
+
+public:
+  void registerSocket(SubSocket *socket);
+  std::vector<SubSocket*> poll(int timeout);
+  ~MSGQPoller(){};
+};

cereal/messaging/impl_zmq.cc

@@ -0,0 +1,155 @@
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <cstdlib>
+#include <cerrno>
+
+#include <zmq.h>
+
+#include "services.h"
+#include "impl_zmq.hpp"
+
+static int get_port(std::string endpoint) {
+  int port = -1;
+  for (const auto& it : services) {
+    std::string name = it.name;
+    if (name == endpoint) {
+      port = it.port;
+      break;
+    }
+  }
+
+  assert(port >= 0);
+  return port;
+}
+
+ZMQContext::ZMQContext() {
+  context = zmq_ctx_new();
+}
+
+ZMQContext::~ZMQContext() {
+  zmq_ctx_term(context);
+}
+
+void ZMQMessage::init(size_t sz) {
+  size = sz;
+  data = new char[size];
+}
+
+void ZMQMessage::init(char * d, size_t sz) {
+  size = sz;
+  data = new char[size];
+  memcpy(data, d, size);
+}
+
+void ZMQMessage::close() {
+  if (size > 0){
+    delete[] data;
+  }
+  size = 0;
+}
+
+ZMQMessage::~ZMQMessage() {
+  this->close();
+}
+
+
+int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate){
+  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
+  if (sock == NULL){
+    return -1;
+  }
+
+  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
+
+  if (conflate){
+    int arg = 1;
+    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
+  }
+
+  int reconnect_ivl = 500;
+  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
+
+  full_endpoint = "tcp://" + address + ":";
+  full_endpoint += std::to_string(get_port(endpoint));
+
+  return zmq_connect(sock, full_endpoint.c_str());
+}
+
+
+Message * ZMQSubSocket::receive(bool non_blocking){
+  zmq_msg_t msg;
+  assert(zmq_msg_init(&msg) == 0);
+
+  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
+  int rc = zmq_msg_recv(&msg, sock, flags);
+  Message *r = NULL;
+
+  if (rc >= 0){
+    // Make a copy to ensure the data is aligned
+    r = new ZMQMessage;
+    r->init((char*)zmq_msg_data(&msg), zmq_msg_size(&msg));
+  }
+
+  zmq_msg_close(&msg);
+  return r;
+}
+
+void ZMQSubSocket::setTimeout(int timeout){
+  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
+}
+
+ZMQSubSocket::~ZMQSubSocket(){
+  zmq_close(sock);
+}
+
+int ZMQPubSocket::connect(Context *context, std::string endpoint){
+  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
+  if (sock == NULL){
+    return -1;
+  }
+
+  full_endpoint = "tcp://*:";
+  full_endpoint += std::to_string(get_port(endpoint));
+
+  return zmq_bind(sock, full_endpoint.c_str());
+}
+
+int ZMQPubSocket::sendMessage(Message *message){
+  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
+}
+
+int ZMQPubSocket::send(char *data, size_t size){
+  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
+}
+
+ZMQPubSocket::~ZMQPubSocket(){
+  zmq_close(sock);
+}
+
+
+void ZMQPoller::registerSocket(SubSocket * socket){
+  assert(num_polls + 1 < MAX_POLLERS);
+  polls[num_polls].socket = socket->getRawSocket();
+  polls[num_polls].events = ZMQ_POLLIN;
+
+  sockets.push_back(socket);
+  num_polls++;
+}
+
+std::vector<SubSocket*> ZMQPoller::poll(int timeout){
+  std::vector<SubSocket*> r;
+
+  int rc = zmq_poll(polls, num_polls, timeout);
+  if (rc < 0){
+    return r;
+  }
+
+  for (size_t i = 0; i < num_polls; i++){
+    if (polls[i].revents){
+      r.push_back(sockets[i]);
+    }
+  }
+
+  return r;
+}

cereal/messaging/impl_zmq.hpp

@@ -0,0 +1,63 @@
+#pragma once
+#include "messaging.hpp"
+#include <zmq.h>
+#include <string>
+
+#define MAX_POLLERS 128
+
+class ZMQContext : public Context {
+private:
+  void * context = NULL;
+public:
+  ZMQContext();
+  void * getRawContext() {return context;}
+  ~ZMQContext();
+};
+
+class ZMQMessage : public Message {
+private:
+  char * data;
+  size_t size;
+public:
+  void init(size_t size);
+  void init(char *data, size_t size);
+  size_t getSize(){return size;}
+  char * getData(){return data;}
+  void close();
+  ~ZMQMessage();
+};
+
+class ZMQSubSocket : public SubSocket {
+private:
+  void * sock;
+  std::string full_endpoint;
+public:
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false);
+  void setTimeout(int timeout);
+  void * getRawSocket() {return sock;}
+  Message *receive(bool non_blocking=false);
+  ~ZMQSubSocket();
+};
+
+class ZMQPubSocket : public PubSocket {
+private:
+  void * sock;
+  std::string full_endpoint;
+public:
+  int connect(Context *context, std::string endpoint);
+  int sendMessage(Message *message);
+  int send(char *data, size_t size);
+  ~ZMQPubSocket();
+};
+
+class ZMQPoller : public Poller {
+private:
+  std::vector<SubSocket*> sockets;
+  zmq_pollitem_t polls[MAX_POLLERS];
+  size_t num_polls = 0;
+
+public:
+  void registerSocket(SubSocket *socket);
+  std::vector<SubSocket*> poll(int timeout);
+  ~ZMQPoller(){};
+};

cereal/messaging/messaging.cc

@@ -0,0 +1,123 @@
+#include "messaging.hpp"
+#include "impl_zmq.hpp"
+#include "impl_msgq.hpp"
+
+#ifdef __APPLE__
+const bool MUST_USE_ZMQ = true;
+#else
+const bool MUST_USE_ZMQ = false;
+#endif
+
+Context * Context::create(){
+  Context * c;
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+    c = new ZMQContext();
+  } else {
+    c = new MSGQContext();
+  }
+  return c;
+}
+
+SubSocket * SubSocket::create(){
+  SubSocket * s;
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+    s = new ZMQSubSocket();
+  } else {
+    s = new MSGQSubSocket();
+  }
+  return s;
+}
+
+SubSocket * SubSocket::create(Context * context, std::string endpoint){
+  SubSocket *s = SubSocket::create();
+  int r = s->connect(context, endpoint, "127.0.0.1");
+
+  if (r == 0) {
+    return s;
+  } else {
+    delete s;
+    return NULL;
+  }
+}
+
+SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address){
+  SubSocket *s = SubSocket::create();
+  int r = s->connect(context, endpoint, address);
+
+  if (r == 0) {
+    return s;
+  } else {
+    delete s;
+    return NULL;
+  }
+}
+
+SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address, bool conflate){
+  SubSocket *s = SubSocket::create();
+  int r = s->connect(context, endpoint, address, conflate);
+
+  if (r == 0) {
+    return s;
+  } else {
+    delete s;
+    return NULL;
+  }
+}
+
+PubSocket * PubSocket::create(){
+  PubSocket * s;
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+    s = new ZMQPubSocket();
+  } else {
+    s = new MSGQPubSocket();
+  }
+  return s;
+}
+
+PubSocket * PubSocket::create(Context * context, std::string endpoint){
+  PubSocket *s = PubSocket::create();
+  int r = s->connect(context, endpoint);
+
+  if (r == 0) {
+    return s;
+  } else {
+    delete s;
+    return NULL;
+  }
+}
+
+Poller * Poller::create(){
+  Poller * p;
+  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+    p = new ZMQPoller();
+  } else {
+    p = new MSGQPoller();
+  }
+  return p;
+}
+
+Poller * Poller::create(std::vector<SubSocket*> sockets){
+  Poller * p = Poller::create();
+
+  for (auto s : sockets){
+    p->registerSocket(s);
+  }
+  return p;
+}
+
+extern "C" Context * messaging_context_create() {
+  return Context::create();
+}
+
+extern "C" SubSocket * messaging_subsocket_create(Context* context, const char* endpoint) {
+  return SubSocket::create(context, std::string(endpoint));
+}
+
+extern "C" PubSocket * messaging_pubsocket_create(Context* context, const char* endpoint) {
+  return PubSocket::create(context, std::string(endpoint));
+}
+
+extern "C" Poller * messaging_poller_create(SubSocket** sockets, int size) {
+  std::vector<SubSocket*> socketsVec(sockets, sockets + size);
+  return Poller::create(socketsVec);
+}

cereal/messaging/messaging.hpp

@@ -0,0 +1,92 @@
+#pragma once
+#include <cstddef>
+#include <map>
+#include <string>
+#include <vector>
+#include <capnp/serialize.h>
+#include "../gen/cpp/log.capnp.h"
+
+#define MSG_MULTIPLE_PUBLISHERS 100
+
+class Context {
+public:
+  virtual void * getRawContext() = 0;
+  static Context * create();
+  virtual ~Context(){};
+};
+
+class Message {
+public:
+  virtual void init(size_t size) = 0;
+  virtual void init(char * data, size_t size) = 0;
+  virtual void close() = 0;
+  virtual size_t getSize() = 0;
+  virtual char * getData() = 0;
+  virtual ~Message(){};
+};
+
+
+class SubSocket {
+public:
+  virtual int connect(Context *context, std::string endpoint, std::string address, bool conflate=false) = 0;
+  virtual void setTimeout(int timeout) = 0;
+  virtual Message *receive(bool non_blocking=false) = 0;
+  virtual void * getRawSocket() = 0;
+  static SubSocket * create();
+  static SubSocket * create(Context * context, std::string endpoint);
+  static SubSocket * create(Context * context, std::string endpoint, std::string address);
+  static SubSocket * create(Context * context, std::string endpoint, std::string address, bool conflate);
+  virtual ~SubSocket(){};
+};
+
+class PubSocket {
+public:
+  virtual int connect(Context *context, std::string endpoint) = 0;
+  virtual int sendMessage(Message *message) = 0;
+  virtual int send(char *data, size_t size) = 0;
+  static PubSocket * create();
+  static PubSocket * create(Context * context, std::string endpoint);
+  virtual ~PubSocket(){};
+};
+
+class Poller {
+public:
+  virtual void registerSocket(SubSocket *socket) = 0;
+  virtual std::vector<SubSocket*> poll(int timeout) = 0;
+  static Poller * create();
+  static Poller * create(std::vector<SubSocket*> sockets);
+  virtual ~Poller(){};
+};
+
+class SubMaster {
+ public:
+  SubMaster(const std::initializer_list<const char *> &service_list,
+            const char *address = nullptr, const std::initializer_list<const char *> &ignore_alive = {});
+  int update(int timeout = 1000);
+  inline bool allAlive(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, false, true); }
+  inline bool allValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, false); }
+  inline bool allAliveAndValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, true); }
+  bool updated(const char *name) const;
+  void drain();
+  cereal::Event::Reader &operator[](const char *name);
+  ~SubMaster();
+
+ private:
+  bool all_(const std::initializer_list<const char *> &service_list, bool valid, bool alive);
+  Poller *poller_ = nullptr;
+  uint64_t frame_ = 0;
+  struct SubMessage;
+  std::map<SubSocket *, SubMessage *> messages_;
+  std::map<std::string, SubMessage *> services_;
+};
+
+class PubMaster {
+ public:
+  PubMaster(const std::initializer_list<const char *> &service_list);
+  inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
+  int send(const char *name, capnp::MessageBuilder &msg);
+  ~PubMaster();
+
+ private:
+  std::map<std::string, PubSocket *> sockets_;
+};

cereal/messaging/messaging.pxd

@@ -0,0 +1,39 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp cimport bool
+
+
+cdef extern from "messaging.hpp":
+  cdef cppclass Context:
+    @staticmethod
+    Context * create()
+
+  cdef cppclass Message:
+    void init(size_t)
+    void init(char *, size_t)
+    void close()
+    size_t getSize()
+    char *getData()
+
+  cdef cppclass SubSocket:
+    @staticmethod
+    SubSocket * create()
+    int connect(Context *, string, string, bool)
+    Message * receive(bool)
+    void setTimeout(int)
+
+  cdef cppclass PubSocket:
+    @staticmethod
+    PubSocket * create()
+    int connect(Context *, string)
+    int sendMessage(Message *)
+    int send(char *, size_t)
+
+  cdef cppclass Poller:
+    @staticmethod
+    Poller * create()
+    void registerSocket(SubSocket *)
+    vector[SubSocket*] poll(int) nogil

cereal/messaging/messaging_pyx.pyx

@@ -0,0 +1,151 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+from libcpp.string cimport string
+from libcpp cimport bool
+from libc cimport errno
+
+
+from messaging cimport Context as cppContext
+from messaging cimport SubSocket as cppSubSocket
+from messaging cimport PubSocket as cppPubSocket
+from messaging cimport Poller as cppPoller
+from messaging cimport Message as cppMessage
+
+
+class MessagingError(Exception):
+  pass
+
+
+class MultiplePublishersError(MessagingError):
+  pass
+
+
+cdef class Context:
+  cdef cppContext * context
+
+  def __cinit__(self):
+    self.context = cppContext.create()
+
+  def term(self):
+    del self.context
+    self.context = NULL
+
+  def __dealloc__(self):
+    pass
+    # Deleting the context will hang if sockets are still active
+    # TODO: Figure out a way to make sure the context is closed last
+    # del self.context
+
+
+cdef class Poller:
+  cdef cppPoller * poller
+  cdef list sub_sockets
+
+  def __cinit__(self):
+    self.sub_sockets = []
+    self.poller = cppPoller.create()
+
+  def __dealloc__(self):
+    del self.poller
+
+  def registerSocket(self, SubSocket socket):
+    self.sub_sockets.append(socket)
+    self.poller.registerSocket(socket.socket)
+
+  def poll(self, timeout):
+    sockets = []
+    cdef int t = timeout
+
+    with nogil:
+        result = self.poller.poll(t)
+
+    for s in result:
+        socket = SubSocket()
+        socket.setPtr(s)
+        sockets.append(socket)
+
+    return sockets
+
+cdef class SubSocket:
+  cdef cppSubSocket * socket
+  cdef bool is_owner
+
+  def __cinit__(self):
+    self.socket = cppSubSocket.create()
+    self.is_owner = True
+
+    if self.socket == NULL:
+      raise MessagingError
+
+  def __dealloc__(self):
+    if self.is_owner:
+      del self.socket
+
+  cdef setPtr(self, cppSubSocket * ptr):
+    if self.is_owner:
+      del self.socket
+
+    self.is_owner = False
+    self.socket = ptr
+
+  def connect(self, Context context, string endpoint, string address=b"127.0.0.1", bool conflate=False):
+    r = self.socket.connect(context.context, endpoint, address, conflate)
+
+    if r != 0:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise MessagingError
+
+  def setTimeout(self, int timeout):
+    self.socket.setTimeout(timeout)
+
+  def receive(self, bool non_blocking=False):
+    msg = self.socket.receive(non_blocking)
+
+    if msg == NULL:
+      # If a blocking read returns no message check errno if SIGINT was caught in the C++ code
+      if errno.errno == errno.EINTR:
+        print("SIGINT received, exiting")
+        sys.exit(1)
+
+      return None
+    else:
+      sz = msg.getSize()
+      m = msg.getData()[:sz]
+      del msg
+
+      return m
+
+
+cdef class PubSocket:
+  cdef cppPubSocket * socket
+
+  def __cinit__(self):
+    self.socket = cppPubSocket.create()
+    if self.socket == NULL:
+      raise MessagingError
+
+  def __dealloc__(self):
+    del self.socket
+
+  def connect(self, Context context, string endpoint):
+    r = self.socket.connect(context.context, endpoint)
+
+    if r != 0:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise MessagingError
+
+  def send(self, string data):
+    length = len(data)
+    r = self.socket.send(<char*>data.c_str(), length)
+
+    if r != length:
+      if errno.errno == errno.EADDRINUSE:
+        raise MultiplePublishersError
+      else:
+        raise MessagingError

cereal/messaging/messaging_pyx_setup.py

@@ -0,0 +1,57 @@
+import os
+import subprocess
+import sysconfig
+from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
+
+from Cython.Build import cythonize
+from Cython.Distutils import build_ext
+
+
+def get_ext_filename_without_platform_suffix(filename):
+  name, ext = os.path.splitext(filename)
+  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+  if ext_suffix == ext:
+    return filename
+
+  ext_suffix = ext_suffix.replace(ext, '')
+  idx = name.find(ext_suffix)
+
+  if idx == -1:
+    return filename
+  else:
+    return name[:idx] + ext
+
+
+class BuildExtWithoutPlatformSuffix(build_ext):
+  def get_ext_filename(self, ext_name):
+    filename = super().get_ext_filename(ext_name)
+    return get_ext_filename_without_platform_suffix(filename)
+
+
+sourcefiles = ['messaging_pyx.pyx']
+extra_compile_args = ["-std=c++14"]
+libraries = ['zmq']
+ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()  # pylint: disable=unexpected-keyword-arg
+
+if ARCH == "aarch64" and os.path.isdir("/system"):
+  # android
+  extra_compile_args += ["-Wno-deprecated-register"]
+  libraries += ['gnustl_shared']
+
+setup(name='CAN parser',
+      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
+      ext_modules=cythonize(
+        Extension(
+          "messaging_pyx",
+          language="c++",
+          sources=sourcefiles,
+          extra_compile_args=extra_compile_args,
+          libraries=libraries,
+          extra_objects=[
+            os.path.join(os.path.dirname(os.path.realpath(__file__)), '../', 'libmessaging.a'),
+          ]
+        )
+      ),
+      nthreads=4,
+)

cereal/messaging/msgq.cc

@@ -0,0 +1,457 @@
+#include <iostream>
+#include <cassert>
+#include <cerrno>
+#include <cmath>
+#include <cstring>
+#include <cstdint>
+#include <chrono>
+#include <algorithm>
+#include <cstdlib>
+#include <csignal>
+#include <random>
+
+#include <poll.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <stdio.h>
+
+#include "msgq.hpp"
+
+void sigusr2_handler(int signal) {
+  assert(signal == SIGUSR2);
+}
+
+uint64_t msgq_get_uid(void){
+  std::random_device rd("/dev/urandom");
+  std::uniform_int_distribution<uint64_t> distribution(0,std::numeric_limits<uint32_t>::max());
+
+  #ifdef __APPLE__
+    // TODO: this doesn't work
+    uint64_t uid = distribution(rd) << 32 | getpid();
+  #else
+    uint64_t uid = distribution(rd) << 32 | syscall(SYS_gettid);
+  #endif
+
+  return uid;
+}
+
+int msgq_msg_init_size(msgq_msg_t * msg, size_t size){
+  msg->size = size;
+  msg->data = new(std::nothrow) char[size];
+
+  return (msg->data == NULL) ? -1 : 0;
+}
+
+
+int msgq_msg_init_data(msgq_msg_t * msg, char * data, size_t size) {
+  int r = msgq_msg_init_size(msg, size);
+
+  if (r == 0)
+    memcpy(msg->data, data, size);
+
+  return r;
+}
+
+int msgq_msg_close(msgq_msg_t * msg){
+  if (msg->size > 0)
+    delete[] msg->data;
+
+  msg->size = 0;
+
+  return 0;
+}
+
+void msgq_reset_reader(msgq_queue_t * q){
+  int id = q->reader_id;
+  q->read_valids[id]->store(true);
+  q->read_pointers[id]->store(*q->write_pointer);
+}
+
+void msgq_wait_for_subscriber(msgq_queue_t *q){
+  while (*q->num_readers == 0){
+    ;
+  }
+
+  return;
+}
+
+
+
+int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){
+  assert(size < 0xFFFFFFFF); // Buffer must be smaller than 2^32 bytes
+
+  std::signal(SIGUSR2, sigusr2_handler);
+
+  const char * prefix = "/dev/shm/";
+  char * full_path = new char[strlen(path) + strlen(prefix) + 1];
+  strcpy(full_path, prefix);
+  strcat(full_path, path);
+
+  auto fd = open(full_path, O_RDWR | O_CREAT, 0777);
+  if (fd < 0) {
+    std::cout << "Warning, could not open: " << full_path << std::endl;
+    delete[] full_path;
+    return -1;
+  }
+  delete[] full_path;
+
+  int rc = ftruncate(fd, size + sizeof(msgq_header_t));
+  if (rc < 0)
+    return -1;
+
+  char * mem = (char*)mmap(NULL, size + sizeof(msgq_header_t), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+  close(fd);
+
+  if (mem == NULL)
+    return -1;
+
+  q->mmap_p = mem;
+
+  msgq_header_t *header = (msgq_header_t *)mem;
+
+  // Setup pointers to header segment
+  q->num_readers = reinterpret_cast<std::atomic<uint64_t>*>(&header->num_readers);
+  q->write_pointer = reinterpret_cast<std::atomic<uint64_t>*>(&header->write_pointer);
+  q->write_uid = reinterpret_cast<std::atomic<uint64_t>*>(&header->write_uid);
+
+  for (size_t i = 0; i < NUM_READERS; i++){
+    q->read_pointers[i] = reinterpret_cast<std::atomic<uint64_t>*>(&header->read_pointers[i]);
+    q->read_valids[i] = reinterpret_cast<std::atomic<uint64_t>*>(&header->read_valids[i]);
+    q->read_uids[i] = reinterpret_cast<std::atomic<uint64_t>*>(&header->read_uids[i]);
+  }
+
+  q->data = mem + sizeof(msgq_header_t);
+  q->size = size;
+  q->reader_id = -1;
+
+  q->endpoint = path;
+  q->read_conflate = false;
+
+  return 0;
+}
+
+void msgq_close_queue(msgq_queue_t *q){
+  if (q->mmap_p != NULL){
+    munmap(q->mmap_p, q->size + sizeof(msgq_header_t));
+  }
+}
+
+
+void msgq_init_publisher(msgq_queue_t * q) {
+  //std::cout << "Starting publisher" << std::endl;
+  uint64_t uid = msgq_get_uid();
+
+  *q->write_uid = uid;
+  *q->num_readers = 0;
+
+  for (size_t i = 0; i < NUM_READERS; i++){
+    *q->read_valids[i] = false;
+    *q->read_uids[i] = 0;
+  }
+
+  q->write_uid_local = uid;
+}
+
+static void thread_signal(uint32_t tid) {
+  #ifndef SYS_tkill
+    // TODO: this won't work for multithreaded programs
+    kill(tid, SIGUSR2);
+  #else
+    syscall(SYS_tkill, tid, SIGUSR2);
+  #endif
+}
+
+void msgq_init_subscriber(msgq_queue_t * q) {
+  assert(q != NULL);
+  assert(q->num_readers != NULL);
+
+  uint64_t uid = msgq_get_uid();
+
+  // Get reader id
+  while (true){
+    uint64_t cur_num_readers = *q->num_readers;
+    uint64_t new_num_readers = cur_num_readers + 1;
+
+    // No more slots available. Reset all subscribers to kick out inactive ones
+    if (new_num_readers > NUM_READERS){
+      std::cout << "Warning, evicting all subscribers!" << std::endl;
+      *q->num_readers = 0;
+
+      for (size_t i = 0; i < NUM_READERS; i++){
+        *q->read_valids[i] = false;
+
+        uint64_t old_uid = *q->read_uids[i];
+        *q->read_uids[i] = 0;
+
+        // Wake up reader in case they are in a poll
+        thread_signal(old_uid & 0xFFFFFFFF);
+      }
+
+      continue;
+    }
+
+    // Use atomic compare and swap to handle race condition
+    // where two subscribers start at the same time
+    if (std::atomic_compare_exchange_strong(q->num_readers,
+                                            &cur_num_readers,
+                                            new_num_readers)){
+      q->reader_id = cur_num_readers;
+      q->read_uid_local = uid;
+
+      // We start with read_valid = false,
+      // on the first read the read pointer will be synchronized with the write pointer
+      *q->read_valids[cur_num_readers] = false;
+      *q->read_pointers[cur_num_readers] = 0;
+      *q->read_uids[cur_num_readers] = uid;
+      break;
+    }
+  }
+
+  //std::cout << "New subscriber id: " << q->reader_id << " uid: " << q->read_uid_local << " " << q->endpoint << std::endl;
+  msgq_reset_reader(q);
+}
+
+int msgq_msg_send(msgq_msg_t * msg, msgq_queue_t *q){
+  // Die if we are no longer the active publisher
+  if (q->write_uid_local != *q->write_uid){
+    std::cout << "Killing old publisher: " << q->endpoint << std::endl;
+    errno = EADDRINUSE;
+    return -1;
+  }
+
+  uint64_t total_msg_size = ALIGN(msg->size + sizeof(int64_t));
+
+  // We need to fit at least three messages in the queue,
+  // then we can always safely access the last message
+  assert(3 * total_msg_size <= q->size);
+
+  uint64_t num_readers = *q->num_readers;
+
+  uint32_t write_cycles, write_pointer;
+  UNPACK64(write_cycles, write_pointer, *q->write_pointer);
+
+  char *p = q->data + write_pointer; // add base offset
+
+  // Check remaining space
+  // Always leave space for a wraparound tag for the next message, including alignment
+  int64_t remaining_space = q->size - write_pointer - total_msg_size - sizeof(int64_t);
+  if (remaining_space <= 0){
+    // Write -1 size tag indicating wraparound
+    *(int64_t*)p = -1;
+
+    // Invalidate all readers that are beyond the write pointer
+    // TODO: should we handle the case where a new reader shows up while this is running?
+    for (uint64_t i = 0; i < num_readers; i++){
+      uint64_t read_pointer = *q->read_pointers[i];
+      uint64_t read_cycles = read_pointer >> 32;
+      read_pointer &= 0xFFFFFFFF;
+
+      if ((read_pointer > write_pointer) && (read_cycles != write_cycles)) {
+        *q->read_valids[i] = false;
+      }
+    }
+
+    // Update global and local copies of write pointer and write_cycles
+    write_pointer = 0;
+    write_cycles = write_cycles + 1;
+    PACK64(*q->write_pointer, write_cycles, write_pointer);
+
+    // Set actual pointer to the beginning of the data segment
+    p = q->data;
+  }
+
+  // Invalidate readers that are in the area that will be written
+  uint64_t start = write_pointer;
+  uint64_t end = ALIGN(start + sizeof(int64_t) + msg->size);
+
+  for (uint64_t i = 0; i < num_readers; i++){
+    uint32_t read_cycles, read_pointer;
+    UNPACK64(read_cycles, read_pointer, *q->read_pointers[i]);
+
+    if ((read_pointer >= start) && (read_pointer < end) && (read_cycles != write_cycles)) {
+      *q->read_valids[i] = false;
+    }
+  }
+
+
+  // Write size tag
+  std::atomic<int64_t> *size_p = reinterpret_cast<std::atomic<int64_t>*>(p);
+  *size_p = msg->size;
+
+  // Copy data
+  memcpy(p + sizeof(int64_t), msg->data, msg->size);
+  __sync_synchronize();
+
+  // Update write pointer
+  uint32_t new_ptr = ALIGN(write_pointer + msg->size + sizeof(int64_t));
+  PACK64(*q->write_pointer, write_cycles, new_ptr);
+
+  // Notify readers
+  for (uint64_t i = 0; i < num_readers; i++){
+    uint64_t reader_uid = *q->read_uids[i];
+    thread_signal(reader_uid & 0xFFFFFFFF);
+  }
+
+  return msg->size;
+}
+
+
+int msgq_msg_ready(msgq_queue_t * q){
+ start:
+  int id = q->reader_id;
+  assert(id >= 0); // Make sure subscriber is initialized
+
+  if (q->read_uid_local != *q->read_uids[id]){
+    std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
+    msgq_init_subscriber(q);
+    goto start;
+  }
+
+  // Check valid
+  if (!*q->read_valids[id]){
+    msgq_reset_reader(q);
+    goto start;
+  }
+
+  uint32_t read_cycles, read_pointer;
+  UNPACK64(read_cycles, read_pointer, *q->read_pointers[id]);
+
+  uint32_t write_cycles, write_pointer;
+  UNPACK64(write_cycles, write_pointer, *q->write_pointer);
+
+  // Check if new message is available
+  return (read_pointer != write_pointer);
+}
+
+int msgq_msg_recv(msgq_msg_t * msg, msgq_queue_t * q){
+ start:
+  int id = q->reader_id;
+  assert(id >= 0); // Make sure subscriber is initialized
+
+  if (q->read_uid_local != *q->read_uids[id]){
+    std::cout << q->endpoint << ": Reader was evicted, reconnecting" << std::endl;
+    msgq_init_subscriber(q);
+    goto start;
+  }
+
+  // Check valid
+  if (!*q->read_valids[id]){
+    msgq_reset_reader(q);
+    goto start;
+  }
+
+  uint32_t read_cycles, read_pointer;
+  UNPACK64(read_cycles, read_pointer, *q->read_pointers[id]);
+
+  uint32_t write_cycles, write_pointer;
+  UNPACK64(write_cycles, write_pointer, *q->write_pointer);
+
+  char * p = q->data + read_pointer;
+
+  // Check if new message is available
+  if (read_pointer == write_pointer) {
+    msg->size = 0;
+    return 0;
+  }
+
+  // Read potential message size
+  std::atomic<int64_t> *size_p = reinterpret_cast<std::atomic<int64_t>*>(p);
+  std::int64_t size = *size_p;
+
+  // Check if the size that was read is valid
+  if (!*q->read_valids[id]){
+    msgq_reset_reader(q);
+    goto start;
+  }
+
+  // If size is -1 the buffer was full, and we need to wrap around
+  if (size == -1){
+    read_cycles++;
+    PACK64(*q->read_pointers[id], read_cycles, 0);
+    goto start;
+  }
+
+  // crashing is better than passing garbage data to the consumer
+  // the size will have weird value if it was overwritten by data accidentally
+  assert((uint64_t)size < q->size);
+  assert(size > 0);
+
+  uint32_t new_read_pointer = ALIGN(read_pointer + sizeof(std::int64_t) + size);
+
+  // If conflate is true, check if this is the latest message, else start over
+  if (q->read_conflate){
+    if (new_read_pointer != write_pointer){
+      // Update read pointer
+      PACK64(*q->read_pointers[id], read_cycles, new_read_pointer);
+      goto start;
+    }
+  }
+
+  // Copy message
+  if (msgq_msg_init_size(msg, size) < 0)
+    return -1;
+
+  __sync_synchronize();
+  memcpy(msg->data, p + sizeof(int64_t), size);
+  __sync_synchronize();
+
+  // Update read pointer
+  PACK64(*q->read_pointers[id], read_cycles, new_read_pointer);
+
+  // Check if the actual data that was copied is valid
+  if (!*q->read_valids[id]){
+    msgq_msg_close(msg);
+    msgq_reset_reader(q);
+    goto start;
+  }
+
+
+  return msg->size;
+}
+
+
+
+int msgq_poll(msgq_pollitem_t * items, size_t nitems, int timeout){
+  assert(timeout >= 0);
+
+  int num = 0;
+
+  // Check if messages ready
+  for (size_t i = 0; i < nitems; i++) {
+    items[i].revents = msgq_msg_ready(items[i].q);
+    if (items[i].revents) num++;
+  }
+
+  int ms = (timeout == -1) ? 100 : timeout;
+  struct timespec ts;
+  ts.tv_sec = ms / 1000;
+  ts.tv_nsec = (ms % 1000) * 1000 * 1000;
+
+
+  while (num == 0) {
+    int ret;
+
+    ret = nanosleep(&ts, &ts);
+
+    // Check if messages ready
+    for (size_t i = 0; i < nitems; i++) {
+      if (items[i].revents == 0 && msgq_msg_ready(items[i].q)){
+        num += 1;
+        items[i].revents = 1;
+      }
+    }
+
+    // exit if we had a timeout and the sleep finished
+    if (timeout != -1 && ret == 0){
+      break;
+    }
+  }
+
+  return num;
+}

cereal/messaging/msgq.hpp

@@ -0,0 +1,66 @@
+#pragma once
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <atomic>
+
+#define DEFAULT_SEGMENT_SIZE (10 * 1024 * 1024)
+#define NUM_READERS 8
+#define ALIGN(n) ((n + (8 - 1)) & -8)
+
+#define UNPACK64(higher, lower, input) do {uint64_t tmp = input; higher = tmp >> 32; lower = tmp & 0xFFFFFFFF;} while (0)
+#define PACK64(output, higher, lower) output = ((uint64_t)higher << 32 ) | ((uint64_t)lower & 0xFFFFFFFF)
+
+struct  msgq_header_t {
+  uint64_t num_readers;
+  uint64_t write_pointer;
+  uint64_t write_uid;
+  uint64_t read_pointers[NUM_READERS];
+  uint64_t read_valids[NUM_READERS];
+  uint64_t read_uids[NUM_READERS];
+};
+
+struct msgq_queue_t {
+  std::atomic<uint64_t> *num_readers;
+  std::atomic<uint64_t> *write_pointer;
+  std::atomic<uint64_t> *write_uid;
+  std::atomic<uint64_t> *read_pointers[NUM_READERS];
+  std::atomic<uint64_t> *read_valids[NUM_READERS];
+  std::atomic<uint64_t> *read_uids[NUM_READERS];
+  char * mmap_p;
+  char * data;
+  size_t size;
+  int reader_id;
+  uint64_t read_uid_local;
+  uint64_t write_uid_local;
+
+  bool read_conflate;
+  std::string endpoint;
+};
+
+struct msgq_msg_t {
+  size_t size;
+  char * data;
+};
+
+struct msgq_pollitem_t {
+  msgq_queue_t *q;
+  int revents;
+};
+
+void msgq_wait_for_subscriber(msgq_queue_t *q);
+void msgq_reset_reader(msgq_queue_t *q);
+
+int msgq_msg_init_size(msgq_msg_t *msg, size_t size);
+int msgq_msg_init_data(msgq_msg_t *msg, char * data, size_t size);
+int msgq_msg_close(msgq_msg_t *msg);
+
+int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size);
+void msgq_close_queue(msgq_queue_t *q);
+void msgq_init_publisher(msgq_queue_t * q);
+void msgq_init_subscriber(msgq_queue_t * q);
+
+int msgq_msg_send(msgq_msg_t *msg, msgq_queue_t *q);
+int msgq_msg_recv(msgq_msg_t *msg, msgq_queue_t *q);
+int msgq_msg_ready(msgq_queue_t * q);
+int msgq_poll(msgq_pollitem_t * items, size_t nitems, int timeout);

cereal/messaging/socketmaster.cc

@@ -0,0 +1,162 @@
+#include <assert.h>
+#include <time.h>
+#include "messaging.hpp"
+#include "services.h"
+#ifdef __APPLE__
+#define CLOCK_BOOTTIME CLOCK_MONOTONIC
+#endif
+static inline uint64_t nanos_since_boot() {
+  struct timespec t;
+  clock_gettime(CLOCK_BOOTTIME, &t);
+  return t.tv_sec * 1000000000ULL + t.tv_nsec;
+}
+static const service *get_service(const char *name) {
+  for (const auto &it : services) {
+    if (strcmp(it.name, name) == 0) return &it;
+  }
+  return nullptr;
+}
+static inline bool inList(const std::initializer_list<const char *> &list, const char *value) {
+  for (auto &v : list) {
+    if (strcmp(value, v) == 0) return true;
+  }
+  return false;
+}
+
+class MessageContext {
+ public:
+  MessageContext() { ctx_ = Context::create(); }
+  ~MessageContext() { delete ctx_; }
+  Context *ctx_;
+};
+MessageContext ctx;
+
+struct SubMaster::SubMessage {
+  std::string name;
+  SubSocket *socket = nullptr;
+  int freq = 0;
+  bool updated = false, alive = false, valid = false, ignore_alive;
+  uint64_t rcv_time = 0, rcv_frame = 0;
+  void *allocated_msg_reader = nullptr;
+  capnp::FlatArrayMessageReader *msg_reader = nullptr;
+  kj::Array<capnp::word> buf;
+  cereal::Event::Reader event;
+};
+
+SubMaster::SubMaster(const std::initializer_list<const char *> &service_list, const char *address,
+                     const std::initializer_list<const char *> &ignore_alive) {
+  poller_ = Poller::create();
+  for (auto name : service_list) {
+    const service *serv = get_service(name);
+    assert(serv != nullptr);
+    SubSocket *socket = SubSocket::create(ctx.ctx_, name, address ? address : "127.0.0.1", true);
+    assert(socket != 0);
+    poller_->registerSocket(socket);
+    SubMessage *m = new SubMessage{
+        .socket = socket,
+        .freq = serv->frequency,
+        .ignore_alive = inList(ignore_alive, name),
+        .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
+        .buf = kj::heapArray<capnp::word>(1024)};
+    messages_[socket] = m;
+    services_[name] = m;
+  }
+}
+
+int SubMaster::update(int timeout) {
+  if (++frame_ == UINT64_MAX) frame_ = 1;
+  for (auto &kv : messages_) kv.second->updated = false;
+
+  int updated = 0;
+  auto sockets = poller_->poll(timeout);
+  uint64_t current_time = nanos_since_boot();
+  for (auto s : sockets) {
+    Message *msg = s->receive(true);
+    if (msg == nullptr) continue;
+
+    SubMessage *m = messages_.at(s);
+    const size_t size = (msg->getSize() / sizeof(capnp::word)) + 1;
+    if (m->buf.size() < size) {
+      m->buf = kj::heapArray<capnp::word>(size);
+    }
+    memcpy(m->buf.begin(), msg->getData(), msg->getSize());
+    delete msg;
+
+    if (m->msg_reader) {
+      m->msg_reader->~FlatArrayMessageReader();
+    }
+    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(kj::ArrayPtr<capnp::word>(m->buf.begin(), size));
+    m->event = m->msg_reader->getRoot<cereal::Event>();
+    m->updated = true;
+    m->rcv_time = current_time;
+    m->rcv_frame = frame_;
+    m->valid = m->event.getValid();
+
+    ++updated;
+  }
+
+  for (auto &kv : messages_) {
+    SubMessage *m = kv.second;
+    m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
+  }
+  return updated;
+}
+
+bool SubMaster::all_(const std::initializer_list<const char *> &service_list, bool valid, bool alive) {
+  int found = 0;
+  for (auto &kv : messages_) {
+    SubMessage *m = kv.second;
+    if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
+      found += (!valid || m->valid) && (!alive || (m->alive && !m->ignore_alive));
+    }
+  }
+  return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
+}
+
+void SubMaster::drain() {
+  while (true) {
+    auto polls = poller_->poll(0);
+    if (polls.size() == 0)
+      break;
+
+    for (auto sock : polls) {
+      Message *msg = sock->receive(true);
+      delete msg;
+    }
+  }
+}
+
+bool SubMaster::updated(const char *name) const { return services_.at(name)->updated; }
+cereal::Event::Reader &SubMaster::operator[](const char *name) { return services_.at(name)->event; };
+
+SubMaster::~SubMaster() {
+  delete poller_;
+  for (auto &kv : messages_) {
+    SubMessage *m = kv.second;
+    if (m->msg_reader) {
+      m->msg_reader->~FlatArrayMessageReader();
+    }
+    free(m->allocated_msg_reader);
+    delete m->socket;
+    delete m;
+  }
+}
+
+PubMaster::PubMaster(const std::initializer_list<const char *> &service_list) {
+  for (auto name : service_list) {
+    assert(get_service(name) != nullptr);
+    PubSocket *socket = PubSocket::create(ctx.ctx_, name);
+    assert(socket);
+    sockets_[name] = socket;
+  }
+}
+
+int PubMaster::send(const char *name, capnp::MessageBuilder &msg) {
+  auto words = capnp::messageToFlatArray(msg);
+  auto bytes = words.asBytes();
+  return send(name, bytes.begin(), bytes.size());
+}
+
+PubMaster::~PubMaster() {
+  for (auto s : sockets_) delete s.second;
+}

cereal/service_list.yaml

@@ -0,0 +1,177 @@
+# TODO: these port numbers are hardcoded in c, fix this
+
+# LogRotate: 8001 is a PUSH PULL socket between loggerd and visiond
+
+# all ZMQ pub sub: port, should_log, frequency, (qlog_decimation)
+
+# frame syncing packet
+frame: [8002, true, 20., 1]
+# accel, gyro, and compass
+sensorEvents: [8003, true, 100., 100]
+# GPS data, also global timestamp
+gpsNMEA: [8004, true, 9.]  # 9 msgs each sec
+# CPU+MEM+GPU+BAT temps
+thermal: [8005, true, 2., 1]
+# List(CanData), list of can messages
+can: [8006, true, 100.]
+controlsState: [8007, true, 100., 100]
+#liveEvent: [8008, true, 0.]
+model: [8009, true, 20., 5]
+features: [8010, true, 0.]
+health: [8011, true, 2., 1]
+radarState: [8012, true, 20., 5]
+#liveUI: [8014, true, 0.]
+encodeIdx: [8015, true, 20.]
+liveTracks: [8016, true, 20.]
+sendcan: [8017, true, 100.]
+logMessage: [8018, true, 0.]
+liveCalibration: [8019, true, 4., 4]
+androidLog: [8020, true, 0.]
+carState: [8021, true, 100., 10]
+# 8022 is reserved for sshd
+carControl: [8023, true, 100., 10]
+plan: [8024, true, 20., 2]
+liveLocation: [8025, true, 0., 1]
+gpsLocation: [8026, true, 1., 1]
+ethernetData: [8027, true, 0.]
+navUpdate: [8028, true, 0.]
+qcomGnss: [8029, true, 0.]
+lidarPts: [8030, true, 0.]
+procLog: [8031, true, 0.5]
+gpsLocationExternal: [8032, true, 10., 1]
+ubloxGnss: [8033, true, 10.]
+clocks: [8034, true, 1., 1]
+liveMpc: [8035, false, 20.]
+liveLongitudinalMpc: [8036, false, 20.]
+navStatus: [8038, true, 0.]
+gpsLocationTrimble: [8039, true, 0.]
+trimbleGnss: [8041, true, 0.]
+ubloxRaw: [8042, true, 20.]
+gpsPlannerPoints: [8043, true, 0.]
+gpsPlannerPlan: [8044, true, 0.]
+applanixRaw: [8046, true, 0.]
+orbLocation: [8047, true, 0.]
+trafficEvents: [8048, true, 0.]
+liveLocationTiming: [8049, true, 0.]
+orbslamCorrection: [8050, true, 0.]
+liveLocationCorrected: [8051, true, 0.]
+orbObservation: [8052, true, 0.]
+applanixLocation: [8053, true, 0.]
+liveLocationKalman: [8054, true, 20., 2]
+uiNavigationEvent: [8055, true, 0.]
+orbOdometry: [8057, true, 0.]
+orbFeatures: [8058, false, 0.]
+orbKeyFrame: [8059, true, 0.]
+uiLayoutState: [8060, true, 0.]
+frontEncodeIdx: [8061, true, 5.]
+orbFeaturesSummary: [8062, true, 0.]
+driverState: [8063, true, 5., 1]
+liveParameters: [8064, true, 20., 2]
+liveMapData: [8065, true, 0.]
+cameraOdometry: [8066, true, 20., 5]
+pathPlan: [8067, true, 20., 2]
+kalmanOdometry: [8068, true, 0.]
+thumbnail: [8069, true, 0.2, 1]
+carEvents: [8070, true, 1., 1]
+carParams: [8071, true, 0.02, 1]
+frontFrame: [8072, true, 10.]
+dMonitoringState: [8073, true, 5., 1]
+offroadLayout: [8074, false, 0.]
+
+testModel: [8040, false, 0.]
+testLiveLocation: [8045, false, 0.]
+testJoystick: [8056, false, 0.]
+
+# 8080 is reserved for slave testing daemon
+# 8762 is reserved for logserver
+
+# manager -- base process to manage starting and stopping of all others
+#   subscribes: thermal
+
+# **** processes that communicate with the outside world ****
+
+# thermald -- decides when to start and stop onroad
+#   subscribes: health, location
+#   publishes: thermal
+
+# boardd -- communicates with the car
+#   subscribes: sendcan
+#   publishes: can, health, ubloxRaw
+
+# sensord -- publishes IMU and Magnetometer
+#   publishes: sensorEvents
+
+# gpsd -- publishes EON's gps
+#   publishes: gpsNMEA
+
+# camerad -- publishes camera frames
+#   publishes: frame, frontFrame, thumbnail
+#   subscribes: driverState
+
+# dmonitoringmodeld -- runs face detection on camera frames
+#   publishes: driverState
+
+# **** stateful data transformers ****
+
+# modeld -- runs & publishes the model
+#   publishes: model, cameraOdometry
+#   subscribes: liveCalibration, pathPlan
+
+# plannerd -- decides where to drive the car
+#   subscribes: carState, model, radarState, controlsState, liveParameters
+#   publishes:  plan, pathPlan, liveMpc, liveLongitudinalMpc
+
+# controlsd -- drives the car by sending CAN messages to panda
+#   subscribes: can, thermal, health, plan, pathPlan, dMonitoringState, liveCalibration, model
+#   publishes:  carState, carControl, sendcan, controlsState, carEvents, carParams
+
+# dmonitoringd -- processes driver monitoring data and publishes driver awareness
+#   subscribes: driverState, liveCalibration, carState, model, gpsLocation
+#   publishes: dMonitoringState
+
+# radard -- processes the radar and vision data
+#   subscribes: can, controlsState, model, liveParameters
+#   publishes:  radarState, liveTracks
+
+# params_learner -- learns vehicle params by observing the vehicle dynamics
+#   subscribes: controlsState, sensorEvents, cameraOdometry
+#   publishes: liveParameters
+
+# calibrationd -- reads posenet and applies a temporal filter on the frame region to look at
+#   subscribes: cameraOdometry
+#   publishes: liveCalibration
+
+# ubloxd -- read raw ublox data and converts them in readable format
+#   subscribes: ubloxRaw
+#   publishes: ubloxGnss
+
+# **** LOGGING SERVICE ****
+
+# loggerd
+#   subscribes: EVERYTHING
+
+# **** NON VITAL SERVICES ****
+
+# ui
+#   subscribes: thermal, model, controlsState, uiLayout, liveCalibration, radarState, liveMpc, plusFrame, liveMapData
+
+# uploader
+#   communicates through file system with loggerd
+
+# deleter
+#   communicates through file system with loggerd and uploader
+
+# logmessaged -- central logging service, can log to cloud
+#   publishes:  logMessage
+
+# logcatd -- fetches logcat info from android
+#   publishes:  androidLog
+
+# proclogd -- fetches process information
+#   publishes: procLog
+
+# tombstoned -- reports native crashes
+
+# athenad -- on request, open a sub socket and return the value
+
+# updated -- waits for network access and tries to update every hour

cereal/services.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import os
+import yaml
+
+
+class Service():
+  def __init__(self, port, should_log, frequency, decimation=None):
+    self.port = port
+    self.should_log = should_log
+    self.frequency = frequency
+    self.decimation = decimation
+
+
+service_list_path = os.path.join(os.path.dirname(__file__), "service_list.yaml")
+
+service_list = {}
+with open(service_list_path, "r") as f:
+  for k, v in yaml.safe_load(f).items():
+    decimation = None
+    if len(v) == 4:
+      decimation = v[3]
+
+    service_list[k] = Service(v[0], v[1], v[2], decimation)
+
+if __name__ == "__main__":
+  print("/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT service_list.yaml */")
+  print("#ifndef __SERVICES_H")
+  print("#define __SERVICES_H")
+  print("struct service { char name[0x100]; int port; bool should_log; int frequency; int decimation; };")
+  print("static struct service services[] = {")
+  for k, v in service_list.items():
+    print('  { .name = "%s", .port = %d, .should_log = %s, .frequency = %d, .decimation = %d },' % (k, v.port, "true" if v.should_log else "false", v.frequency, -1 if v.decimation is None else v.decimation))
+  print("};")
+  print("#endif")

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -0,0 +1,6 @@
+Import('env')
+
+# parser
+env.Command(['common_pyx.so'],
+  ['common_pyx_setup.py', 'clock.pyx'],
+  "cd common && python3 common_pyx_setup.py build_ext --inplace")

common/android.py

@@ -0,0 +1,286 @@
+import os
+import binascii
+import itertools
+import re
+import struct
+import subprocess
+import random
+from cereal import log
+
+NetworkType = log.ThermalData.NetworkType
+NetworkStrength = log.ThermalData.NetworkStrength
+
+ANDROID = os.path.isfile('/EON')
+
+def get_sound_card_online():
+  return (os.path.isfile('/proc/asound/card0/state') and
+          open('/proc/asound/card0/state').read().strip() == 'ONLINE')
+
+def getprop(key):
+  if not ANDROID:
+    return ""
+  return subprocess.check_output(["getprop", key], encoding='utf8').strip()
+
+def get_imei(slot):
+  slot = str(slot)
+  if slot not in ("0", "1"):
+    raise ValueError("SIM slot must be 0 or 1")
+
+  ret = parse_service_call_string(service_call(["iphonesubinfo", "3" , "i32", str(slot)]))
+  if not ret:
+    # allow non android to be identified differently
+    ret = "%015d" % random.randint(0, 1 << 32)
+  return ret
+
+def get_serial():
+  ret = getprop("ro.serialno")
+  if ret == "":
+    ret = "cccccccc"
+  return ret
+
+def get_subscriber_info():
+  ret = parse_service_call_string(service_call(["iphonesubinfo", "7"]))
+  if ret is None or len(ret) < 8:
+    return ""
+  return ret
+
+def reboot(reason=None):
+  if reason is None:
+    reason_args = ["null"]
+  else:
+    reason_args = ["s16", reason]
+
+  subprocess.check_output([
+    "service", "call", "power", "16",  # IPowerManager.reboot
+    "i32", "0",  # no confirmation,
+    *reason_args,
+    "i32", "1"  # wait
+  ])
+
+def service_call(call):
+  if not ANDROID:
+    return None
+
+  ret = subprocess.check_output(["service", "call", *call], encoding='utf8').strip()
+  if 'Parcel' not in ret:
+    return None
+
+  return parse_service_call_bytes(ret)
+
+def parse_service_call_unpack(r, fmt):
+  try:
+    return struct.unpack(fmt, r)[0]
+  except Exception:
+    return None
+
+def parse_service_call_string(r):
+  try:
+    r = r[8:]  # Cut off length field
+    r = r.decode('utf_16_be')
+
+    # All pairs of two characters seem to be swapped. Not sure why
+    result = ""
+    for a, b, in itertools.zip_longest(r[::2], r[1::2], fillvalue='\x00'):
+        result += b + a
+
+    result = result.replace('\x00', '')
+
+    return result
+  except Exception:
+    return None
+
+def parse_service_call_bytes(ret):
+  try:
+    r = b""
+    for hex_part in re.findall(r'[ (]([0-9a-f]{8})', ret):
+      r += binascii.unhexlify(hex_part)
+    return r
+  except Exception:
+    return None
+
+def get_network_type():
+  if not ANDROID:
+    return NetworkType.none
+
+  wifi_check = parse_service_call_string(service_call(["connectivity", "2"]))
+  if wifi_check is None:
+    return NetworkType.none
+  elif 'WIFI' in wifi_check:
+    return NetworkType.wifi
+  else:
+    cell_check = parse_service_call_unpack(service_call(['phone', '59']), ">q")
+    # from TelephonyManager.java
+    cell_networks = {
+      0: NetworkType.none,
+      1: NetworkType.cell2G,
+      2: NetworkType.cell2G,
+      3: NetworkType.cell3G,
+      4: NetworkType.cell2G,
+      5: NetworkType.cell3G,
+      6: NetworkType.cell3G,
+      7: NetworkType.cell3G,
+      8: NetworkType.cell3G,
+      9: NetworkType.cell3G,
+      10: NetworkType.cell3G,
+      11: NetworkType.cell2G,
+      12: NetworkType.cell3G,
+      13: NetworkType.cell4G,
+      14: NetworkType.cell4G,
+      15: NetworkType.cell3G,
+      16: NetworkType.cell2G,
+      17: NetworkType.cell3G,
+      18: NetworkType.cell4G,
+      19: NetworkType.cell4G
+    }
+    return cell_networks.get(cell_check, NetworkType.none)
+
+def get_network_strength(network_type):
+  network_strength = NetworkStrength.unknown
+
+  # from SignalStrength.java
+  def get_lte_level(rsrp, rssnr):
+    INT_MAX = 2147483647
+    if rsrp == INT_MAX:
+      lvl_rsrp = NetworkStrength.unknown
+    elif rsrp >= -95:
+      lvl_rsrp = NetworkStrength.great
+    elif rsrp >= -105:
+      lvl_rsrp = NetworkStrength.good
+    elif rsrp >= -115:
+      lvl_rsrp = NetworkStrength.moderate
+    else:
+      lvl_rsrp = NetworkStrength.poor
+    if rssnr == INT_MAX:
+      lvl_rssnr = NetworkStrength.unknown
+    elif rssnr >= 45:
+      lvl_rssnr = NetworkStrength.great
+    elif rssnr >= 10:
+      lvl_rssnr = NetworkStrength.good
+    elif rssnr >= -30:
+      lvl_rssnr = NetworkStrength.moderate
+    else:
+      lvl_rssnr = NetworkStrength.poor
+    return max(lvl_rsrp, lvl_rssnr)
+
+  def get_tdscdma_level(tdscmadbm):
+    lvl = NetworkStrength.unknown
+    if tdscmadbm > -25:
+      lvl = NetworkStrength.unknown
+    elif tdscmadbm >= -49:
+      lvl = NetworkStrength.great
+    elif tdscmadbm >= -73:
+      lvl = NetworkStrength.good
+    elif tdscmadbm >= -97:
+      lvl = NetworkStrength.moderate
+    elif tdscmadbm >= -110:
+      lvl = NetworkStrength.poor
+    return lvl
+
+  def get_gsm_level(asu):
+    if asu <= 2 or asu == 99:
+      lvl = NetworkStrength.unknown
+    elif asu >= 12:
+      lvl = NetworkStrength.great
+    elif asu >= 8:
+      lvl = NetworkStrength.good
+    elif asu >= 5:
+      lvl = NetworkStrength.moderate
+    else:
+      lvl = NetworkStrength.poor
+    return lvl
+
+  def get_evdo_level(evdodbm, evdosnr):
+    lvl_evdodbm = NetworkStrength.unknown
+    lvl_evdosnr = NetworkStrength.unknown
+    if evdodbm >= -65:
+      lvl_evdodbm = NetworkStrength.great
+    elif evdodbm >= -75:
+      lvl_evdodbm = NetworkStrength.good
+    elif evdodbm >= -90:
+      lvl_evdodbm = NetworkStrength.moderate
+    elif evdodbm >= -105:
+      lvl_evdodbm = NetworkStrength.poor
+    if evdosnr >= 7:
+      lvl_evdosnr = NetworkStrength.great
+    elif evdosnr >= 5:
+      lvl_evdosnr = NetworkStrength.good
+    elif evdosnr >= 3:
+      lvl_evdosnr = NetworkStrength.moderate
+    elif evdosnr >= 1:
+      lvl_evdosnr = NetworkStrength.poor
+    return max(lvl_evdodbm, lvl_evdosnr)
+
+  def get_cdma_level(cdmadbm, cdmaecio):
+    lvl_cdmadbm = NetworkStrength.unknown
+    lvl_cdmaecio = NetworkStrength.unknown
+    if cdmadbm >= -75:
+      lvl_cdmadbm = NetworkStrength.great
+    elif cdmadbm >= -85:
+      lvl_cdmadbm = NetworkStrength.good
+    elif cdmadbm >= -95:
+      lvl_cdmadbm = NetworkStrength.moderate
+    elif cdmadbm >= -100:
+      lvl_cdmadbm = NetworkStrength.poor
+    if cdmaecio >= -90:
+      lvl_cdmaecio = NetworkStrength.great
+    elif cdmaecio >= -110:
+      lvl_cdmaecio = NetworkStrength.good
+    elif cdmaecio >= -130:
+      lvl_cdmaecio = NetworkStrength.moderate
+    elif cdmaecio >= -150:
+      lvl_cdmaecio = NetworkStrength.poor
+    return max(lvl_cdmadbm, lvl_cdmaecio)
+
+  if network_type == NetworkType.none:
+    return network_strength
+  if network_type == NetworkType.wifi:
+    out = subprocess.check_output('dumpsys connectivity', shell=True).decode('utf-8')
+    network_strength = NetworkStrength.unknown
+    for line in out.split('\n'):
+      signal_str = "SignalStrength: "
+      if signal_str in line:
+        lvl_idx_start = line.find(signal_str) + len(signal_str)
+        lvl_idx_end = line.find(']', lvl_idx_start)
+        lvl = int(line[lvl_idx_start : lvl_idx_end])
+        if lvl >= -50:
+          network_strength = NetworkStrength.great
+        elif lvl >= -60:
+          network_strength = NetworkStrength.good
+        elif lvl >= -70:
+          network_strength = NetworkStrength.moderate
+        else:
+          network_strength = NetworkStrength.poor
+    return network_strength
+  else:
+    # check cell strength
+    out = subprocess.check_output('dumpsys telephony.registry', shell=True).decode('utf-8')
+    for line in out.split('\n'):
+      if "mSignalStrength" in line:
+        arr = line.split(' ')
+        ns = 0
+        if ("gsm" in arr[14]):
+          rsrp = int(arr[9])
+          rssnr = int(arr[11])
+          ns = get_lte_level(rsrp, rssnr)
+          if ns == NetworkStrength.unknown:
+            tdscmadbm = int(arr[13])
+            ns = get_tdscdma_level(tdscmadbm)
+            if ns == NetworkStrength.unknown:
+              asu = int(arr[1])
+              ns = get_gsm_level(asu)
+        else:
+          cdmadbm = int(arr[3])
+          cdmaecio = int(arr[4])
+          evdodbm = int(arr[5])
+          evdosnr = int(arr[7])
+          lvl_cdma = get_cdma_level(cdmadbm, cdmaecio)
+          lvl_edmo = get_evdo_level(evdodbm, evdosnr)
+          if lvl_edmo == NetworkStrength.unknown:
+            ns = lvl_cdma
+          elif lvl_cdma == NetworkStrength.unknown:
+            ns = lvl_edmo
+          else:
+            ns = min(lvl_cdma, lvl_edmo)
+        network_strength = max(network_strength, ns)
+
+    return network_strength

common/api/__init__.py

@@ -1,7 +1,34 @@
+import jwt
 import requests
-
+from datetime import datetime, timedelta
+from common.basedir import PERSIST
 from selfdrive.version import version
 
+class Api():
+  def __init__(self, dongle_id):
+    self.dongle_id = dongle_id
+    with open(PERSIST+'/comma/id_rsa') as f:
+      self.private_key = f.read()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def get_token(self):
+    now = datetime.utcnow()
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=1)
+    }
+    return jwt.encode(payload, self.private_key, algorithm='RS256').decode('utf8')
+
 def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
   backend = "https://api.commadotai.com/"
 
@@ -11,5 +38,4 @@ def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
 
   headers['User-Agent'] = "openpilot-" + version
 
-  return requests.request(method, backend+endpoint, timeout=timeout, headers = headers, params=params)
-
+  return requests.request(method, backend+endpoint, timeout=timeout, headers=headers, params=params)

common/apk.py

@@ -0,0 +1,97 @@
+import os
+import subprocess
+import glob
+import hashlib
+import shutil
+from common.basedir import BASEDIR
+from selfdrive.swaglog import cloudlog
+
+android_packages = ("ai.comma.plus.offroad",)
+
+def get_installed_apks():
+  dat = subprocess.check_output(["pm", "list", "packages", "-f"], encoding='utf8').strip().split("\n")
+  ret = {}
+  for x in dat:
+    if x.startswith("package:"):
+      v, k = x.split("package:")[1].split("=")
+      ret[k] = v
+  return ret
+
+def install_apk(path):
+  # can only install from world readable path
+  install_path = "/sdcard/%s" % os.path.basename(path)
+  shutil.copyfile(path, install_path)
+
+  ret = subprocess.call(["pm", "install", "-r", install_path])
+  os.remove(install_path)
+  return ret == 0
+
+def start_offroad():
+  set_package_permissions()
+  system("am start -n ai.comma.plus.offroad/.MainActivity")
+
+def set_package_permissions():
+  pm_grant("ai.comma.plus.offroad", "android.permission.ACCESS_FINE_LOCATION")
+  pm_grant("ai.comma.plus.offroad", "android.permission.READ_PHONE_STATE")
+  pm_grant("ai.comma.plus.offroad", "android.permission.READ_EXTERNAL_STORAGE")
+  appops_set("ai.comma.plus.offroad", "SU", "allow")
+  appops_set("ai.comma.plus.offroad", "WIFI_SCAN", "allow")
+
+def appops_set(package, op, mode):
+  system(f"LD_LIBRARY_PATH= appops set {package} {op} {mode}")
+
+def pm_grant(package, permission):
+  system(f"pm grant {package} {permission}")
+
+def system(cmd):
+  try:
+    cloudlog.info("running %s" % cmd)
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
+  except subprocess.CalledProcessError as e:
+    cloudlog.event("running failed",
+      cmd=e.cmd,
+      output=e.output[-1024:],
+      returncode=e.returncode)
+
+# *** external functions ***
+
+def update_apks():
+  # install apks
+  installed = get_installed_apks()
+
+  install_apks = glob.glob(os.path.join(BASEDIR, "apk/*.apk"))
+  for apk in install_apks:
+    app = os.path.basename(apk)[:-4]
+    if app not in installed:
+      installed[app] = None
+
+  cloudlog.info("installed apks %s" % (str(installed), ))
+
+  for app in installed.keys():
+    apk_path = os.path.join(BASEDIR, "apk/"+app+".apk")
+    if not os.path.exists(apk_path):
+      continue
+
+    h1 = hashlib.sha1(open(apk_path, 'rb').read()).hexdigest()
+    h2 = None
+    if installed[app] is not None:
+      h2 = hashlib.sha1(open(installed[app], 'rb').read()).hexdigest()
+      cloudlog.info("comparing version of %s  %s vs %s" % (app, h1, h2))
+
+    if h2 is None or h1 != h2:
+      cloudlog.info("installing %s" % app)
+
+      success = install_apk(apk_path)
+      if not success:
+        cloudlog.info("needing to uninstall %s" % app)
+        system("pm uninstall %s" % app)
+        success = install_apk(apk_path)
+
+      assert success
+
+def pm_apply_packages(cmd):
+  for p in android_packages:
+    system("pm %s %s" % (cmd, p))
+
+if __name__ == "__main__":
+  update_apks()

common/basedir.py

@@ -1,4 +1,10 @@
 import os
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
 
-
+from common.android import ANDROID
+if ANDROID:
+  PERSIST = "/persist"
+  PARAMS = "/data/params"
+else:
+  PERSIST = os.path.join(BASEDIR, "persist")
+  PARAMS = os.path.join(BASEDIR, "persist", "params")

common/clock.pyx

@@ -0,0 +1,22 @@
+from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
+
+IF UNAME_SYSNAME == "Darwin":
+  # Darwin doesn't have a CLOCK_BOOTTIME
+  CLOCK_BOOTTIME = CLOCK_MONOTONIC_RAW
+ELSE:
+  from posix.time cimport CLOCK_BOOTTIME
+
+cdef double readclock(clockid_t clock_id):
+  cdef timespec ts
+  cdef double current
+
+  clock_gettime(clock_id, &ts)
+  current = ts.tv_sec + (ts.tv_nsec / 1000000000.)
+  return current
+
+def monotonic_time():
+  return readclock(CLOCK_MONOTONIC_RAW)
+
+def sec_since_boot():
+  return readclock(CLOCK_BOOTTIME)
+

common/common_pyx_setup.py

@@ -0,0 +1,20 @@
+from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
+from Cython.Build import cythonize
+
+from common.cython_hacks import BuildExtWithoutPlatformSuffix
+
+sourcefiles = ['clock.pyx']
+extra_compile_args = ["-std=c++11"]
+
+setup(name='Common',
+      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
+      ext_modules=cythonize(
+        Extension(
+          "common_pyx",
+          language="c++",
+          sources=sourcefiles,
+          extra_compile_args=extra_compile_args,
+        )
+      ),
+      nthreads=4,
+)

common/cython_hacks.py

@@ -0,0 +1,23 @@
+import os
+import sysconfig
+from Cython.Distutils import build_ext
+
+def get_ext_filename_without_platform_suffix(filename):
+  name, ext = os.path.splitext(filename)
+  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+  if ext_suffix == ext:
+    return filename
+
+  ext_suffix = ext_suffix.replace(ext, '')
+  idx = name.find(ext_suffix)
+
+  if idx == -1:
+    return filename
+  else:
+    return name[:idx] + ext
+
+class BuildExtWithoutPlatformSuffix(build_ext):
+  def get_ext_filename(self, ext_name):
+    filename = super().get_ext_filename(ext_name)
+    return get_ext_filename_without_platform_suffix(filename)

common/ffi_wrapper.py

@@ -0,0 +1,55 @@
+import os
+import sys
+import fcntl
+import hashlib
+import platform
+from cffi import FFI
+
+def suffix():
+  if platform.system() == "Darwin":
+    return ".dylib"
+  else:
+    return ".so"
+
+def ffi_wrap(name, c_code, c_header, tmpdir="/tmp/ccache", cflags="", libraries=None):
+  if libraries is None:
+    libraries = []
+
+  cache = name + "_" + hashlib.sha1(c_code.encode('utf-8')).hexdigest()
+  try:
+    os.mkdir(tmpdir)
+  except OSError:
+    pass
+
+  fd = os.open(tmpdir, 0)
+  fcntl.flock(fd, fcntl.LOCK_EX)
+  try:
+    sys.path.append(tmpdir)
+    try:
+      mod = __import__(cache)
+    except Exception:
+      print("cache miss {0}".format(cache))
+      compile_code(cache, c_code, c_header, tmpdir, cflags, libraries)
+      mod = __import__(cache)
+  finally:
+    os.close(fd)
+
+  return mod.ffi, mod.lib
+
+
+def compile_code(name, c_code, c_header, directory, cflags="", libraries=None):
+  if libraries is None:
+    libraries = []
+
+  ffibuilder = FFI()
+  ffibuilder.set_source(name, c_code, source_extension='.cpp', libraries=libraries)
+  ffibuilder.cdef(c_header)
+  os.environ['OPT'] = "-fwrapv -O2 -DNDEBUG -std=c++11"
+  os.environ['CFLAGS'] = cflags
+  ffibuilder.compile(verbose=True, debug=False, tmpdir=directory)
+
+
+def wrap_compiled(name, directory):
+  sys.path.append(directory)
+  mod = __import__(name)
+  return mod.ffi, mod.lib

common/file_helpers.py

@@ -0,0 +1,122 @@
+import os
+import shutil
+import tempfile
+from atomicwrites import AtomicWriter
+
+
+def mkdirs_exists_ok(path):
+  if path.startswith('http://') or path.startswith('https://'):
+    raise ValueError('URL path')
+  try:
+    os.makedirs(path)
+  except OSError:
+    if not os.path.isdir(path):
+      raise
+
+
+def rm_not_exists_ok(path):
+  try:
+    os.remove(path)
+  except OSError:
+    if os.path.exists(path):
+      raise
+
+
+def rm_tree_or_link(path):
+  if os.path.islink(path):
+    os.unlink(path)
+  elif os.path.isdir(path):
+    shutil.rmtree(path)
+
+
+def get_tmpdir_on_same_filesystem(path):
+  normpath = os.path.normpath(path)
+  parts = normpath.split("/")
+  if len(parts) > 1 and parts[1] == "scratch":
+    return "/scratch/tmp"
+  elif len(parts) > 2 and parts[2] == "runner":
+    return "/{}/runner/tmp".format(parts[1])
+  return "/tmp"
+
+
+class AutoMoveTempdir():
+  def __init__(self, target_path, temp_dir=None):
+    self._target_path = target_path
+    self._path = tempfile.mkdtemp(dir=temp_dir)
+
+  @property
+  def name(self):
+    return self._path
+
+  def close(self):
+    os.rename(self._path, self._target_path)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    if exc_type is None:
+      self.close()
+    else:
+      shutil.rmtree(self._path)
+
+
+class NamedTemporaryDir():
+  def __init__(self, temp_dir=None):
+    self._path = tempfile.mkdtemp(dir=temp_dir)
+
+  @property
+  def name(self):
+    return self._path
+
+  def close(self):
+    shutil.rmtree(self._path)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+def _get_fileobject_func(writer, temp_dir):
+  def _get_fileobject():
+    file_obj = writer.get_fileobject(dir=temp_dir)
+    os.chmod(file_obj.name, 0o644)
+    return file_obj
+  return _get_fileobject
+
+
+def atomic_write_on_fs_tmp(path, **kwargs):
+  """Creates an atomic writer using a temporary file in a temporary directory
+     on the same filesystem as path.
+  """
+  # TODO(mgraczyk): This use of AtomicWriter relies on implementation details to set the temp
+  #                 directory.
+  writer = AtomicWriter(path, **kwargs)
+  return writer._open(_get_fileobject_func(writer, get_tmpdir_on_same_filesystem(path)))
+
+
+def atomic_write_in_dir(path, **kwargs):
+  """Creates an atomic writer using a temporary file in the same directory
+     as the destination file.
+  """
+  writer = AtomicWriter(path, **kwargs)
+  return writer._open(_get_fileobject_func(writer, os.path.dirname(path)))
+
+
+def atomic_write_in_dir_neos(path, contents, mode=None):
+  """
+  Atomically writes contents to path using a temporary file in the same directory
+  as path. Useful on NEOS, where `os.link` (required by atomic_write_in_dir) is missing.
+  """
+
+  f = tempfile.NamedTemporaryFile(delete=False, prefix=".tmp", dir=os.path.dirname(path))
+  f.write(contents)
+  f.flush()
+  if mode is not None:
+    os.fchmod(f.fileno(), mode)
+  os.fsync(f.fileno())
+  f.close()
+
+  os.rename(f.name, path)

common/filter_simple.py

@@ -0,0 +1,9 @@
+class FirstOrderFilter():
+  # first order filter
+  def __init__(self, x0, ts, dt):
+    self.k = (dt / ts) / (1. + dt / ts)
+    self.x = x0
+
+  def update(self, x):
+    self.x = (1. - self.k) * self.x + self.k * x
+    return self.x

common/kalman/.gitignore

@@ -0,0 +1 @@
+simple_kalman_impl.c

common/kalman/SConscript

@@ -0,0 +1,6 @@
+Import('env')
+
+env.Command(['simple_kalman_impl.so'],
+  ['simple_kalman_impl.pyx', 'simple_kalman_impl.pxd', 'simple_kalman_setup.py'],
+  "cd common/kalman && python3 simple_kalman_setup.py build_ext --inplace")
+

common/kalman/ekf.py

@@ -1,253 +0,0 @@
-# pylint: skip-file
-from __future__ import print_function
-import abc
-import numpy as np
-# The EKF class contains the framework for an Extended Kalman Filter, but must be subclassed to use.
-# A subclass must implement:
-#   1) calc_transfer_fun(); see bottom of file for more info.
-#   2) __init__() to initialize self.state, self.covar, and self.process_noise appropriately
-
-# Alternatively, the existing implementations of EKF can be used (e.g. EKF2D)
-
-# Sensor classes are optionally used to pass measurement information into the EKF, to keep
-#   sensor parameters and processing methods for a each sensor together.
-# Sensor classes have a read() method which takes raw sensor data and returns
-#   a SensorReading object, which can be passed to the EKF update() method.
-
-# For usage, see run_ekf1d.py in selfdrive/new for a simple example.
-# ekf.predict(dt) should be called between update cycles with the time since it was last called.
-# Ideally, predict(dt) should be called at a relatively constant rate.
-# update() should be called once per sensor, and can be called multiple times between predict steps.
-# Access and set the state of the filter directly with ekf.state and ekf.covar.
-
-
-class SensorReading:
-  # Given a perfect model and no noise, data = obs_model * state
-  def __init__(self, data, covar, obs_model):
-    self.data = data
-    self.obs_model = obs_model
-    self.covar = covar
-
-  def __repr__(self):
-    return "SensorReading(data={}, covar={}, obs_model={})".format(
-      repr(self.data), repr(self.covar), repr(self.obs_model))
-
-
-# A generic sensor class that does no pre-processing of data
-class SimpleSensor:
-  # obs_model can be
-  #   a full observation model matrix, or
-  #   an integer or tuple of indices into ekf.state, indicating which variables are being directly observed
-  # covar can be
-  #   a full covariance matrix
-  #   a float or tuple of individual covars for each component of the sensor reading
-  # dims is the number of states in the EKF
-  def __init__(self, obs_model, covar, dims):
-    # Allow for integer covar/obs_model
-    if not hasattr(obs_model, "__len__"):
-      obs_model = (obs_model, )
-    if not hasattr(covar, "__len__"):
-      covar = (covar, )
-
-    # Full observation model passed
-    if dims in np.array(obs_model).shape:
-      self.obs_model = np.asmatrix(obs_model)
-      self.covar = np.asmatrix(covar)
-    # Indices of unit observations passed
-    else:
-      self.obs_model = np.matlib.zeros((len(obs_model), dims))
-      self.obs_model[:, list(obs_model)] = np.identity(len(obs_model))
-      if np.asarray(covar).ndim == 2:
-        self.covar = np.asmatrix(covar)
-      elif len(covar) == len(obs_model):
-        self.covar = np.matlib.diag(covar)
-      else:
-        self.covar = np.matlib.identity(len(obs_model)) * covar
-
-  def read(self, data, covar=None):
-    if covar:
-      self.covar = covar
-    return SensorReading(data, self.covar, self.obs_model)
-
-
-class EKF:
-  __metaclass__ = abc.ABCMeta
-
-  def __init__(self, debug=False):
-    self.DEBUG = debug
-
-
-  def __str__(self):
-    return "EKF(state={}, covar={})".format(self.state, self.covar)
-
-  # Measurement update
-  # Reading should be a SensorReading object with data, covar, and obs_model attributes
-  def update(self, reading):
-    # Potential improvements:
-    # deal with negative covars
-    # add noise to really low covars to ensure stability
-    # use mahalanobis distance to reject outliers
-    # wrap angles after state updates and innovation
-
-    # y = z - H*x
-    innovation = reading.data - reading.obs_model * self.state
-
-    if self.DEBUG:
-      print("reading:\n",reading.data)
-      print("innovation:\n",innovation)
-
-    # S = H*P*H' + R
-    innovation_covar = reading.obs_model * self.covar * reading.obs_model.T + reading.covar
-
-    # K = P*H'*S^-1
-    kalman_gain = self.covar * reading.obs_model.T * np.linalg.inv(
-      innovation_covar)
-
-    if self.DEBUG:
-      print("gain:\n", kalman_gain)
-      print("innovation_covar:\n", innovation_covar)
-      print("innovation: ", innovation)
-      print("test: ", self.covar * reading.obs_model.T * (
-        reading.obs_model * self.covar * reading.obs_model.T + reading.covar *
-        0).I)
-
-    # x = x + K*y
-    self.state += kalman_gain*innovation
-
-    # print "covar", np.diag(self.covar)
-    #self.state[(roll_vel, yaw_vel, pitch_vel),:] = reading.data
-
-    # Standard form: P = (I - K*H)*P
-    # self.covar = (self.identity - kalman_gain*reading.obs_model) * self.covar
-
-    # Use the Joseph form for numerical stability: P = (I-K*H)*P*(I - K*H)' + K*R*K'
-    aux_mtrx = (self.identity - kalman_gain * reading.obs_model)
-    self.covar = aux_mtrx * self.covar * aux_mtrx.T + kalman_gain * reading.covar * kalman_gain.T
-
-    if self.DEBUG:
-      print("After update")
-      print("state\n", self.state)
-      print("covar:\n",self.covar)
-
-  def update_scalar(self, reading):
-    # like update but knowing that measurement is a scalar
-    # this avoids matrix inversions and speeds up (surprisingly) drived.py a lot
-
-    # innovation = reading.data - np.matmul(reading.obs_model, self.state)
-    # innovation_covar = np.matmul(np.matmul(reading.obs_model, self.covar), reading.obs_model.T) + reading.covar
-    # kalman_gain = np.matmul(self.covar, reading.obs_model.T)/innovation_covar
-    # self.state += np.matmul(kalman_gain, innovation)
-    # aux_mtrx = self.identity - np.matmul(kalman_gain, reading.obs_model)
-    # self.covar =  np.matmul(aux_mtrx, np.matmul(self.covar, aux_mtrx.T)) + np.matmul(kalman_gain, np.matmul(reading.covar, kalman_gain.T))
-
-    # written without np.matmul
-    es = np.einsum
-    ABC_T = "ij,jk,lk->il"
-    AB_T = "ij,kj->ik"
-    AB = "ij,jk->ik"
-    innovation = reading.data - es(AB, reading.obs_model, self.state)
-    innovation_covar = es(ABC_T, reading.obs_model, self.covar,
-                          reading.obs_model) + reading.covar
-    kalman_gain = es(AB_T, self.covar, reading.obs_model) / innovation_covar
-
-    self.state += es(AB, kalman_gain, innovation)
-    aux_mtrx = self.identity - es(AB, kalman_gain, reading.obs_model)
-    self.covar = es(ABC_T, aux_mtrx, self.covar, aux_mtrx) + \
-                 es(ABC_T, kalman_gain, reading.covar, kalman_gain)
-
-  # Prediction update
-  def predict(self, dt):
-    es = np.einsum
-    ABC_T = "ij,jk,lk->il"
-    AB = "ij,jk->ik"
-
-    # State update
-    transfer_fun, transfer_fun_jacobian = self.calc_transfer_fun(dt)
-
-    # self.state = np.matmul(transfer_fun, self.state)
-    # self.covar = np.matmul(np.matmul(transfer_fun_jacobian, self.covar), transfer_fun_jacobian.T) + self.process_noise * dt
-
-    # x = f(x, u), written in the form x = A(x, u)*x
-    self.state = es(AB, transfer_fun, self.state)
-
-    # P = J*P*J' + Q
-    self.covar = es(ABC_T, transfer_fun_jacobian, self.covar,
-                    transfer_fun_jacobian) + self.process_noise * dt  #!dt
-
-    #! Clip covariance to avoid explosions
-    self.covar = np.clip(self.covar,-1e10,1e10)
-
-  @abc.abstractmethod
-  def calc_transfer_fun(self, dt):
-    """Return a tuple with the transfer function and transfer function jacobian
-    The transfer function and jacobian should both be a numpy matrix of size DIMSxDIMS
-
-    The transfer function matrix A should satisfy the state-update equation
-      x_(k+1) = A * x_k
-
-    The jacobian J is the direct jacobian A*x_k. For linear systems J=A.
-
-    Current implementations calculate A and J as functions of state. Control input
-      can be added trivially by adding a control parameter to predict() and calc_tranfer_update(),
-      and using it during calculation of A and J
-    """
-
-
-class FastEKF1D(EKF):
-  """Fast version of EKF for 1D problems with scalar readings."""
-
-  def __init__(self, dt, var_init, Q):
-    super(FastEKF1D, self).__init__(False)
-    self.state = [0, 0]
-    self.covar = [var_init, var_init, 0]
-
-    # Process Noise
-    self.dtQ0 = dt * Q[0]
-    self.dtQ1 = dt * Q[1]
-
-  def update(self, reading):
-    raise NotImplementedError
-
-  def update_scalar(self, reading):
-    # TODO(mgraczyk): Delete this for speed.
-    # assert np.all(reading.obs_model == [1, 0])
-
-    rcov = reading.covar[0, 0]
-
-    x = self.state
-    S = self.covar
-
-    innovation = reading.data - x[0]
-    innovation_covar = S[0] + rcov
-
-    k0 = S[0] / innovation_covar
-    k1 = S[2] / innovation_covar
-
-    x[0] += k0 * innovation
-    x[1] += k1 * innovation
-
-    mk = 1 - k0
-    S[1] += k1 * (k1 * (S[0] + rcov) - 2 * S[2])
-    S[2] = mk * (S[2] - k1 * S[0]) + rcov * k0 * k1
-    S[0] = mk * mk * S[0] + rcov * k0 * k0
-
-  def predict(self, dt):
-    # State update
-    x = self.state
-
-    x[0] += dt * x[1]
-
-    # P = J*P*J' + Q
-    S = self.covar
-    S[0] += dt * (2 * S[2] + dt * S[1]) + self.dtQ0
-    S[2] += dt * S[1]
-    S[1] += self.dtQ1
-
-    # Clip covariance to avoid explosions
-    S = max(-1e10, min(S, 1e10))
-
-  def calc_transfer_fun(self, dt):
-    tf = np.identity(2)
-    tf[0, 1] = dt
-    tfj = tf
-    return tf, tfj

common/kalman/simple_kalman.py

@@ -1,23 +1,3 @@
-import numpy as np
-
-
-class KF1D:
-  # this EKF assumes constant covariance matrix, so calculations are much simpler
-  # the Kalman gain also needs to be precomputed using the control module
-
-  def __init__(self, x0, A, C, K):
-    self.x = x0
-    self.A = A
-    self.C = C
-    self.K = K
-
-    self.A_K = self.A - np.dot(self.K, self.C)
-
-    # K matrix needs to  be pre-computed as follow:
-    # import control
-    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
-    # self.K = np.transpose(K)
-
-  def update(self, meas):
-    self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
-    return self.x
+# pylint: skip-file
+from common.kalman.simple_kalman_impl import KF1D as KF1D
+assert KF1D

common/kalman/simple_kalman_impl.pxd

@@ -0,0 +1,16 @@
+cdef class KF1D:
+  cdef public:
+    double x0_0
+    double x1_0
+    double K0_0
+    double K1_0
+    double A0_0
+    double A0_1
+    double A1_0
+    double A1_1
+    double C0_0
+    double C0_1
+    double A_K_0
+    double A_K_1
+    double A_K_2
+    double A_K_3

common/kalman/simple_kalman_impl.pyx

@@ -0,0 +1,36 @@
+# cython: language_level=3
+
+cdef class KF1D:
+  def __init__(self, x0, A, C, K):
+    self.x0_0 = x0[0][0]
+    self.x1_0 = x0[1][0]
+    self.A0_0 = A[0][0]
+    self.A0_1 = A[0][1]
+    self.A1_0 = A[1][0]
+    self.A1_1 = A[1][1]
+    self.C0_0 = C[0]
+    self.C0_1 = C[1]
+    self.K0_0 = K[0][0]
+    self.K1_0 = K[1][0]
+
+    self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
+    self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
+    self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
+    self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
+
+  def update(self, meas):
+    cdef double x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
+    cdef double x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
+    self.x0_0 = x0_0
+    self.x1_0 = x1_0
+
+    return [self.x0_0, self.x1_0]
+
+  @property
+  def x(self):
+    return [[self.x0_0], [self.x1_0]]
+
+  @x.setter
+  def x(self, x):
+    self.x0_0 = x[0][0]
+    self.x1_0 = x[1][0]

common/kalman/simple_kalman_old.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+
+class KF1D:
+  # this EKF assumes constant covariance matrix, so calculations are much simpler
+  # the Kalman gain also needs to be precomputed using the control module
+
+  def __init__(self, x0, A, C, K):
+    self.x = x0
+    self.A = A
+    self.C = C
+    self.K = K
+
+    self.A_K = self.A - np.dot(self.K, self.C)
+
+    # K matrix needs to  be pre-computed as follow:
+    # import control
+    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
+    # self.K = np.transpose(K)
+
+  def update(self, meas):
+    self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
+    return self.x

common/kalman/simple_kalman_setup.py

@@ -0,0 +1,9 @@
+from distutils.core import Extension, setup
+
+from Cython.Build import cythonize
+
+from common.cython_hacks import BuildExtWithoutPlatformSuffix
+
+setup(name='Simple Kalman Implementation',
+      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
+      ext_modules=cythonize(Extension("simple_kalman_impl", ["simple_kalman_impl.pyx"])))

common/kalman/tests/test_simple_kalman.py

@@ -0,0 +1,84 @@
+import unittest
+import random
+import timeit
+import numpy as np
+
+from common.kalman.simple_kalman import KF1D
+from common.kalman.simple_kalman_old import KF1D as KF1D_old
+
+
+class TestSimpleKalman(unittest.TestCase):
+  def setUp(self):
+    dt = 0.01
+    x0_0 = 0.0
+    x1_0 = 0.0
+    A0_0 = 1.0
+    A0_1 = dt
+    A1_0 = 0.0
+    A1_1 = 1.0
+    C0_0 = 1.0
+    C0_1 = 0.0
+    K0_0 = 0.12287673
+    K1_0 = 0.29666309
+
+    self.kf_old = KF1D_old(x0=np.matrix([[x0_0], [x1_0]]),
+                           A=np.matrix([[A0_0, A0_1], [A1_0, A1_1]]),
+                           C=np.matrix([C0_0, C0_1]),
+                           K=np.matrix([[K0_0], [K1_0]]))
+
+    self.kf = KF1D(x0=[[x0_0], [x1_0]],
+                   A=[[A0_0, A0_1], [A1_0, A1_1]],
+                   C=[C0_0, C0_1],
+                   K=[[K0_0], [K1_0]])
+
+  def test_getter_setter(self):
+    self.kf.x = [[1.0], [1.0]]
+    self.assertEqual(self.kf.x, [[1.0], [1.0]])
+
+  def update_returns_state(self):
+      x = self.kf.update(100)
+      self.assertEqual(x, self.kf.x)
+
+  def test_old_equal_new(self):
+    for _ in range(1000):
+      v_wheel = random.uniform(0, 200)
+
+      x_old = self.kf_old.update(v_wheel)
+      x = self.kf.update(v_wheel)
+
+      # Compare the output x, verify that the error is less than 1e-4
+      self.assertAlmostEqual(x_old[0], x[0])
+      self.assertAlmostEqual(x_old[1], x[1])
+
+  def test_new_is_faster(self):
+    setup = """
+import numpy as np
+
+from common.kalman.simple_kalman import KF1D
+from common.kalman.simple_kalman_old import KF1D as KF1D_old
+
+dt = 0.01
+x0_0 = 0.0
+x1_0 = 0.0
+A0_0 = 1.0
+A0_1 = dt
+A1_0 = 0.0
+A1_1 = 1.0
+C0_0 = 1.0
+C0_1 = 0.0
+K0_0 = 0.12287673
+K1_0 = 0.29666309
+
+kf_old = KF1D_old(x0=np.matrix([[x0_0], [x1_0]]),
+                  A=np.matrix([[A0_0, A0_1], [A1_0, A1_1]]),
+                  C=np.matrix([C0_0, C0_1]),
+                  K=np.matrix([[K0_0], [K1_0]]))
+
+kf = KF1D(x0=[[x0_0], [x1_0]],
+          A=[[A0_0, A0_1], [A1_0, A1_1]],
+          C=[C0_0, C0_1],
+          K=[[K0_0], [K1_0]])
+    """
+    kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000)
+    kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000)
+    self.assertTrue(kf_speed < kf_old_speed / 4)

common/logging_extra.py

@@ -1,9 +1,11 @@
+import io
 import os
 import sys
 import copy
 import json
 import socket
 import logging
+import traceback
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
@@ -66,8 +68,11 @@ class SwagErrorFilter(logging.Filter):
   def filter(self, record):
     return record.levelno < logging.ERROR
 
-_tmpfunc = lambda: 0
-_srcfile = os.path.normcase(_tmpfunc.__code__.co_filename)
+def _tmpfunc():
+  return 0
+
+def _srcfile():
+  return os.path.normcase(_tmpfunc.__code__.co_filename)
 
 class SwagLogger(logging.Logger):
   def __init__(self):
@@ -78,28 +83,6 @@ class SwagLogger(logging.Logger):
     self.log_local = local()
     self.log_local.ctx = {}
 
-  def findCaller(self, stack_info=None):
-    """
-      Find the stack frame of the caller so that we can note the source
-      file name, line number and function name.
-      """
-    # f = currentframe()
-    f = sys._getframe(3)
-    #On some versions of IronPython, currentframe() returns None if
-    #IronPython isn't run with -X:Frames.
-    if f is not None:
-      f = f.f_back
-    rv = "(unknown file)", 0, "(unknown function)"
-    while hasattr(f, "f_code"):
-      co = f.f_code
-      filename = os.path.normcase(co.co_filename)
-      if filename in (logging._srcfile, _srcfile):
-        f = f.f_back
-        continue
-      rv = (co.co_filename, f.f_lineno, co.co_name)
-      break
-    return rv
-
   def local_ctx(self):
     try:
       return self.log_local.ctx
@@ -132,14 +115,49 @@ class SwagLogger(logging.Logger):
     if args:
       evt['args'] = args
     evt.update(kwargs)
-    ctx = self.get_ctx()
-    if ctx:
-      evt['ctx'] = self.get_ctx()
     if 'error' in kwargs:
       self.error(evt)
     else:
       self.info(evt)
 
+  def findCaller(self, stack_info=False, stacklevel=1):
+    """
+    Find the stack frame of the caller so that we can note the source
+    file name, line number and function name.
+    """
+    f = sys._getframe(3)
+    #On some versions of IronPython, currentframe() returns None if
+    #IronPython isn't run with -X:Frames.
+    if f is not None:
+        f = f.f_back
+    orig_f = f
+    while f and stacklevel > 1:
+        f = f.f_back
+        stacklevel -= 1
+    if not f:
+        f = orig_f
+    rv = "(unknown file)", 0, "(unknown function)", None
+    while hasattr(f, "f_code"):
+        co = f.f_code
+        filename = os.path.normcase(co.co_filename)
+
+        # TODO: is this pylint exception correct?
+        if filename == _srcfile:  # pylint: disable=comparison-with-callable
+            f = f.f_back
+            continue
+        sinfo = None
+        if stack_info:
+            sio = io.StringIO()
+            sio.write('Stack (most recent call last):\n')
+            traceback.print_stack(f, file=sio)
+            sinfo = sio.getvalue()
+            if sinfo[-1] == '\n':
+                sinfo = sinfo[:-1]
+            sio.close()
+        rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
+        break
+    return rv
+
 if __name__ == "__main__":
   log = SwagLogger()
 

common/numpy_fast.py

@@ -6,13 +6,17 @@ def clip(x, lo, hi):
 
 def interp(x, xp, fp):
   N = len(xp)
+
   def get_interp(xv):
     hi = 0
     while hi < N and xv > xp[hi]:
       hi += 1
     low = hi - 1
     return fp[-1] if hi == N and xv > xp[low] else (
-      fp[0] if hi == 0 else 
+      fp[0] if hi == 0 else
       (xv - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]) + fp[low])
-  return [get_interp(v) for v in x] if hasattr(
-    x, '__iter__') else get_interp(x)
+
+  return [get_interp(v) for v in x] if hasattr(x, '__iter__') else get_interp(x)
+
+def mean(x):
+  return sum(x) / len(x)

common/params.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """ROS has a parameter server, we have files.
 
 The parameter store is a persistent key value store, implemented as a directory with a writer lock.
@@ -23,11 +23,13 @@ file in place without messing with <params_dir>/d.
 import time
 import os
 import errno
-import sys
 import shutil
 import fcntl
 import tempfile
+import threading
 from enum import Enum
+from common.basedir import PARAMS
+
 
 def mkdirs_exists_ok(path):
   try:
@@ -36,51 +38,79 @@ def mkdirs_exists_ok(path):
     if not os.path.isdir(path):
       raise
 
+
 class TxType(Enum):
   PERSISTENT = 1
   CLEAR_ON_MANAGER_START = 2
-  CLEAR_ON_CAR_START = 3
+  CLEAR_ON_PANDA_DISCONNECT = 3
+
 
 class UnknownKeyName(Exception):
   pass
 
+
 keys = {
-# written: manager
-# read:    loggerd, uploaderd, offroad
-  "DongleId": TxType.PERSISTENT,
-  "AccessToken": TxType.PERSISTENT,
-  "Version": TxType.PERSISTENT,
-  "TrainingVersion": TxType.PERSISTENT,
-  "GitCommit": TxType.PERSISTENT,
-  "GitBranch": TxType.PERSISTENT,
-  "GitRemote": TxType.PERSISTENT,
-# written: baseui
-# read:    ui, controls
-  "IsMetric": TxType.PERSISTENT,
-  "IsFcwEnabled": TxType.PERSISTENT,
-  "HasAcceptedTerms": TxType.PERSISTENT,
-  "CompletedTrainingVersion": TxType.PERSISTENT,
-  "IsUploadVideoOverCellularEnabled": TxType.PERSISTENT,
-  "IsDriverMonitoringEnabled": TxType.PERSISTENT,
-  "IsGeofenceEnabled": TxType.PERSISTENT,
-# written: visiond
-# read:    visiond, controlsd
-  "CalibrationParams": TxType.PERSISTENT,
-# written: visiond
-# read:    visiond, ui
-  "CloudCalibration": TxType.PERSISTENT,
-# written: controlsd
-# read:    radard
-  "CarParams": TxType.CLEAR_ON_CAR_START,
-
-  "Passive": TxType.PERSISTENT,
-  "DoUninstall": TxType.CLEAR_ON_MANAGER_START,
-  "ShouldDoUpdate": TxType.CLEAR_ON_MANAGER_START,
-  "IsUpdateAvailable": TxType.PERSISTENT,
-
-  "RecordFront": TxType.PERSISTENT,
+  "AccessToken": [TxType.CLEAR_ON_MANAGER_START],
+  "AthenadPid": [TxType.PERSISTENT],
+  "CalibrationParams": [TxType.PERSISTENT],
+  "CarParams": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "CarParamsCache": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "CarVin": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "CommunityFeaturesToggle": [TxType.PERSISTENT],
+  "CompletedTrainingVersion": [TxType.PERSISTENT],
+  "ControlsParams": [TxType.PERSISTENT],
+  "DisablePowerDown": [TxType.PERSISTENT],
+  "DisableUpdates": [TxType.PERSISTENT],
+  "DoUninstall": [TxType.CLEAR_ON_MANAGER_START],
+  "DongleId": [TxType.PERSISTENT],
+  "GitBranch": [TxType.PERSISTENT],
+  "GitCommit": [TxType.PERSISTENT],
+  "GitRemote": [TxType.PERSISTENT],
+  "GithubSshKeys": [TxType.PERSISTENT],
+  "HasAcceptedTerms": [TxType.PERSISTENT],
+  "HasCompletedSetup": [TxType.PERSISTENT],
+  "IsDriverViewEnabled": [TxType.CLEAR_ON_MANAGER_START],
+  "IsLdwEnabled": [TxType.PERSISTENT],
+  "IsGeofenceEnabled": [TxType.PERSISTENT],
+  "IsMetric": [TxType.PERSISTENT],
+  "IsOffroad": [TxType.CLEAR_ON_MANAGER_START],
+  "IsRHD": [TxType.PERSISTENT],
+  "IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
+  "IsUpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
+  "IsUploadRawEnabled": [TxType.PERSISTENT],
+  "LastAthenaPingTime": [TxType.PERSISTENT],
+  "LastUpdateTime": [TxType.PERSISTENT],
+  "LimitSetSpeed": [TxType.PERSISTENT],
+  "LimitSetSpeedNeural": [TxType.PERSISTENT],
+  "LiveParameters": [TxType.PERSISTENT],
+  "LongitudinalControl": [TxType.PERSISTENT],
+  "OpenpilotEnabledToggle": [TxType.PERSISTENT],
+  "LaneChangeEnabled": [TxType.PERSISTENT],
+  "PandaFirmware": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "PandaFirmwareHex": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "PandaDongleId": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "Passive": [TxType.PERSISTENT],
+  "RecordFront": [TxType.PERSISTENT],
+  "ReleaseNotes": [TxType.PERSISTENT],
+  "ShouldDoUpdate": [TxType.CLEAR_ON_MANAGER_START],
+  "SpeedLimitOffset": [TxType.PERSISTENT],
+  "SubscriberInfo": [TxType.PERSISTENT],
+  "TermsVersion": [TxType.PERSISTENT],
+  "TrainingVersion": [TxType.PERSISTENT],
+  "UpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
+  "UpdateFailedCount": [TxType.CLEAR_ON_MANAGER_START],
+  "Version": [TxType.PERSISTENT],
+  "Offroad_ChargeDisabled": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "Offroad_ConnectivityNeeded": [TxType.CLEAR_ON_MANAGER_START],
+  "Offroad_ConnectivityNeededPrompt": [TxType.CLEAR_ON_MANAGER_START],
+  "Offroad_TemperatureTooHigh": [TxType.CLEAR_ON_MANAGER_START],
+  "Offroad_PandaFirmwareMismatch": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  "Offroad_InvalidTime": [TxType.CLEAR_ON_MANAGER_START],
+  "Offroad_IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
+  "Offroad_NeosUpdate": [TxType.CLEAR_ON_MANAGER_START],
 }
 
+
 def fsync_dir(path):
   fd = os.open(path, os.O_RDONLY)
   try:
@@ -89,7 +119,7 @@ def fsync_dir(path):
     os.close(fd)
 
 
-class FileLock(object):
+class FileLock():
   def __init__(self, path, create):
     self._path = path
     self._create = create
@@ -105,7 +135,7 @@ class FileLock(object):
       self._fd = None
 
 
-class DBAccessor(object):
+class DBAccessor():
   def __init__(self, path):
     self._path = path
     self._vals = None
@@ -116,6 +146,10 @@ class DBAccessor(object):
 
   def get(self, key):
     self._check_entered()
+
+    if self._vals is None:
+      return None
+
     try:
       return self._vals[key]
     except KeyError:
@@ -168,7 +202,8 @@ class DBReader(DBAccessor):
     finally:
       lock.release()
 
-  def __exit__(self, type, value, traceback): pass
+  def __exit__(self, exc_type, exc_value, traceback):
+    pass
 
 
 class DBWriter(DBAccessor):
@@ -193,14 +228,14 @@ class DBWriter(DBAccessor):
       os.chmod(self._path, 0o777)
       self._lock = self._get_lock(True)
       self._vals = self._read_values_locked()
-    except:
+    except Exception:
       os.umask(self._prev_umask)
       self._prev_umask = None
       raise
 
     return self
 
-  def __exit__(self, type, value, traceback):
+  def __exit__(self, exc_type, exc_value, traceback):
     self._check_entered()
 
     try:
@@ -266,71 +301,110 @@ class DBWriter(DBAccessor):
       self._lock = None
 
 
+def read_db(params_path, key):
+  path = "%s/d/%s" % (params_path, key)
+  try:
+    with open(path, "rb") as f:
+      return f.read()
+  except IOError:
+    return None
 
-class JSDB(object):
-  def __init__(self, fn):
-    self._fn = fn
 
-  def begin(self, write=False):
+def write_db(params_path, key, value):
+  if isinstance(value, str):
+    value = value.encode('utf8')
+
+  prev_umask = os.umask(0)
+  lock = FileLock(params_path + "/.lock", True)
+  lock.acquire()
+
+  try:
+    tmp_path = tempfile.mktemp(prefix=".tmp", dir=params_path)
+    with open(tmp_path, "wb") as f:
+      f.write(value)
+      f.flush()
+      os.fsync(f.fileno())
+
+    path = "%s/d/%s" % (params_path, key)
+    os.rename(tmp_path, path)
+    fsync_dir(os.path.dirname(path))
+  finally:
+    os.umask(prev_umask)
+    lock.release()
+
+
+class Params():
+  def __init__(self, db=PARAMS):
+    self.db = db
+
+    # create the database if it doesn't exist...
+    if not os.path.exists(self.db + "/d"):
+      with self.transaction(write=True):
+        pass
+
+  def clear_all(self):
+    shutil.rmtree(self.db, ignore_errors=True)
+    with self.transaction(write=True):
+      pass
+
+  def transaction(self, write=False):
     if write:
-      return DBWriter(self._fn)
+      return DBWriter(self.db)
     else:
-      return DBReader(self._fn)
-
-class Params(object):
-  def __init__(self, db='/data/params'):
-    self.env = JSDB(db)
+      return DBReader(self.db)
 
   def _clear_keys_with_type(self, tx_type):
-    with self.env.begin(write=True) as txn:
+    with self.transaction(write=True) as txn:
       for key in keys:
-        if keys[key] == tx_type:
+        if tx_type in keys[key]:
           txn.delete(key)
 
   def manager_start(self):
     self._clear_keys_with_type(TxType.CLEAR_ON_MANAGER_START)
 
-  def car_start(self):
-    self._clear_keys_with_type(TxType.CLEAR_ON_CAR_START)
+  def panda_disconnect(self):
+    self._clear_keys_with_type(TxType.CLEAR_ON_PANDA_DISCONNECT)
 
   def delete(self, key):
-    with self.env.begin(write=True) as txn:
+    with self.transaction(write=True) as txn:
       txn.delete(key)
 
-  def get(self, key, block=False):
+  def get(self, key, block=False, encoding=None):
     if key not in keys:
       raise UnknownKeyName(key)
 
     while 1:
-      with self.env.begin() as txn:
-        ret = txn.get(key)
+      ret = read_db(self.db, key)
       if not block or ret is not None:
         break
       # is polling really the best we can do?
       time.sleep(0.05)
+
+    if ret is not None and encoding is not None:
+      ret = ret.decode(encoding)
+
     return ret
 
   def put(self, key, dat):
+    """
+    Warning: This function blocks until the param is written to disk!
+    In very rare cases this can take over a second, and your code will hang.
+
+    Use the put_nonblocking helper function in time sensitive code, but
+    in general try to avoid writing params as much as possible.
+    """
+
     if key not in keys:
       raise UnknownKeyName(key)
 
-    with self.env.begin(write=True) as txn:
-      txn.put(key, dat)
+    write_db(self.db, key, dat)
 
-if __name__ == "__main__":
-  params = Params()
-  if len(sys.argv) > 2:
-    params.put(sys.argv[1], sys.argv[2])
-  else:
-    for k in keys:
-      pp = params.get(k)
-      if pp is None:
-        print("%s is None" % k)
-      elif all(ord(c) < 128 and ord(c) >= 32 for c in pp):
-        print("%s = %s" % (k, pp))
-      else:
-        print("%s = %s" % (k, pp.encode("hex")))
 
-  # Test multiprocess:
-  # seq 0 100000 | xargs -P20 -I{} python common/params.py DongleId {} && sleep 0.05
-  # while python common/params.py DongleId; do sleep 0.05; done
+def put_nonblocking(key, val):
+  def f(key, val):
+    params = Params()
+    params.put(key, val)
+
+  t = threading.Thread(target=f, args=(key, val))
+  t.start()
+  return t

common/profiler.py

@@ -1,6 +1,6 @@
 import time
 
-class Profiler(object):
+class Profiler():
   def __init__(self, enabled=False):
     self.enabled = enabled
     self.cp = {}
@@ -39,8 +39,7 @@ class Profiler(object):
     print("******* Profiling *******")
     for n, ms in sorted(self.cp.items(), key=lambda x: -x[1]):
       if n in self.cp_ignored:
-        print("%30s: %7.2f   percent: %3.0f   IGNORED" % (n, ms*1000.0, ms/self.tot*100))
+        print("%30s: %9.2f   percent: %3.0f   IGNORED" % (n, ms*1000.0, ms/self.tot*100))
       else:
-        print("%30s: %7.2f   percent: %3.0f" % (n, ms*1000.0, ms/self.tot*100))
+        print("%30s: %9.2f   percent: %3.0f" % (n, ms*1000.0, ms/self.tot*100))
     print("Iter clock: %2.6f   TOTAL: %2.2f" % (self.tot/self.iter, self.tot))
-

common/realtime.py

@@ -2,62 +2,26 @@
 import os
 import time
 import platform
-import threading
 import subprocess
 import multiprocessing
-
 from cffi import FFI
+
+from common.android import ANDROID
+from common.common_pyx import sec_since_boot  # pylint: disable=no-name-in-module, import-error
+
+
+# time step for each process
+DT_CTRL = 0.01  # controlsd
+DT_MDL = 0.05  # model
+DT_DMON = 0.1  # driver monitoring
+DT_TRML = 0.5  # thermald and manager
+
+
 ffi = FFI()
-ffi.cdef("""
-
-typedef int clockid_t;
-struct timespec {
-    long tv_sec;   /* Seconds.  */
-    long tv_nsec;  /* Nanoseconds.  */
-};
-int clock_gettime (clockid_t clk_id, struct timespec *tp);
-
-long syscall(long number, ...);
-
-"""
-)
+ffi.cdef("long syscall(long number, ...);")
 libc = ffi.dlopen(None)
 
-
-# see <linux/time.h>
-CLOCK_MONOTONIC_RAW = 4
-CLOCK_BOOTTIME = 7
-
-if platform.system() != 'Darwin' and hasattr(libc, 'clock_gettime'):
-  c_clock_gettime = libc.clock_gettime
-
-  tlocal = threading.local()
-  def clock_gettime(clk_id):
-    if not hasattr(tlocal, 'ts'):
-      tlocal.ts = ffi.new('struct timespec *')
-
-    ts = tlocal.ts
-
-    r = c_clock_gettime(clk_id, ts)
-    if r != 0:
-      raise OSError("clock_gettime")
-    return ts.tv_sec + ts.tv_nsec * 1e-9
-else:
-  # hack. only for OS X < 10.12
-  def clock_gettime(clk_id):
-    return time.time()
-
-def monotonic_time():
-  return clock_gettime(CLOCK_MONOTONIC_RAW)
-
-def sec_since_boot():
-  return clock_gettime(CLOCK_BOOTTIME)
-
-
-def set_realtime_priority(level):
-  if os.getuid() != 0:
-    print("not setting priority, not root")
-    return
+def _get_tid():
   if platform.machine() == "x86_64":
     NR_gettid = 186
   elif platform.machine() == "aarch64":
@@ -65,11 +29,28 @@ def set_realtime_priority(level):
   else:
     raise NotImplementedError
 
-  tid = libc.syscall(NR_gettid)
-  return subprocess.call(['chrt', '-f', '-p', str(level), str(tid)])
+  return libc.syscall(NR_gettid)
 
 
-class Ratekeeper(object):
+def set_realtime_priority(level):
+  if os.getuid() != 0:
+    print("not setting priority, not root")
+    return
+
+  return subprocess.call(['chrt', '-f', '-p', str(level), str(_get_tid())])
+
+def set_core_affinity(core):
+  if os.getuid() != 0:
+    print("not setting affinity, not root")
+    return
+
+  if ANDROID:
+    return subprocess.call(['taskset', '-p', str(core), str(_get_tid())])
+  else:
+    return -1
+
+
+class Ratekeeper():
   def __init__(self, rate, print_delay_threshold=0.):
     """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
     self._interval = 1. / rate
@@ -99,10 +80,9 @@ class Ratekeeper(object):
     lagged = False
     remaining = self._next_frame_time - sec_since_boot()
     self._next_frame_time += self._interval
-    if remaining < -self._print_delay_threshold:
+    if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
       print("%s lagging by %.2f ms" % (self._process_name, -remaining * 1000))
       lagged = True
     self._frame += 1
     self._remaining = remaining
     return lagged
-

common/spinner.py

@@ -0,0 +1,66 @@
+import os
+import subprocess
+from common.basedir import BASEDIR
+
+
+class Spinner():
+  def __init__(self):
+    try:
+      self.spinner_proc = subprocess.Popen(["./spinner"],
+                                           stdin=subprocess.PIPE,
+                                           cwd=os.path.join(BASEDIR, "selfdrive", "ui", "spinner"),
+                                           close_fds=True)
+    except OSError:
+      self.spinner_proc = None
+
+  def __enter__(self):
+    return self
+
+  def update(self, spinner_text):
+    if self.spinner_proc is not None:
+      self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
+      try:
+        self.spinner_proc.stdin.flush()
+      except BrokenPipeError:
+        pass
+
+  def close(self):
+    if self.spinner_proc is not None:
+      try:
+        self.spinner_proc.stdin.close()
+      except BrokenPipeError:
+        pass
+      self.spinner_proc.terminate()
+      self.spinner_proc = None
+
+  def __del__(self):
+    self.close()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+class FakeSpinner(Spinner):
+  def __init__(self):  # pylint: disable=super-init-not-called
+    pass
+
+  def __enter__(self):
+    return self
+
+  def update(self, _):
+    pass
+
+  def close(self):
+    pass
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    pass
+
+
+if __name__ == "__main__":
+  import time
+  with Spinner() as s:
+    s.update("Spinner text")
+    time.sleep(5.0)
+  print("gone")
+  time.sleep(5.0)

common/stat_live.py

@@ -0,0 +1,73 @@
+import numpy as np
+
+class RunningStat():
+  # tracks realtime mean and standard deviation without storing any data
+  def __init__(self, priors=None, max_trackable=-1):
+    self.max_trackable = max_trackable
+    if priors is not None:
+      # initialize from history
+      self.M = priors[0]
+      self.S = priors[1]
+      self.n = priors[2]
+      self.M_last = self.M
+      self.S_last = self.S
+
+    else:
+      self.reset()
+
+  def reset(self):
+    self.M = 0.
+    self.S = 0.
+    self.M_last = 0.
+    self.S_last = 0.
+    self.n = 0
+
+  def push_data(self, new_data):
+    # short term memory hack
+    if self.max_trackable < 0 or self.n < self.max_trackable:
+      self.n += 1
+    if self.n == 0:
+      self.M_last = new_data
+      self.M = self.M_last
+      self.S_last = 0.
+    else:
+      self.M = self.M_last + (new_data - self.M_last) / self.n
+      self.S = self.S_last + (new_data - self.M_last) * (new_data - self.M)
+      self.M_last = self.M
+      self.S_last = self.S
+
+  def mean(self):
+    return self.M
+
+  def variance(self):
+    if self.n >= 2:
+      return self.S / (self.n - 1.)
+    else:
+      return 0
+
+  def std(self):
+    return np.sqrt(self.variance())
+
+  def params_to_save(self):
+    return [self.M, self.S, self.n]
+
+class RunningStatFilter():
+  def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1):
+    self.raw_stat = RunningStat(raw_priors, -1)
+    self.filtered_stat = RunningStat(filtered_priors, max_trackable)
+
+  def reset(self):
+    self.raw_stat.reset()
+    self.filtered_stat.reset()
+
+  def push_and_update(self, new_data):
+    _std_last = self.raw_stat.std()
+    self.raw_stat.push_data(new_data)
+    _delta_std = self.raw_stat.std() - _std_last
+    if _delta_std <= 0:
+      self.filtered_stat.push_data(new_data)
+    else:
+      pass
+      # self.filtered_stat.push_data(self.filtered_stat.mean())
+
+# class SequentialBayesian():

common/testing.py

@@ -1,9 +0,0 @@
-import os
-from nose.tools import nottest
-
-def phone_only(x):
-  if os.path.isfile("/init.qcom.rc"):
-    return x
-  else:
-    return nottest(x)
-

common/text_window.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import os
+import time
+import subprocess
+from common.basedir import BASEDIR
+
+
+class TextWindow():
+  def __init__(self, s):
+    try:
+      self.text_proc = subprocess.Popen(["./text", s],
+                                        stdin=subprocess.PIPE,
+                                        cwd=os.path.join(BASEDIR, "selfdrive", "ui", "text"),
+                                        close_fds=True)
+    except OSError:
+      self.text_proc = None
+
+  def get_status(self):
+    if self.text_proc is not None:
+      self.text_proc.poll()
+      return self.text_proc.returncode
+
+    return None
+
+  def __enter__(self):
+    return self
+
+  def close(self):
+    if self.text_proc is not None:
+      self.text_proc.terminate()
+      self.text_proc = None
+
+  def wait_for_exit(self):
+    while True:
+      if self.get_status() == 1:
+        return
+      time.sleep(0.1)
+
+  def __del__(self):
+    self.close()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+class FakeTextWindow(TextWindow):
+  def __init__(self, s):  # pylint: disable=super-init-not-called
+    pass
+
+  def get_status(self):
+    return 1
+
+  def wait_for_exit(self):
+    return
+
+  def __enter__(self):
+    return self
+
+  def update(self, _):
+    pass
+
+  def close(self):
+    pass
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    pass
+
+
+if __name__ == "__main__":
+  text = """Traceback (most recent call last):
+  File "./controlsd.py", line 608, in <module>
+    main()
+  File "./controlsd.py", line 604, in main
+    controlsd_thread(sm, pm, logcan)
+  File "./controlsd.py", line 455, in controlsd_thread
+    1/0
+ZeroDivisionError: division by zero"""
+  print(text)
+
+  with TextWindow(text) as s:
+    for _ in range(100):
+      if s.get_status() == 1:
+        print("Got exit button")
+        break
+      time.sleep(0.1)
+  print("gone")

common/timeout.py

@@ -0,0 +1,27 @@
+import signal
+
+class TimeoutException(Exception):
+  pass
+
+class Timeout:
+  """
+  Timeout context manager.
+  For example this code will raise a TimeoutException:
+  with Timeout(seconds=5, error_msg="Sleep was too long"):
+    time.sleep(10)
+  """
+  def __init__(self, seconds, error_msg=None):
+    if error_msg is None:
+      error_msg = 'Timed out after {} seconds'.format(seconds)
+    self.seconds = seconds
+    self.error_msg = error_msg
+
+  def handle_timeout(self, signume, frame):
+    raise TimeoutException(self.error_msg)
+
+  def __enter__(self):
+    signal.signal(signal.SIGALRM, self.handle_timeout)
+    signal.alarm(self.seconds)
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    signal.alarm(0)

common/transformations/SConscript

@@ -0,0 +1,9 @@
+Import('env')
+
+d = Dir('.')
+
+env.Command(
+  ['transformations.so'],
+  ['transformations.pxd', 'transformations.pyx',
+   'coordinates.cc', 'orientation.cc', 'coordinates.hpp', 'orientation.hpp'],
+  'cd ' + d.path + ' && python3 setup.py build_ext --inplace')

common/transformations/camera.py

@@ -0,0 +1,147 @@
+import numpy as np
+import common.transformations.orientation as orient
+
+FULL_FRAME_SIZE = (1164, 874)
+W, H = FULL_FRAME_SIZE[0], FULL_FRAME_SIZE[1]
+eon_focal_length = FOCAL = 910.0
+
+# aka 'K' aka camera_frame_from_view_frame
+eon_intrinsics = np.array([
+  [FOCAL,   0.,   W/2.],
+  [  0.,  FOCAL,  H/2.],
+  [  0.,    0.,     1.]])
+
+
+leon_dcam_intrinsics = np.array([
+  [650,   0,   816//2],
+  [  0,  650,  612//2],
+  [  0,    0,     1]])
+
+eon_dcam_intrinsics = np.array([
+  [860,   0,   1152//2],
+  [  0,  860,  864//2],
+  [  0,    0,     1]])
+
+# aka 'K_inv' aka view_frame_from_camera_frame
+eon_intrinsics_inv = np.linalg.inv(eon_intrinsics)
+
+
+# device/mesh : x->forward, y-> right, z->down
+# view : x->right, y->down, z->forward
+device_frame_from_view_frame = np.array([
+  [ 0.,  0.,  1.],
+  [ 1.,  0.,  0.],
+  [ 0.,  1.,  0.]
+])
+view_frame_from_device_frame = device_frame_from_view_frame.T
+
+
+def get_calib_from_vp(vp):
+  vp_norm = normalize(vp)
+  yaw_calib = np.arctan(vp_norm[0])
+  pitch_calib = -np.arctan(vp_norm[1]*np.cos(yaw_calib))
+  roll_calib = 0
+  return roll_calib, pitch_calib, yaw_calib
+
+
+# aka 'extrinsic_matrix'
+# road : x->forward, y -> left, z->up
+def get_view_frame_from_road_frame(roll, pitch, yaw, height):
+  device_from_road = orient.rot_from_euler([roll, pitch, yaw]).dot(np.diag([1, -1, -1]))
+  view_from_road = view_frame_from_device_frame.dot(device_from_road)
+  return np.hstack((view_from_road, [[0], [height], [0]]))
+
+
+def vp_from_ke(m):
+  """
+  Computes the vanishing point from the product of the intrinsic and extrinsic
+  matrices C = KE.
+
+  The vanishing point is defined as lim x->infinity C (x, 0, 0, 1).T
+  """
+  return (m[0, 0]/m[2, 0], m[1, 0]/m[2, 0])
+
+
+def vp_from_rpy(rpy):
+  e = get_view_frame_from_road_frame(rpy[0], rpy[1], rpy[2], 1.22)
+  ke = np.dot(eon_intrinsics, e)
+  return vp_from_ke(ke)
+
+
+def roll_from_ke(m):
+  # note: different from calibration.h/RollAnglefromKE: i think that one's just wrong
+  return np.arctan2(-(m[1, 0] - m[1, 1] * m[2, 0] / m[2, 1]),
+                    -(m[0, 0] - m[0, 1] * m[2, 0] / m[2, 1]))
+
+
+def normalize(img_pts, intrinsics=eon_intrinsics):
+  # normalizes image coordinates
+  # accepts single pt or array of pts
+  intrinsics_inv = np.linalg.inv(intrinsics)
+  img_pts = np.array(img_pts)
+  input_shape = img_pts.shape
+  img_pts = np.atleast_2d(img_pts)
+  img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1))))
+  img_pts_normalized = img_pts.dot(intrinsics_inv.T)
+  img_pts_normalized[(img_pts < 0).any(axis=1)] = np.nan
+  return img_pts_normalized[:, :2].reshape(input_shape)
+
+
+def denormalize(img_pts, intrinsics=eon_intrinsics):
+  # denormalizes image coordinates
+  # accepts single pt or array of pts
+  img_pts = np.array(img_pts)
+  input_shape = img_pts.shape
+  img_pts = np.atleast_2d(img_pts)
+  img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1))))
+  img_pts_denormalized = img_pts.dot(intrinsics.T)
+  img_pts_denormalized[img_pts_denormalized[:, 0] > W] = np.nan
+  img_pts_denormalized[img_pts_denormalized[:, 0] < 0] = np.nan
+  img_pts_denormalized[img_pts_denormalized[:, 1] > H] = np.nan
+  img_pts_denormalized[img_pts_denormalized[:, 1] < 0] = np.nan
+  return img_pts_denormalized[:, :2].reshape(input_shape)
+
+
+def device_from_ecef(pos_ecef, orientation_ecef, pt_ecef):
+  # device from ecef frame
+  # device frame is x -> forward, y-> right, z -> down
+  # accepts single pt or array of pts
+  input_shape = pt_ecef.shape
+  pt_ecef = np.atleast_2d(pt_ecef)
+  ecef_from_device_rot = orient.rotations_from_quats(orientation_ecef)
+  device_from_ecef_rot = ecef_from_device_rot.T
+  pt_ecef_rel = pt_ecef - pos_ecef
+  pt_device = np.einsum('jk,ik->ij', device_from_ecef_rot, pt_ecef_rel)
+  return pt_device.reshape(input_shape)
+
+
+def img_from_device(pt_device):
+  # img coordinates from pts in device frame
+  # first transforms to view frame, then to img coords
+  # accepts single pt or array of pts
+  input_shape = pt_device.shape
+  pt_device = np.atleast_2d(pt_device)
+  pt_view = np.einsum('jk,ik->ij', view_frame_from_device_frame, pt_device)
+
+  # This function should never return negative depths
+  pt_view[pt_view[:, 2] < 0] = np.nan
+
+  pt_img = pt_view/pt_view[:, 2:3]
+  return pt_img.reshape(input_shape)[:, :2]
+
+
+def get_camera_frame_from_calib_frame(camera_frame_from_road_frame):
+  camera_frame_from_ground = camera_frame_from_road_frame[:, (0, 1, 3)]
+  calib_frame_from_ground = np.dot(eon_intrinsics,
+                                     get_view_frame_from_road_frame(0, 0, 0, 1.22))[:, (0, 1, 3)]
+  ground_from_calib_frame = np.linalg.inv(calib_frame_from_ground)
+  camera_frame_from_calib_frame = np.dot(camera_frame_from_ground, ground_from_calib_frame)
+  return camera_frame_from_calib_frame
+
+
+def pretransform_from_calib(calib):
+  roll, pitch, yaw, height = calib
+  view_frame_from_road_frame = get_view_frame_from_road_frame(roll, pitch, yaw, height)
+  camera_frame_from_road_frame = np.dot(eon_intrinsics, view_frame_from_road_frame)
+  camera_frame_from_calib_frame = get_camera_frame_from_calib_frame(camera_frame_from_road_frame)
+  return np.linalg.inv(camera_frame_from_calib_frame)

common/transformations/coordinates.cc

@@ -0,0 +1,104 @@
+#define _USE_MATH_DEFINES
+
+#include <iostream>
+#include <cmath>
+#include <eigen3/Eigen/Dense>
+
+#include "coordinates.hpp"
+
+#define DEG2RAD(x) ((x) * M_PI / 180.0)
+#define RAD2DEG(x) ((x) * 180.0 / M_PI)
+
+
+double a = 6378137;
+double b = 6356752.3142;
+double esq = 6.69437999014 * 0.001;
+double e1sq = 6.73949674228 * 0.001;
+
+
+static Geodetic to_degrees(Geodetic geodetic){
+  geodetic.lat = RAD2DEG(geodetic.lat);
+  geodetic.lon = RAD2DEG(geodetic.lon);
+  return geodetic;
+}
+
+static Geodetic to_radians(Geodetic geodetic){
+  geodetic.lat = DEG2RAD(geodetic.lat);
+  geodetic.lon = DEG2RAD(geodetic.lon);
+  return geodetic;
+}
+
+
+ECEF geodetic2ecef(Geodetic g){
+  g = to_radians(g);
+  double xi = sqrt(1.0 - esq * pow(sin(g.lat), 2));
+  double x = (a / xi + g.alt) * cos(g.lat) * cos(g.lon);
+  double y = (a / xi + g.alt) * cos(g.lat) * sin(g.lon);
+  double z = (a / xi * (1.0 - esq) + g.alt) * sin(g.lat);
+  return {x, y, z};
+}
+
+Geodetic ecef2geodetic(ECEF e){
+  // Convert from ECEF to geodetic using Ferrari's methods
+  // https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#Ferrari.27s_solution
+  double x = e.x;
+  double y = e.y;
+  double z = e.z;
+
+  double r = sqrt(x * x + y * y);
+  double Esq = a * a - b * b;
+  double F = 54 * b * b * z * z;
+  double G = r * r + (1 - esq) * z * z - esq * Esq;
+  double C = (esq * esq * F * r * r) / (pow(G, 3));
+  double S = cbrt(1 + C + sqrt(C * C + 2 * C));
+  double P = F / (3 * pow((S + 1 / S + 1), 2) * G * G);
+  double Q = sqrt(1 + 2 * esq * esq * P);
+  double r_0 = -(P * esq * r) / (1 + Q) + sqrt(0.5 * a * a*(1 + 1.0 / Q) - P * (1 - esq) * z * z / (Q * (1 + Q)) - 0.5 * P * r * r);
+  double U = sqrt(pow((r - esq * r_0), 2) + z * z);
+  double V = sqrt(pow((r - esq * r_0), 2) + (1 - esq) * z * z);
+  double Z_0 = b * b * z / (a * V);
+  double h = U * (1 - b * b / (a * V));
+
+  double lat = atan((z + e1sq * Z_0) / r);
+  double lon = atan2(y, x);
+
+  return to_degrees({lat, lon, h});
+}
+
+LocalCoord::LocalCoord(Geodetic g, ECEF e){
+  init_ecef <<  e.x, e.y, e.z;
+
+  g = to_radians(g);
+
+  ned2ecef_matrix <<
+    -sin(g.lat)*cos(g.lon), -sin(g.lon), -cos(g.lat)*cos(g.lon),
+    -sin(g.lat)*sin(g.lon), cos(g.lon), -cos(g.lat)*sin(g.lon),
+    cos(g.lat), 0, -sin(g.lat);
+  ecef2ned_matrix = ned2ecef_matrix.transpose();
+}
+
+NED LocalCoord::ecef2ned(ECEF e) {
+  Eigen::Vector3d ecef;
+  ecef << e.x, e.y, e.z;
+
+  Eigen::Vector3d ned = (ecef2ned_matrix * (ecef - init_ecef));
+  return {ned[0], ned[1], ned[2]};
+}
+
+ECEF LocalCoord::ned2ecef(NED n) {
+  Eigen::Vector3d ned;
+  ned << n.n, n.e, n.d;
+
+  Eigen::Vector3d ecef = (ned2ecef_matrix * ned) + init_ecef;
+  return {ecef[0], ecef[1], ecef[2]};
+}
+
+NED LocalCoord::geodetic2ned(Geodetic g) {
+  ECEF e = ::geodetic2ecef(g);
+  return ecef2ned(e);
+}
+
+Geodetic LocalCoord::ned2geodetic(NED n){
+  ECEF e = ned2ecef(n);
+  return ::ecef2geodetic(e);
+}

common/transformations/coordinates.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+struct ECEF {
+  double x, y, z;
+  Eigen::Vector3d to_vector(){
+    return Eigen::Vector3d(x, y, z);
+  }
+};
+
+struct NED {
+  double n, e, d;
+};
+
+struct Geodetic {
+  double lat, lon, alt;
+  bool radians=false;
+};
+
+ECEF geodetic2ecef(Geodetic g);
+Geodetic ecef2geodetic(ECEF e);
+
+class LocalCoord {
+public:
+  Eigen::Matrix3d ned2ecef_matrix;
+  Eigen::Matrix3d ecef2ned_matrix;
+  Eigen::Vector3d init_ecef;
+  LocalCoord(Geodetic g, ECEF e);
+  LocalCoord(Geodetic g) : LocalCoord(g, ::geodetic2ecef(g)) {}
+  LocalCoord(ECEF e) : LocalCoord(::ecef2geodetic(e), e) {}
+
+  NED ecef2ned(ECEF e);
+  ECEF ned2ecef(NED n);
+  NED geodetic2ned(Geodetic g);
+  Geodetic ned2geodetic(NED n);
+};

common/transformations/coordinates.py

@@ -1,106 +1,19 @@
-import numpy as np
-"""
-Coordinate transformation module. All methods accept arrays as input
-with each row as a position.
-"""
+# pylint: skip-file
+from common.transformations.orientation import numpy_wrap
+from common.transformations.transformations import (ecef2geodetic_single,
+                                                    geodetic2ecef_single)
+from common.transformations.transformations import LocalCoord as LocalCoord_single
 
 
-
-a = 6378137
-b = 6356752.3142
-esq = 6.69437999014 * 0.001
-e1sq = 6.73949674228 * 0.001
+class LocalCoord(LocalCoord_single):
+  ecef2ned = numpy_wrap(LocalCoord_single.ecef2ned_single, (3,), (3,))
+  ned2ecef = numpy_wrap(LocalCoord_single.ned2ecef_single, (3,), (3,))
+  geodetic2ned = numpy_wrap(LocalCoord_single.geodetic2ned_single, (3,), (3,))
+  ned2geodetic = numpy_wrap(LocalCoord_single.ned2geodetic_single, (3,), (3,))
 
 
-def geodetic2ecef(geodetic):
-  geodetic = np.array(geodetic)
-  input_shape = geodetic.shape
-  geodetic = np.atleast_2d(geodetic)
-  lat = (np.pi/180)*geodetic[:,0]
-  lon = (np.pi/180)*geodetic[:,1]
-  alt = geodetic[:,2]
+geodetic2ecef = numpy_wrap(geodetic2ecef_single, (3,), (3,))
+ecef2geodetic = numpy_wrap(ecef2geodetic_single, (3,), (3,))
 
-  xi = np.sqrt(1 - esq * np.sin(lat)**2)
-  x = (a / xi + alt) * np.cos(lat) * np.cos(lon)
-  y = (a / xi + alt) * np.cos(lat) * np.sin(lon)
-  z = (a / xi * (1 - esq) + alt) * np.sin(lat)
-  ecef = np.array([x, y, z]).T
-  return ecef.reshape(input_shape)
-
-
-def ecef2geodetic(ecef):
-  """
-  Convert ECEF coordinates to geodetic using ferrari's method
-  """
-  def ferrari(x, y, z):
-    # ferrari's method
-    r = np.sqrt(x * x + y * y)
-    Esq = a * a - b * b
-    F = 54 * b * b * z * z
-    G = r * r + (1 - esq) * z * z - esq * Esq
-    C = (esq * esq * F * r * r) / (pow(G, 3))
-    S = np.cbrt(1 + C + np.sqrt(C * C + 2 * C))
-    P = F / (3 * pow((S + 1 / S + 1), 2) * G * G)
-    Q = np.sqrt(1 + 2 * esq * esq * P)
-    r_0 =  -(P * esq * r) / (1 + Q) + np.sqrt(0.5 * a * a*(1 + 1.0 / Q) - \
-          P * (1 - esq) * z * z / (Q * (1 + Q)) - 0.5 * P * r * r)
-    U = np.sqrt(pow((r - esq * r_0), 2) + z * z)
-    V = np.sqrt(pow((r - esq * r_0), 2) + (1 - esq) * z * z)
-    Z_0 = b * b * z / (a * V)
-    h = U * (1 - b * b / (a * V))
-    lat = (180/np.pi)*np.arctan((z + e1sq * Z_0) / r)
-    lon = (180/np.pi)*np.arctan2(y, x)
-    return lat, lon, h
-
-  geodetic = []
-  ecef = np.array(ecef)
-  input_shape = ecef.shape
-  ecef = np.atleast_2d(ecef)
-  for p in ecef:
-    geodetic.append(ferrari(*p))
-  geodetic = np.array(geodetic)
-  return geodetic.reshape(input_shape)
-
-
-
-class LocalCoord(object):
-  """
-   Allows conversions to local frames. In this case NED.
-   That is: North East Down from the start position in
-   meters.
-  """
-  def __init__(self, init_geodetic, init_ecef):
-    self.init_ecef = init_ecef
-    lat, lon, _ = (np.pi/180)*np.array(init_geodetic)
-    self.ned2ecef_matrix = np.array([[-np.sin(lat)*np.cos(lon), -np.sin(lon), -np.cos(lat)*np.cos(lon)],
-                                     [-np.sin(lat)*np.sin(lon), np.cos(lon), -np.cos(lat)*np.sin(lon)],
-                                     [np.cos(lat), 0, -np.sin(lat)]])
-    self.ecef2ned_matrix = self.ned2ecef_matrix.T
-
-  @classmethod
-  def from_geodetic(cls, init_geodetic):
-    init_ecef = geodetic2ecef(init_geodetic)
-    return LocalCoord(init_geodetic, init_ecef)
-
-  @classmethod
-  def from_ecef(cls, init_ecef):
-    init_geodetic = ecef2geodetic(init_ecef)
-    return LocalCoord(init_geodetic, init_ecef)
-
-
-  def ecef2ned(self, ecef):
-    ecef = np.array(ecef)
-    return np.dot(self.ecef2ned_matrix, (ecef - self.init_ecef).T).T
-
-  def ned2ecef(self, ned):
-    ned = np.array(ned)
-    # Transpose so that init_ecef will broadcast correctly for 1d or 2d ned.
-    return (np.dot(self.ned2ecef_matrix, ned.T).T + self.init_ecef)
-
-  def geodetic2ned(self, geodetic):
-    ecef = geodetic2ecef(geodetic)
-    return self.ecef2ned(ecef)
-
-  def ned2geodetic(self, ned):
-    ecef = self.ned2ecef(ned)
-    return ecef2geodetic(ecef)
+geodetic_from_ecef = ecef2geodetic
+ecef_from_geodetic = geodetic2ecef

common/transformations/model.py

@@ -0,0 +1,153 @@
+import numpy as np
+
+from common.transformations.camera import (FULL_FRAME_SIZE, eon_focal_length,
+                                           get_view_frame_from_road_frame,
+                                           vp_from_ke)
+
+# segnet
+
+SEGNET_SIZE = (512, 384)
+
+segnet_frame_from_camera_frame = np.array([
+  [float(SEGNET_SIZE[0])/FULL_FRAME_SIZE[0],    0.,          ],
+  [     0.,          float(SEGNET_SIZE[1])/FULL_FRAME_SIZE[1]]])
+
+
+# model
+
+MODEL_INPUT_SIZE = (320, 160)
+MODEL_YUV_SIZE = (MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1] * 3 // 2)
+MODEL_CX = MODEL_INPUT_SIZE[0]/2.
+MODEL_CY = 21.
+
+model_zoom = 1.25
+model_height = 1.22
+
+# canonical model transform
+model_intrinsics = np.array(
+  [[ eon_focal_length / model_zoom,    0. ,  MODEL_CX],
+   [   0. ,  eon_focal_length / model_zoom,  MODEL_CY],
+   [   0. ,                            0. ,   1.]])
+
+
+# MED model
+MEDMODEL_INPUT_SIZE = (512, 256)
+MEDMODEL_YUV_SIZE = (MEDMODEL_INPUT_SIZE[0], MEDMODEL_INPUT_SIZE[1] * 3 // 2)
+MEDMODEL_CY = 47.6
+
+medmodel_zoom = 1.
+medmodel_intrinsics = np.array(
+  [[ eon_focal_length / medmodel_zoom,    0. ,  0.5 * MEDMODEL_INPUT_SIZE[0]],
+   [   0. ,  eon_focal_length / medmodel_zoom,  MEDMODEL_CY],
+   [   0. ,                            0. ,   1.]])
+
+# CAL model
+CALMODEL_INPUT_SIZE = (512, 256)
+CALMODEL_YUV_SIZE = (CALMODEL_INPUT_SIZE[0], CALMODEL_INPUT_SIZE[1] * 3 // 2)
+CALMODEL_CY = 47.6
+
+calmodel_zoom = 1.5
+calmodel_intrinsics = np.array(
+  [[ eon_focal_length / calmodel_zoom,    0. ,  0.5 * CALMODEL_INPUT_SIZE[0]],
+   [   0. ,  eon_focal_length / calmodel_zoom,  CALMODEL_CY],
+   [   0. ,                            0. ,   1.]])
+
+
+# BIG model
+
+BIGMODEL_INPUT_SIZE = (1024, 512)
+BIGMODEL_YUV_SIZE = (BIGMODEL_INPUT_SIZE[0], BIGMODEL_INPUT_SIZE[1] * 3 // 2)
+
+bigmodel_zoom = 1.
+bigmodel_intrinsics = np.array(
+  [[ eon_focal_length / bigmodel_zoom,    0. , 0.5 * BIGMODEL_INPUT_SIZE[0]],
+   [   0. ,  eon_focal_length / bigmodel_zoom,  256+MEDMODEL_CY],
+   [   0. ,                            0. ,   1.]])
+
+model_frame_from_road_frame = np.dot(model_intrinsics,
+  get_view_frame_from_road_frame(0, 0, 0, model_height))
+
+bigmodel_frame_from_road_frame = np.dot(bigmodel_intrinsics,
+  get_view_frame_from_road_frame(0, 0, 0, model_height))
+
+medmodel_frame_from_road_frame = np.dot(medmodel_intrinsics,
+  get_view_frame_from_road_frame(0, 0, 0, model_height))
+
+model_frame_from_bigmodel_frame = np.dot(model_intrinsics, np.linalg.inv(bigmodel_intrinsics))
+medmodel_frame_from_bigmodel_frame = np.dot(medmodel_intrinsics, np.linalg.inv(bigmodel_intrinsics))
+
+# 'camera from model camera'
+def get_model_height_transform(camera_frame_from_road_frame, height):
+  camera_frame_from_road_ground = np.dot(camera_frame_from_road_frame, np.array([
+      [1, 0, 0],
+      [0, 1, 0],
+      [0, 0, 0],
+      [0, 0, 1],
+  ]))
+
+  camera_frame_from_road_high = np.dot(camera_frame_from_road_frame, np.array([
+      [1, 0, 0],
+      [0, 1, 0],
+      [0, 0, height - model_height],
+      [0, 0, 1],
+  ]))
+
+  road_high_from_camera_frame = np.linalg.inv(camera_frame_from_road_high)
+  high_camera_from_low_camera = np.dot(camera_frame_from_road_ground, road_high_from_camera_frame)
+
+  return high_camera_from_low_camera
+
+
+# camera_frame_from_model_frame aka 'warp matrix'
+# was: calibration.h/CalibrationTransform
+def get_camera_frame_from_model_frame(camera_frame_from_road_frame, height=model_height):
+  vp = vp_from_ke(camera_frame_from_road_frame)
+
+  model_camera_from_model_frame = np.array([
+    [model_zoom,         0., vp[0] - MODEL_CX * model_zoom],
+    [        0., model_zoom, vp[1] - MODEL_CY * model_zoom],
+    [        0.,         0.,                            1.],
+  ])
+
+  # This function is super slow, so skip it if height is very close to canonical
+  # TODO: speed it up!
+  if abs(height - model_height) > 0.001:
+    camera_from_model_camera = get_model_height_transform(camera_frame_from_road_frame, height)
+  else:
+    camera_from_model_camera = np.eye(3)
+
+  return np.dot(camera_from_model_camera, model_camera_from_model_frame)
+
+
+def get_camera_frame_from_medmodel_frame(camera_frame_from_road_frame):
+  camera_frame_from_ground = camera_frame_from_road_frame[:, (0, 1, 3)]
+  medmodel_frame_from_ground = medmodel_frame_from_road_frame[:, (0, 1, 3)]
+
+  ground_from_medmodel_frame = np.linalg.inv(medmodel_frame_from_ground)
+  camera_frame_from_medmodel_frame = np.dot(camera_frame_from_ground, ground_from_medmodel_frame)
+
+  return camera_frame_from_medmodel_frame
+
+
+def get_camera_frame_from_bigmodel_frame(camera_frame_from_road_frame):
+  camera_frame_from_ground = camera_frame_from_road_frame[:, (0, 1, 3)]
+  bigmodel_frame_from_ground = bigmodel_frame_from_road_frame[:, (0, 1, 3)]
+
+  ground_from_bigmodel_frame = np.linalg.inv(bigmodel_frame_from_ground)
+  camera_frame_from_bigmodel_frame = np.dot(camera_frame_from_ground, ground_from_bigmodel_frame)
+
+  return camera_frame_from_bigmodel_frame
+
+
+def get_model_frame(snu_full, camera_frame_from_model_frame, size):
+  idxs = camera_frame_from_model_frame.dot(np.column_stack([np.tile(np.arange(size[0]), size[1]),
+                                                            np.tile(np.arange(size[1]), (size[0], 1)).T.flatten(),
+                                                            np.ones(size[0] * size[1])]).T).T.astype(int)
+  calib_flat = snu_full[idxs[:, 1], idxs[:, 0]]
+  if len(snu_full.shape) == 3:
+    calib = calib_flat.reshape((size[1], size[0], 3))
+  elif len(snu_full.shape) == 2:
+    calib = calib_flat.reshape((size[1], size[0]))
+  else:
+    raise ValueError("shape of input img is weird")
+  return calib

common/transformations/orientation.cc

@@ -0,0 +1,147 @@
+#define _USE_MATH_DEFINES
+
+#include <iostream>
+#include <cmath>
+#include <eigen3/Eigen/Dense>
+
+#include "orientation.hpp"
+#include "coordinates.hpp"
+
+Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat){
+  if (quat.w() > 0){
+    return quat;
+  } else {
+    return Eigen::Quaterniond(-quat.w(), -quat.x(), -quat.y(), -quat.z());
+  }
+}
+
+Eigen::Quaterniond euler2quat(Eigen::Vector3d euler){
+  Eigen::Quaterniond q;
+
+  q = Eigen::AngleAxisd(euler(2), Eigen::Vector3d::UnitZ())
+    * Eigen::AngleAxisd(euler(1), Eigen::Vector3d::UnitY())
+    * Eigen::AngleAxisd(euler(0), Eigen::Vector3d::UnitX());
+  return ensure_unique(q);
+}
+
+
+Eigen::Vector3d quat2euler(Eigen::Quaterniond quat){
+  // TODO: switch to eigen implementation if the range of the Euler angles doesn't matter anymore
+  // Eigen::Vector3d euler = quat.toRotationMatrix().eulerAngles(2, 1, 0);
+  // return {euler(2), euler(1), euler(0)};
+  double gamma = atan2(2 * (quat.w() * quat.x() + quat.y() * quat.z()), 1 - 2 * (quat.x()*quat.x() + quat.y()*quat.y()));
+  double theta = asin(2 * (quat.w() * quat.y() - quat.z() * quat.x()));
+  double psi = atan2(2 * (quat.w() * quat.z() + quat.x() * quat.y()), 1 - 2 * (quat.y()*quat.y() + quat.z()*quat.z()));
+  return {gamma, theta, psi};
+}
+
+Eigen::Matrix3d quat2rot(Eigen::Quaterniond quat){
+  return quat.toRotationMatrix();
+}
+
+Eigen::Quaterniond rot2quat(Eigen::Matrix3d rot){
+  return ensure_unique(Eigen::Quaterniond(rot));
+}
+
+Eigen::Matrix3d euler2rot(Eigen::Vector3d euler){
+  return quat2rot(euler2quat(euler));
+}
+
+Eigen::Vector3d rot2euler(Eigen::Matrix3d rot){
+  return quat2euler(rot2quat(rot));
+}
+
+Eigen::Matrix3d rot_matrix(double roll, double pitch, double yaw){
+  return euler2rot({roll, pitch, yaw});
+}
+
+Eigen::Matrix3d rot(Eigen::Vector3d axis, double angle){
+  Eigen::Quaterniond q;
+  q = Eigen::AngleAxisd(angle, axis);
+  return q.toRotationMatrix();
+}
+
+
+Eigen::Vector3d ecef_euler_from_ned(ECEF ecef_init, Eigen::Vector3d ned_pose) {
+  /*
+    Using Rotations to Build Aerospace Coordinate Systems
+    Don Koks
+    https://apps.dtic.mil/dtic/tr/fulltext/u2/a484864.pdf
+  */
+  LocalCoord converter = LocalCoord(ecef_init);
+  Eigen::Vector3d zero = ecef_init.to_vector();
+
+  Eigen::Vector3d x0 = converter.ned2ecef({1, 0, 0}).to_vector() - zero;
+  Eigen::Vector3d y0 = converter.ned2ecef({0, 1, 0}).to_vector() - zero;
+  Eigen::Vector3d z0 = converter.ned2ecef({0, 0, 1}).to_vector() - zero;
+
+  Eigen::Vector3d x1 = rot(z0, ned_pose(2)) * x0;
+  Eigen::Vector3d y1 = rot(z0, ned_pose(2)) * y0;
+  Eigen::Vector3d z1 = rot(z0, ned_pose(2)) * z0;
+
+  Eigen::Vector3d x2 = rot(y1, ned_pose(1)) * x1;
+  Eigen::Vector3d y2 = rot(y1, ned_pose(1)) * y1;
+  Eigen::Vector3d z2 = rot(y1, ned_pose(1)) * z1;
+
+  Eigen::Vector3d x3 = rot(x2, ned_pose(0)) * x2;
+  Eigen::Vector3d y3 = rot(x2, ned_pose(0)) * y2;
+
+
+  x0 = Eigen::Vector3d(1, 0, 0);
+  y0 = Eigen::Vector3d(0, 1, 0);
+  z0 = Eigen::Vector3d(0, 0, 1);
+
+  double psi = atan2(x3.dot(y0), x3.dot(x0));
+  double theta = atan2(-x3.dot(z0), sqrt(pow(x3.dot(x0), 2) + pow(x3.dot(y0), 2)));
+
+  y2 = rot(z0, psi) * y0;
+  z2 = rot(y2, theta) * z0;
+
+  double phi = atan2(y3.dot(z2), y3.dot(y2));
+
+  return {phi, theta, psi};
+}
+
+Eigen::Vector3d ned_euler_from_ecef(ECEF ecef_init, Eigen::Vector3d ecef_pose){
+  /*
+    Using Rotations to Build Aerospace Coordinate Systems
+    Don Koks
+    https://apps.dtic.mil/dtic/tr/fulltext/u2/a484864.pdf
+  */
+  LocalCoord converter = LocalCoord(ecef_init);
+
+  Eigen::Vector3d x0 = Eigen::Vector3d(1, 0, 0);
+  Eigen::Vector3d y0 = Eigen::Vector3d(0, 1, 0);
+  Eigen::Vector3d z0 = Eigen::Vector3d(0, 0, 1);
+
+  Eigen::Vector3d x1 = rot(z0, ecef_pose(2)) * x0;
+  Eigen::Vector3d y1 = rot(z0, ecef_pose(2)) * y0;
+  Eigen::Vector3d z1 = rot(z0, ecef_pose(2)) * z0;
+
+  Eigen::Vector3d x2 = rot(y1, ecef_pose(1)) * x1;
+  Eigen::Vector3d y2 = rot(y1, ecef_pose(1)) * y1;
+  Eigen::Vector3d z2 = rot(y1, ecef_pose(1)) * z1;
+
+  Eigen::Vector3d x3 = rot(x2, ecef_pose(0)) * x2;
+  Eigen::Vector3d y3 = rot(x2, ecef_pose(0)) * y2;
+
+  Eigen::Vector3d zero = ecef_init.to_vector();
+  x0 = converter.ned2ecef({1, 0, 0}).to_vector() - zero;
+  y0 = converter.ned2ecef({0, 1, 0}).to_vector() - zero;
+  z0 = converter.ned2ecef({0, 0, 1}).to_vector() - zero;
+
+  double psi = atan2(x3.dot(y0), x3.dot(x0));
+  double theta = atan2(-x3.dot(z0), sqrt(pow(x3.dot(x0), 2) + pow(x3.dot(y0), 2)));
+
+  y2 = rot(z0, psi) * y0;
+  z2 = rot(y2, theta) * z0;
+
+  double phi = atan2(y3.dot(z2), y3.dot(y2));
+
+  return {phi, theta, psi};
+}
+
+
+
+int main(void){
+}

common/transformations/orientation.hpp

@@ -0,0 +1,17 @@
+#pragma once
+#include <eigen3/Eigen/Dense>
+#include "coordinates.hpp"
+
+
+Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat);
+
+Eigen::Quaterniond euler2quat(Eigen::Vector3d euler);
+Eigen::Vector3d quat2euler(Eigen::Quaterniond quat);
+Eigen::Matrix3d quat2rot(Eigen::Quaterniond quat);
+Eigen::Quaterniond rot2quat(Eigen::Matrix3d rot);
+Eigen::Matrix3d euler2rot(Eigen::Vector3d euler);
+Eigen::Vector3d rot2euler(Eigen::Matrix3d rot);
+Eigen::Matrix3d rot_matrix(double roll, double pitch, double yaw);
+Eigen::Matrix3d rot(Eigen::Vector3d axis, double angle);
+Eigen::Vector3d ecef_euler_from_ned(ECEF ecef_init, Eigen::Vector3d ned_pose);
+Eigen::Vector3d ned_euler_from_ecef(ECEF ecef_init, Eigen::Vector3d ecef_pose);

common/transformations/orientation.py

@@ -0,0 +1,53 @@
+# pylint: skip-file
+import numpy as np
+
+from common.transformations.transformations import (ecef_euler_from_ned_single,
+                                                    euler2quat_single,
+                                                    euler2rot_single,
+                                                    ned_euler_from_ecef_single,
+                                                    quat2euler_single,
+                                                    quat2rot_single,
+                                                    rot2euler_single,
+                                                    rot2quat_single)
+
+
+def numpy_wrap(function, input_shape, output_shape):
+  """Wrap a function to take either an input or list of inputs and return the correct shape"""
+  def f(*inps):
+    *args, inp = inps
+    inp = np.array(inp)
+    shape = inp.shape
+
+    if len(shape) == len(input_shape):
+      out_shape = output_shape
+    else:
+      out_shape = (shape[0],) + output_shape
+
+    # Add empty dimension if inputs is not a list
+    if len(shape) == len(input_shape):
+      inp.shape = (1, ) + inp.shape
+
+    result = np.asarray([function(*args, i) for i in inp])
+    result.shape = out_shape
+    return result
+  return f
+
+
+euler2quat = numpy_wrap(euler2quat_single, (3,), (4,))
+quat2euler = numpy_wrap(quat2euler_single, (4,), (3,))
+quat2rot = numpy_wrap(quat2rot_single, (4,), (3, 3))
+rot2quat = numpy_wrap(rot2quat_single, (3, 3), (4,))
+euler2rot = numpy_wrap(euler2rot_single, (3,), (3, 3))
+rot2euler = numpy_wrap(rot2euler_single, (3, 3), (3,))
+ecef_euler_from_ned = numpy_wrap(ecef_euler_from_ned_single, (3,), (3,))
+ned_euler_from_ecef = numpy_wrap(ned_euler_from_ecef_single, (3,), (3,))
+
+quats_from_rotations = rot2quat
+quat_from_rot = rot2quat
+rotations_from_quats = quat2rot
+rot_from_quat = quat2rot
+rot_from_quat = quat2rot
+euler_from_rot = rot2euler
+euler_from_quat = quat2euler
+rot_from_euler = euler2rot
+quat_from_euler = euler2quat

common/transformations/setup.py

@@ -0,0 +1,42 @@
+import os
+import numpy
+import sysconfig
+
+from Cython.Build import cythonize
+from Cython.Distutils import build_ext
+from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
+
+def get_ext_filename_without_platform_suffix(filename):
+  name, ext = os.path.splitext(filename)
+  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+  if ext_suffix == ext:
+    return filename
+
+  ext_suffix = ext_suffix.replace(ext, '')
+  idx = name.find(ext_suffix)
+
+  if idx == -1:
+    return filename
+  else:
+    return name[:idx] + ext
+
+
+class BuildExtWithoutPlatformSuffix(build_ext):
+  def get_ext_filename(self, ext_name):
+    filename = super().get_ext_filename(ext_name)
+    return get_ext_filename_without_platform_suffix(filename)
+
+
+setup(
+  name='Cython transformations wrapper',
+  cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
+  ext_modules=cythonize(
+  Extension(
+    "transformations",
+    sources=["transformations.pyx"],
+    language="c++",
+    extra_compile_args=["-std=c++14"],
+    include_dirs=[numpy.get_include()],
+  )
+))

common/transformations/transformations.pxd

@@ -0,0 +1,71 @@
+from libcpp cimport bool
+
+cdef extern from "orientation.cc":
+  pass
+
+cdef extern from "orientation.hpp":
+  cdef cppclass Quaternion "Eigen::Quaterniond":
+    Quaternion()
+    Quaternion(double, double, double, double)
+    double w()
+    double x()
+    double y()
+    double z()
+
+  cdef cppclass Vector3 "Eigen::Vector3d":
+    Vector3()
+    Vector3(double, double, double)
+    double operator()(int)
+
+  cdef cppclass Matrix3 "Eigen::Matrix3d":
+    Matrix3()
+    Matrix3(double*)
+
+    double operator()(int, int)
+
+  Quaternion euler2quat(Vector3)
+  Vector3 quat2euler(Quaternion)
+  Matrix3 quat2rot(Quaternion)
+  Quaternion rot2quat(Matrix3)
+  Vector3 rot2euler(Matrix3)
+  Matrix3 euler2rot(Vector3)
+  Matrix3 rot_matrix(double, double, double)
+  Vector3 ecef_euler_from_ned(ECEF, Vector3)
+  Vector3 ned_euler_from_ecef(ECEF, Vector3)
+
+
+cdef extern from "coordinates.cc":
+  cdef struct ECEF:
+    double x
+    double y
+    double z
+
+  cdef struct NED:
+    double n
+    double e
+    double d
+
+  cdef struct Geodetic:
+    double lat
+    double lon
+    double alt
+    bool radians
+
+  ECEF geodetic2ecef(Geodetic)
+  Geodetic ecef2geodetic(ECEF)
+
+  cdef cppclass LocalCoord_c "LocalCoord":
+    Matrix3 ned2ecef_matrix
+    Matrix3 ecef2ned_matrix
+
+    LocalCoord_c(Geodetic, ECEF)
+    LocalCoord_c(Geodetic)
+    LocalCoord_c(ECEF)
+
+    NED ecef2ned(ECEF)
+    ECEF ned2ecef(NED)
+    NED geodetic2ned(Geodetic)
+    Geodetic ned2geodetic(NED)
+
+cdef extern from "coordinates.hpp":
+  pass

common/transformations/transformations.pyx

@@ -0,0 +1,172 @@
+from transformations cimport Matrix3, Vector3, Quaternion
+from transformations cimport ECEF, NED, Geodetic
+
+from transformations cimport euler2quat as euler2quat_c
+from transformations cimport quat2euler as quat2euler_c
+from transformations cimport quat2rot as quat2rot_c
+from transformations cimport rot2quat as rot2quat_c
+from transformations cimport euler2rot as euler2rot_c
+from transformations cimport rot2euler as rot2euler_c
+from transformations cimport rot_matrix as rot_matrix_c
+from transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from transformations cimport geodetic2ecef as geodetic2ecef_c
+from transformations cimport ecef2geodetic as ecef2geodetic_c
+from transformations cimport LocalCoord_c
+
+
+import cython
+import numpy as np
+cimport numpy as np
+
+cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m):
+    return np.array([
+        [m(0, 0), m(0, 1), m(0, 2)],
+        [m(1, 0), m(1, 1), m(1, 2)],
+        [m(2, 0), m(2, 1), m(2, 2)],
+    ])
+
+cdef Matrix3 numpy2matrix (np.ndarray[double, ndim=2, mode="fortran"] m):
+    assert m.shape[0] == 3
+    assert m.shape[1] == 3
+    return Matrix3(<double*>m.data)
+
+cdef ECEF list2ecef(ecef):
+    cdef ECEF e;
+    e.x = ecef[0]
+    e.y = ecef[1]
+    e.z = ecef[2]
+    return e
+
+cdef NED list2ned(ned):
+    cdef NED n;
+    n.n = ned[0]
+    n.e = ned[1]
+    n.d = ned[2]
+    return n
+
+cdef Geodetic list2geodetic(geodetic):
+    cdef Geodetic g
+    g.lat = geodetic[0]
+    g.lon = geodetic[1]
+    g.alt = geodetic[2]
+    return g
+
+def euler2quat_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Quaternion q = euler2quat_c(e)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def quat2euler_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Vector3 e = quat2euler_c(q);
+    return [e(0), e(1), e(2)]
+
+def quat2rot_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Matrix3 r = quat2rot_c(q)
+    return matrix2numpy(r)
+
+def rot2quat_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Quaternion q = rot2quat_c(r)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def euler2rot_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Matrix3 r = euler2rot_c(e)
+    return matrix2numpy(r)
+
+def rot2euler_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Vector3 e = rot2euler_c(r)
+    return [e(0), e(1), e(2)]
+
+def rot_matrix(roll, pitch, yaw):
+    return matrix2numpy(rot_matrix_c(roll, pitch, yaw))
+
+def ecef_euler_from_ned_single(ecef_init, ned_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ned_pose[0], ned_pose[1], ned_pose[2])
+
+    cdef Vector3 e = ecef_euler_from_ned_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def ned_euler_from_ecef_single(ecef_init, ecef_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ecef_pose[0], ecef_pose[1], ecef_pose[2])
+
+    cdef Vector3 e = ned_euler_from_ecef_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def geodetic2ecef_single(geodetic):
+    cdef Geodetic g = list2geodetic(geodetic)
+    cdef ECEF e = geodetic2ecef_c(g)
+    return [e.x, e.y, e.z]
+
+def ecef2geodetic_single(ecef):
+    cdef ECEF e = list2ecef(ecef)
+    cdef Geodetic g = ecef2geodetic_c(e)
+    return [g.lat, g.lon, g.alt]
+
+
+cdef class LocalCoord:
+    cdef LocalCoord_c * lc
+
+    def __init__(self, geodetic=None, ecef=None):
+        assert (geodetic is not None) or (ecef is not None)
+        if geodetic is not None:
+            self.lc = new LocalCoord_c(list2geodetic(geodetic))
+        elif ecef is not None:
+            self.lc = new LocalCoord_c(list2ecef(ecef))
+
+    @property
+    def ned2ecef_matrix(self):
+        return matrix2numpy(self.lc.ned2ecef_matrix)
+
+    @property
+    def ecef2ned_matrix(self):
+        return matrix2numpy(self.lc.ecef2ned_matrix)
+
+    @property
+    def ned_from_ecef_matrix(self):
+        return self.ecef2ned_matrix
+
+    @property
+    def ecef_from_ned_matrix(self):
+        return self.ned2ecef_matrix
+
+    @classmethod
+    def from_geodetic(cls, geodetic):
+        return cls(geodetic=geodetic)
+
+    @classmethod
+    def from_ecef(cls, ecef):
+        return cls(ecef=ecef)
+
+    def ecef2ned_single(self, ecef):
+        assert self.lc
+        cdef ECEF e = list2ecef(ecef)
+        cdef NED n = self.lc.ecef2ned(e)
+        return [n.n, n.e, n.d]
+
+    def ned2ecef_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef ECEF e = self.lc.ned2ecef(n)
+        return [e.x, e.y, e.z]
+
+    def geodetic2ned_single(self, geodetic):
+        assert self.lc
+        cdef Geodetic g = list2geodetic(geodetic)
+        cdef NED n = self.lc.geodetic2ned(g)
+        return [n.n, n.e, n.d]
+
+    def ned2geodetic_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef Geodetic g = self.lc.ned2geodetic(n)
+        return [g.lat, g.lon, g.alt]
+
+    def __dealloc__(self):
+        del self.lc

common/xattr.py

@@ -0,0 +1,45 @@
+import os
+from cffi import FFI
+
+# Workaround for the EON/termux build of Python having os.*xattr removed.
+ffi = FFI()
+ffi.cdef("""
+int setxattr(const char *path, const char *name, const void *value, size_t size, int flags);
+ssize_t getxattr(const char *path, const char *name, void *value, size_t size);
+ssize_t listxattr(const char *path, char *list, size_t size);
+int removexattr(const char *path, const char *name);
+""")
+libc = ffi.dlopen(None)
+
+def setxattr(path, name, value, flags=0):
+  path = path.encode()
+  name = name.encode()
+  if libc.setxattr(path, name, value, len(value), flags) == -1:
+    raise OSError(ffi.errno, f"{os.strerror(ffi.errno)}: setxattr({path}, {name}, {value}, {flags})")
+
+def getxattr(path, name, size=128):
+  path = path.encode()
+  name = name.encode()
+  value = ffi.new(f"char[{size}]")
+  l = libc.getxattr(path, name, value, size)
+  if l == -1:
+    # errno 61 means attribute hasn't been set
+    if ffi.errno == 61:
+      return None
+    raise OSError(ffi.errno, f"{os.strerror(ffi.errno)}: getxattr({path}, {name}, {size})")
+  return ffi.buffer(value)[:l]
+
+def listxattr(path, size=128):
+  path = path.encode()
+  attrs = ffi.new(f"char[{size}]")
+  l = libc.listxattr(path, attrs, size)
+  if l == -1:
+    raise OSError(ffi.errno, f"{os.strerror(ffi.errno)}: listxattr({path}, {size})")
+  # attrs is b'\0' delimited values (so chop off trailing empty item)
+  return [a.decode() for a in ffi.buffer(attrs)[:l].split(b"\0")[0:-1]]
+
+def removexattr(path, name):
+  path = path.encode()
+  name = name.encode()
+  if libc.removexattr(path, name) == -1:
+    raise OSError(ffi.errno, f"{os.strerror(ffi.errno)}: removexattr({path}, {name})")

installer/updater/Makefile

@@ -0,0 +1,98 @@
+CC = clang
+CXX = clang++
+
+PHONELIBS = ../../phonelibs
+
+WARN_FLAGS = -Werror=implicit-function-declaration \
+             -Werror=incompatible-pointer-types \
+             -Werror=int-conversion \
+             -Werror=return-type \
+             -Werror=format-extra-args
+
+CFLAGS = -std=gnu11 -g -fPIC -O2 $(WARN_FLAGS)
+CXXFLAGS = -std=c++11 -g -fPIC -O2 $(WARN_FLAGS)
+
+CURL_FLAGS = -I$(PHONELIBS)/curl/include
+CURL_LIBS = $(PHONELIBS)/curl/lib/libcurl.a \
+            $(PHONELIBS)/zlib/lib/libz.a
+
+BORINGSSL_FLAGS = -I$(PHONELIBS)/boringssl/include
+BORINGSSL_LIBS = $(PHONELIBS)/boringssl/lib/libssl_static.a \
+                 $(PHONELIBS)/boringssl/lib/libcrypto_static.a \
+
+NANOVG_FLAGS = -I$(PHONELIBS)/nanovg
+
+JSON11_FLAGS = -I$(PHONELIBS)/json11
+
+OPENGL_LIBS = -lGLESv3
+
+FRAMEBUFFER_LIBS = -lutils -lgui -lEGL
+
+.PHONY: all
+all: updater
+
+OBJS = opensans_regular.ttf.o \
+			 opensans_semibold.ttf.o \
+			 opensans_bold.ttf.o \
+       ../../selfdrive/common/touch.o \
+       ../../selfdrive/common/framebuffer.o \
+       $(PHONELIBS)/json11/json11.o \
+       $(PHONELIBS)/nanovg/nanovg.o
+
+DEPS := $(OBJS:.o=.d)
+
+updater: updater.o $(OBJS)
+	@echo "[ LINK ] $@"
+	$(CXX) $(CPPFLAGS) -fPIC -o 'updater' $^ \
+    $(FRAMEBUFFER_LIBS) \
+    $(CURL_LIBS) \
+    $(BORINGSSL_LIBS) \
+    -L/system/vendor/lib64 \
+    $(OPENGL_LIBS) \
+    -lcutils -lm -llog
+	strip updater
+
+opensans_regular.ttf.o: ../../selfdrive/assets/fonts/opensans_regular.ttf
+	@echo "[ bin2o ] $@"
+	cd '$(dir $<)' && ld -r -b binary '$(notdir $<)' -o '$(abspath $@)'
+
+opensans_bold.ttf.o: ../../selfdrive/assets/fonts/opensans_bold.ttf
+	@echo "[ bin2o ] $@"
+	cd '$(dir $<)' && ld -r -b binary '$(notdir $<)' -o '$(abspath $@)'
+
+opensans_semibold.ttf.o: ../../selfdrive/assets/fonts/opensans_semibold.ttf
+	@echo "[ bin2o ] $@"
+	cd '$(dir $<)' && ld -r -b binary '$(notdir $<)' -o '$(abspath $@)'
+
+%.o: %.c
+	mkdir -p $(@D)
+	@echo "[ CC ] $@"
+	$(CC) $(CPPFLAGS) $(CFLAGS) \
+          -I../.. \
+          -I$(PHONELIBS)/android_frameworks_native/include \
+          -I$(PHONELIBS)/android_system_core/include \
+          -I$(PHONELIBS)/android_hardware_libhardware/include \
+          $(NANOVG_FLAGS) \
+          -c -o '$@' '$<'
+
+%.o: %.cc
+	mkdir -p $(@D)
+	@echo "[ CXX ] $@"
+	$(CXX) $(CPPFLAGS) $(CXXFLAGS) \
+           -I../../selfdrive \
+           -I../../ \
+           -I$(PHONELIBS)/android_frameworks_native/include \
+           -I$(PHONELIBS)/android_system_core/include \
+           -I$(PHONELIBS)/android_hardware_libhardware/include \
+           $(NANOVG_FLAGS) \
+           $(JSON11_FLAGS) \
+           $(CURL_FLAGS) \
+           $(BORINGSSL_FLAGS) \
+           -c -o '$@' '$<'
+
+
+.PHONY: clean
+clean:
+	rm -f $(OBJS) $(DEPS)
+
+-include $(DEPS)

installer/updater/update.json

@@ -0,0 +1,7 @@
+{
+  "ota_url": "https://commadist.azureedge.net/neosupdate/ota-signed-efdf7de63b1aef63d68301e6175930991bf9a5927d16ec6fcc69287e2ee7ca4a.zip",
+  "ota_hash": "efdf7de63b1aef63d68301e6175930991bf9a5927d16ec6fcc69287e2ee7ca4a",
+  "recovery_url": "https://commadist.azureedge.net/neosupdate/recovery-97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e.img",
+  "recovery_len": 15861036,
+  "recovery_hash": "97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e"
+}

installer/updater/update_kernel.json

@@ -0,0 +1,7 @@
+{
+  "ota_url": "https://commadist.azureedge.net/neosupdate/ota-signed-3bd2b3bdd6a501569e00b8f12786d65e0fd2788c0dd238f8c986e3e2e504683a-kernel.zip",
+  "ota_hash": "3bd2b3bdd6a501569e00b8f12786d65e0fd2788c0dd238f8c986e3e2e504683a",
+  "recovery_url": "https://commadist.azureedge.net/neosupdate/recovery-97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e.img",
+  "recovery_len": 15861036,
+  "recovery_hash": "97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e"
+}

installer/updater/updater.cc

@@ -0,0 +1,768 @@
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cassert>
+
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <mutex>
+#include <thread>
+
+#include <curl/curl.h>
+#include <openssl/sha.h>
+
+#include <GLES3/gl3.h>
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+
+#include "nanovg.h"
+#define NANOVG_GLES3_IMPLEMENTATION
+#include "nanovg_gl.h"
+#include "nanovg_gl_utils.h"
+
+#include "json11.hpp"
+
+#include "common/framebuffer.h"
+#include "common/touch.h"
+#include "common/utilpp.h"
+
+#define USER_AGENT "NEOSUpdater-0.2"
+
+#define MANIFEST_URL_EON_STAGING "https://github.com/commaai/eon-neos/raw/master/update.staging.json"
+#define MANIFEST_URL_EON_LOCAL "http://192.168.5.1:8000/neosupdate/update.local.json"
+#define MANIFEST_URL_EON "https://github.com/commaai/eon-neos/raw/master/update.json"
+const char *manifest_url = MANIFEST_URL_EON;
+
+#define RECOVERY_DEV "/dev/block/bootdevice/by-name/recovery"
+#define RECOVERY_COMMAND "/cache/recovery/command"
+
+#define UPDATE_DIR "/data/neoupdate"
+
+extern const uint8_t bin_opensans_regular[] asm("_binary_opensans_regular_ttf_start");
+extern const uint8_t bin_opensans_regular_end[] asm("_binary_opensans_regular_ttf_end");
+extern const uint8_t bin_opensans_semibold[] asm("_binary_opensans_semibold_ttf_start");
+extern const uint8_t bin_opensans_semibold_end[] asm("_binary_opensans_semibold_ttf_end");
+extern const uint8_t bin_opensans_bold[] asm("_binary_opensans_bold_ttf_start");
+extern const uint8_t bin_opensans_bold_end[] asm("_binary_opensans_bold_ttf_end");
+
+namespace {
+
+std::string sha256_file(std::string fn, size_t limit=0) {
+  SHA256_CTX ctx;
+  SHA256_Init(&ctx);
+
+  FILE *file = fopen(fn.c_str(), "rb");
+  if (!file) return "";
+
+  const size_t buf_size = 8192;
+  std::unique_ptr<char[]> buffer( new char[ buf_size ] );
+
+  bool read_limit = (limit != 0);
+  while (true) {
+    size_t read_size = buf_size;
+    if (read_limit) read_size = std::min(read_size, limit);
+    size_t bytes_read = fread(buffer.get(), 1, read_size, file);
+    if (!bytes_read) break;
+
+    SHA256_Update(&ctx, buffer.get(), bytes_read);
+
+    if (read_limit) {
+      limit -= bytes_read;
+      if (limit == 0) break;
+    }
+  }
+
+  uint8_t hash[SHA256_DIGEST_LENGTH];
+  SHA256_Final(hash, &ctx);
+
+  fclose(file);
+
+  return util::tohex(hash, sizeof(hash));
+}
+
+size_t download_string_write(void *ptr, size_t size, size_t nmeb, void *up) {
+  size_t sz = size * nmeb;
+  ((std::string*)up)->append((char*)ptr, sz);
+  return sz;
+}
+
+std::string download_string(CURL *curl, std::string url) {
+  std::string os;
+
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+  curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+  curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT);
+  curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+  curl_easy_setopt(curl, CURLOPT_RESUME_FROM, 0);
+
+  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, download_string_write);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &os);
+  CURLcode res = curl_easy_perform(curl);
+  if (res != CURLE_OK) {
+    return "";
+  }
+
+  return os;
+}
+
+size_t download_file_write(void *ptr, size_t size, size_t nmeb, void *up) {
+  return fwrite(ptr, size, nmeb, (FILE*)up);
+}
+
+int battery_capacity() {
+  std::string bat_cap_s = util::read_file("/sys/class/power_supply/battery/capacity");
+  return atoi(bat_cap_s.c_str());
+}
+
+int battery_current() {
+  std::string current_now_s = util::read_file("/sys/class/power_supply/battery/current_now");
+  return atoi(current_now_s.c_str());
+}
+
+bool check_battery() {
+  int bat_cap = battery_capacity();
+  int current_now = battery_current();
+  return bat_cap > 35 || (current_now < 0 && bat_cap > 10);
+}
+
+bool check_space() {
+  struct statvfs stat;
+  if (statvfs("/data/", &stat) != 0) {
+    return false;
+  }
+  size_t space = stat.f_bsize * stat.f_bavail;
+  return space > 2000000000ULL; // 2GB
+}
+
+static void start_settings_activity(const char* name) {
+  char launch_cmd[1024];
+  snprintf(launch_cmd, sizeof(launch_cmd),
+           "am start -W --ez :settings:show_fragment_as_subsetting true -n 'com.android.settings/.%s'", name);
+  system(launch_cmd);
+}
+
+struct Updater {
+  bool do_exit = false;
+
+  TouchState touch;
+
+  int fb_w, fb_h;
+
+  FramebufferState *fb = NULL;
+  NVGcontext *vg = NULL;
+  int font_regular;
+  int font_semibold;
+  int font_bold;
+
+  std::thread update_thread_handle;
+
+  std::mutex lock;
+
+  // i hate state machines give me coroutines already
+  enum UpdateState {
+    CONFIRMATION,
+    LOW_BATTERY,
+    RUNNING,
+    ERROR,
+  };
+  UpdateState state;
+
+  std::string progress_text;
+  float progress_frac;
+
+  std::string error_text;
+
+  std::string low_battery_text;
+  std::string low_battery_title;
+  std::string low_battery_context;
+  std::string battery_cap_text;
+  int min_battery_cap = 35;
+
+  // button
+  int b_x, b_w, b_y, b_h;
+  int balt_x;
+
+  CURL *curl = NULL;
+
+  Updater() {
+    touch_init(&touch);
+
+    fb = framebuffer_init("updater", 0x00001000, false,
+                          &fb_w, &fb_h);
+    assert(fb);
+
+    vg = nvgCreateGLES3(NVG_ANTIALIAS | NVG_STENCIL_STROKES | NVG_DEBUG);
+    assert(vg);
+
+    font_regular = nvgCreateFontMem(vg, "opensans_regular", (unsigned char*)bin_opensans_regular, (bin_opensans_regular_end - bin_opensans_regular), 0);
+    assert(font_regular >= 0);
+
+    font_semibold = nvgCreateFontMem(vg, "opensans_semibold", (unsigned char*)bin_opensans_semibold, (bin_opensans_semibold_end - bin_opensans_semibold), 0);
+    assert(font_semibold >= 0);
+
+    font_bold = nvgCreateFontMem(vg, "opensans_bold", (unsigned char*)bin_opensans_bold, (bin_opensans_bold_end - bin_opensans_bold), 0);
+    assert(font_bold >= 0);
+
+    b_w = 640;
+    balt_x = 200;
+    b_x = fb_w-b_w-200;
+    b_y = 720;
+    b_h = 220;
+
+    state = CONFIRMATION;
+
+  }
+
+  int download_file_xferinfo(curl_off_t dltotal, curl_off_t dlno,
+                             curl_off_t ultotal, curl_off_t ulnow) {
+    {
+      std::lock_guard<std::mutex> guard(lock);
+      if (dltotal != 0) {
+        progress_frac = (float) dlno / dltotal;
+      }
+    }
+    // printf("info: %ld %ld %f\n", dltotal, dlno, progress_frac);
+    return 0;
+  }
+
+  bool download_file(std::string url, std::string out_fn) {
+    FILE *of = fopen(out_fn.c_str(), "ab");
+    assert(of);
+
+    CURLcode res;
+    long last_resume_from = 0;
+
+    fseek(of, 0, SEEK_END);
+
+    int tries = 4;
+
+    bool ret = false;
+
+    while (true) {
+      long resume_from = ftell(of);
+
+      curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+      curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+      curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+      curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT);
+      curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+      curl_easy_setopt(curl, CURLOPT_RESUME_FROM, resume_from);
+
+      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, download_file_write);
+      curl_easy_setopt(curl, CURLOPT_WRITEDATA, of);
+
+      curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+
+      curl_easy_setopt(curl, CURLOPT_XFERINFODATA, this);
+      curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, &Updater::download_file_xferinfo);
+
+      CURLcode res = curl_easy_perform(curl);
+
+      long response_code = 0;
+      curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
+
+      // double content_length = 0.0;
+      // curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length);
+
+      printf("download %s res %d, code %ld, resume from %ld\n", url.c_str(), res, response_code, resume_from);
+      if (res == CURLE_OK) {
+        ret = true;
+        break;
+      } else if (res == CURLE_HTTP_RETURNED_ERROR && response_code == 416) {
+        // failed because the file is already complete?
+        ret = true;
+        break;
+      } else if (resume_from == last_resume_from) {
+        // failed and dind't make make forward progress. only retry a couple times
+        tries--;
+        if (tries <= 0) {
+          break;
+        }
+      }
+      last_resume_from = resume_from;
+    }
+    // printf("res %d\n", res);
+
+    // printf("- %ld %f\n", response_code, content_length);
+
+    fclose(of);
+
+    return ret;
+  }
+
+  void set_progress(std::string text) {
+    std::lock_guard<std::mutex> guard(lock);
+    progress_text = text;
+  }
+
+  void set_error(std::string text) {
+    std::lock_guard<std::mutex> guard(lock);
+    error_text = text;
+    state = ERROR;
+  }
+
+  void set_battery_low() {
+    std::lock_guard<std::mutex> guard(lock);
+    state = LOW_BATTERY;
+  }
+
+  void set_running() {
+    std::lock_guard<std::mutex> guard(lock);
+    state = RUNNING;
+  }
+
+  std::string stage_download(std::string url, std::string hash, std::string name) {
+    std::string out_fn = UPDATE_DIR "/" + util::base_name(url);
+
+    set_progress("Downloading " + name + "...");
+    bool r = download_file(url, out_fn);
+    if (!r) {
+      set_error("failed to download " + name);
+      return "";
+    }
+
+    set_progress("Verifying " + name + "...");
+    std::string fn_hash = sha256_file(out_fn);
+    printf("got %s hash: %s\n", name.c_str(), hash.c_str());
+    if (fn_hash != hash) {
+      set_error(name + " was corrupt");
+      unlink(out_fn.c_str());
+      return "";
+    }
+
+    return out_fn;
+  }
+
+  void run_stages() {
+    curl = curl_easy_init();
+    assert(curl);
+
+    if (!check_battery()) {
+      set_battery_low();
+      int battery_cap = battery_capacity();
+      while(battery_cap < min_battery_cap) {
+        battery_cap = battery_capacity();
+        battery_cap_text = std::to_string(battery_cap);
+        usleep(1000000);
+      }
+      set_running();
+    }
+
+    if (!check_space()) {
+      set_error("2GB of free space required to update");
+      return;
+    }
+
+    mkdir(UPDATE_DIR, 0777);
+
+    const int EON = (access("/EON", F_OK) != -1);
+
+    set_progress("Finding latest version...");
+    std::string manifest_s;
+    if (EON) {
+      manifest_s = download_string(curl, manifest_url);
+    } else {
+      // don't update NEO
+      exit(0);
+    }
+
+    printf("manifest: %s\n", manifest_s.c_str());
+
+    std::string err;
+    auto manifest = json11::Json::parse(manifest_s, err);
+    if (manifest.is_null() || !err.empty()) {
+      set_error("failed to load update manifest");
+      return;
+    }
+
+    std::string ota_url = manifest["ota_url"].string_value();
+    std::string ota_hash = manifest["ota_hash"].string_value();
+
+    std::string recovery_url = manifest["recovery_url"].string_value();
+    std::string recovery_hash = manifest["recovery_hash"].string_value();
+    int recovery_len = manifest["recovery_len"].int_value();
+
+    // std::string installer_url = manifest["installer_url"].string_value();
+    // std::string installer_hash = manifest["installer_hash"].string_value();
+
+    if (ota_url.empty() || ota_hash.empty()) {
+      set_error("invalid update manifest");
+      return;
+    }
+
+    // std::string installer_fn = stage_download(installer_url, installer_hash, "installer");
+    // if (installer_fn.empty()) {
+    //   //error'd
+    //   return;
+    // }
+
+    std::string recovery_fn;
+    if (recovery_url.empty() || recovery_hash.empty() || recovery_len == 0) {
+      set_progress("Skipping recovery flash...");
+    } else {
+      // only download the recovery if it differs from what's flashed
+      set_progress("Checking recovery...");
+      std::string existing_recovery_hash = sha256_file(RECOVERY_DEV, recovery_len);
+      printf("existing recovery hash: %s\n", existing_recovery_hash.c_str());
+
+      if (existing_recovery_hash != recovery_hash) {
+        recovery_fn = stage_download(recovery_url, recovery_hash, "recovery");
+        if (recovery_fn.empty()) {
+          // error'd
+          return;
+        }
+      }
+    }
+
+    std::string ota_fn = stage_download(ota_url, ota_hash, "update");
+    if (ota_fn.empty()) {
+      //error'd
+      return;
+    }
+
+    if (!check_battery()) {
+      set_battery_low();
+      int battery_cap = battery_capacity();
+      while(battery_cap < min_battery_cap) {
+        battery_cap = battery_capacity();
+        battery_cap_text = std::to_string(battery_cap);
+        usleep(1000000);
+      }
+      set_running();
+    }
+
+    if (!recovery_fn.empty()) {
+      // flash recovery
+      set_progress("Flashing recovery...");
+
+      FILE *flash_file = fopen(recovery_fn.c_str(), "rb");
+      if (!flash_file) {
+        set_error("failed to flash recovery");
+        return;
+      }
+
+      FILE *recovery_dev = fopen(RECOVERY_DEV, "w+b");
+      if (!recovery_dev) {
+        fclose(flash_file);
+        set_error("failed to flash recovery");
+        return;
+      }
+
+      const size_t buf_size = 4096;
+      std::unique_ptr<char[]> buffer( new char[ buf_size ] );
+
+      while (true) {
+        size_t bytes_read = fread(buffer.get(), 1, buf_size, flash_file);
+        if (!bytes_read) break;
+
+        size_t bytes_written = fwrite(buffer.get(), 1, bytes_read, recovery_dev);
+        if (bytes_read != bytes_written) {
+          fclose(recovery_dev);
+          fclose(flash_file);
+          set_error("failed to flash recovery: write failed");
+          return;
+        }
+      }
+
+      fclose(recovery_dev);
+      fclose(flash_file);
+
+      set_progress("Verifying flash...");
+      std::string new_recovery_hash = sha256_file(RECOVERY_DEV, recovery_len);
+      printf("new recovery hash: %s\n", new_recovery_hash.c_str());
+
+      if (new_recovery_hash != recovery_hash) {
+        set_error("recovery flash corrupted");
+        return;
+      }
+
+    }
+
+    // write arguments to recovery
+    FILE *cmd_file = fopen(RECOVERY_COMMAND, "wb");
+    if (!cmd_file) {
+      set_error("failed to reboot into recovery");
+      return;
+    }
+    fprintf(cmd_file, "--update_package=%s\n", ota_fn.c_str());
+    fclose(cmd_file);
+
+    set_progress("Rebooting");
+
+    // remove the continue.sh so we come back into the setup.
+    // maybe we should go directly into the installer, but what if we don't come back with internet? :/
+    //unlink("/data/data/com.termux/files/continue.sh");
+
+    // TODO: this should be generic between android versions
+    // IPowerManager.reboot(confirm=false, reason="recovery", wait=true)
+    system("service call power 16 i32 0 s16 recovery i32 1");
+    while(1) pause();
+
+    // execl("/system/bin/reboot", "recovery");
+    // set_error("failed to reboot into recovery");
+  }
+
+  void draw_ack_screen(const char *title, const char *message, const char *button, const char *altbutton) {
+    nvgFillColor(vg, nvgRGBA(255,255,255,255));
+    nvgTextAlign(vg, NVG_ALIGN_CENTER | NVG_ALIGN_BASELINE);
+
+    nvgFontFace(vg, "opensans_bold");
+    nvgFontSize(vg, 120.0f);
+    nvgTextBox(vg, 110, 220, fb_w-240, title, NULL);
+
+    nvgFontFace(vg, "opensans_regular");
+    nvgFontSize(vg, 86.0f);
+    nvgTextBox(vg, 130, 380, fb_w-260, message, NULL);
+
+    // draw button
+    if (button) {
+      nvgBeginPath(vg);
+      nvgFillColor(vg, nvgRGBA(8, 8, 8, 255));
+      nvgRoundedRect(vg, b_x, b_y, b_w, b_h, 20);
+      nvgFill(vg);
+
+      nvgFillColor(vg, nvgRGBA(255, 255, 255, 255));
+      nvgFontFace(vg, "opensans_semibold");
+      nvgTextAlign(vg, NVG_ALIGN_CENTER | NVG_ALIGN_MIDDLE);
+      nvgText(vg, b_x+b_w/2, b_y+b_h/2, button, NULL);
+
+      nvgBeginPath(vg);
+      nvgStrokeColor(vg, nvgRGBA(255, 255, 255, 50));
+      nvgStrokeWidth(vg, 5);
+      nvgRoundedRect(vg, b_x, b_y, b_w, b_h, 20);
+      nvgStroke(vg);
+    }
+
+    // draw button
+    if (altbutton) {
+      nvgBeginPath(vg);
+      nvgFillColor(vg, nvgRGBA(8, 8, 8, 255));
+      nvgRoundedRect(vg, balt_x, b_y, b_w, b_h, 20);
+      nvgFill(vg);
+
+      nvgFillColor(vg, nvgRGBA(255, 255, 255, 255));
+      nvgFontFace(vg, "opensans_semibold");
+      nvgTextAlign(vg, NVG_ALIGN_CENTER | NVG_ALIGN_MIDDLE);
+      nvgText(vg, balt_x+b_w/2, b_y+b_h/2, altbutton, NULL);
+
+      nvgBeginPath(vg);
+      nvgStrokeColor(vg, nvgRGBA(255, 255, 255, 50));
+      nvgStrokeWidth(vg, 5);
+      nvgRoundedRect(vg, balt_x, b_y, b_w, b_h, 20);
+      nvgStroke(vg);
+    }
+  }
+
+  void draw_battery_screen() {
+    low_battery_title = "Low Battery";
+    low_battery_text = "Please connect EON to your charger. Update will continue once EON battery reaches 35%.";
+    low_battery_context = "Current battery charge: " + battery_cap_text + "%";
+
+    nvgFillColor(vg, nvgRGBA(255,255,255,255));
+    nvgTextAlign(vg, NVG_ALIGN_CENTER | NVG_ALIGN_BASELINE);
+
+    nvgFontFace(vg, "opensans_bold");
+    nvgFontSize(vg, 120.0f);
+    nvgTextBox(vg, 110, 220, fb_w-240, low_battery_title.c_str(), NULL);
+
+    nvgFontFace(vg, "opensans_regular");
+    nvgFontSize(vg, 86.0f);
+    nvgTextBox(vg, 130, 380, fb_w-260, low_battery_text.c_str(), NULL);
+
+    nvgFontFace(vg, "opensans_bold");
+    nvgFontSize(vg, 86.0f);
+    nvgTextBox(vg, 130, 700, fb_w-260, low_battery_context.c_str(), NULL);
+  }
+
+  void draw_progress_screen() {
+    // draw progress message
+    nvgFontSize(vg, 64.0f);
+    nvgFillColor(vg, nvgRGBA(255,255,255,255));
+    nvgTextAlign(vg, NVG_ALIGN_CENTER | NVG_ALIGN_BASELINE);
+    nvgFontFace(vg, "opensans_bold");
+    nvgFontSize(vg, 86.0f);
+    nvgTextBox(vg, 0, 380, fb_w, progress_text.c_str(), NULL);
+
+    // draw progress bar
+    {
+      int progress_width = 1000;
+      int progress_x = fb_w/2-progress_width/2;
+      int progress_y = 520;
+      int progress_height = 50;
+
+      int powerprompt_y = 312;
+      nvgFontFace(vg, "opensans_regular");
+      nvgFontSize(vg, 64.0f);
+      nvgText(vg, fb_w/2, 740, "Ensure EON is connected to power.", NULL);
+
+      NVGpaint paint = nvgBoxGradient(
+          vg, progress_x + 1, progress_y + 1,
+          progress_width - 2, progress_height, 3, 4, nvgRGB(27, 27, 27), nvgRGB(27, 27, 27));
+      nvgBeginPath(vg);
+      nvgRoundedRect(vg, progress_x, progress_y, progress_width, progress_height, 12);
+      nvgFillPaint(vg, paint);
+      nvgFill(vg);
+
+      float value = std::min(std::max(0.0f, progress_frac), 1.0f);
+      int bar_pos = ((progress_width - 2) * value);
+
+      paint = nvgBoxGradient(
+          vg, progress_x, progress_y,
+          bar_pos+1.5f, progress_height-1, 3, 4,
+          nvgRGB(245, 245, 245), nvgRGB(105, 105, 105));
+
+      nvgBeginPath(vg);
+      nvgRoundedRect(
+          vg, progress_x+1, progress_y+1,
+          bar_pos, progress_height-2, 12);
+      nvgFillPaint(vg, paint);
+      nvgFill(vg);
+    }
+  }
+
+  void ui_draw() {
+    std::lock_guard<std::mutex> guard(lock);
+
+    nvgBeginFrame(vg, fb_w, fb_h, 1.0f);
+
+    switch (state) {
+    case CONFIRMATION:
+      draw_ack_screen("An update to NEOS is required.",
+                      "Your device will now be reset and upgraded. You may want to connect to wifi as download is around 1 GB. Existing data on device should not be lost.",
+                      "Continue",
+                      "Connect to WiFi");
+      break;
+    case LOW_BATTERY:
+      draw_battery_screen();
+      break;
+    case RUNNING:
+      draw_progress_screen();
+      break;
+    case ERROR:
+      draw_ack_screen("There was an error", (error_text).c_str(), NULL, "Reboot");
+      break;
+    }
+
+    nvgEndFrame(vg);
+  }
+
+  void ui_update() {
+    std::lock_guard<std::mutex> guard(lock);
+
+    switch (state) {
+    case ERROR:
+    case CONFIRMATION: {
+      int touch_x = -1, touch_y = -1;
+      int res = touch_poll(&touch, &touch_x, &touch_y, 0);
+      if (res == 1 && !is_settings_active()) {
+        if (touch_x >= b_x && touch_x < b_x+b_w && touch_y >= b_y && touch_y < b_y+b_h) {
+          if (state == CONFIRMATION) {
+            state = RUNNING;
+            update_thread_handle = std::thread(&Updater::run_stages, this);
+          }
+        }
+        if (touch_x >= balt_x && touch_x < balt_x+b_w && touch_y >= b_y && touch_y < b_y+b_h) {
+          if (state == CONFIRMATION) {
+            start_settings_activity("Settings$WifiSettingsActivity");
+          } else if (state == ERROR) {
+            do_exit = 1;
+          }
+        }
+      }
+    }
+    default:
+      break;
+    }
+  }
+
+
+  void go() {
+    while (!do_exit) {
+      ui_update();
+
+      glClearColor(0.08, 0.08, 0.08, 1.0);
+      glClear(GL_STENCIL_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
+
+      // background
+      nvgBeginPath(vg);
+      NVGpaint bg = nvgLinearGradient(vg, fb_w, 0, fb_w, fb_h,
+        nvgRGBA(0, 0, 0, 0), nvgRGBA(0, 0, 0, 255));
+      nvgFillPaint(vg, bg);
+      nvgRect(vg, 0, 0, fb_w, fb_h);
+      nvgFill(vg);
+
+      glEnable(GL_BLEND);
+      glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+
+      ui_draw();
+
+      glDisable(GL_BLEND);
+
+      framebuffer_swap(fb);
+
+      assert(glGetError() == GL_NO_ERROR);
+
+      // no simple way to do 30fps vsync with surfaceflinger...
+      usleep(30000);
+    }
+
+    if (update_thread_handle.joinable()) {
+      update_thread_handle.join();
+    }
+
+    system("service call power 16 i32 0 i32 0 i32 1");
+  }
+
+  bool is_settings_active() {
+    FILE *fp;
+    char sys_output[4096];
+
+    fp = popen("/bin/dumpsys window windows", "r");
+    if (fp == NULL) {
+      return false;
+    }
+
+    bool active = false;
+    while (fgets(sys_output, sizeof(sys_output), fp) != NULL) {
+      if (strstr(sys_output, "mCurrentFocus=null")  != NULL) {
+        break;
+      }
+
+      if (strstr(sys_output, "mCurrentFocus=Window") != NULL) {
+        active = true;
+        break;
+      }
+    }
+
+    pclose(fp);
+
+    return active;
+  }
+
+};
+
+}
+int main(int argc, char *argv[]) {
+  if (argc > 1) {
+    if (strcmp(argv[1], "local") == 0) {
+      manifest_url = MANIFEST_URL_EON_LOCAL;
+    } else if (strcmp(argv[1], "staging") == 0) {
+      manifest_url = MANIFEST_URL_EON_STAGING;
+    } else {
+      manifest_url = argv[1];
+    }
+  }
+  printf("updating from %s\n", manifest_url);
+  Updater updater;
+  updater.go();
+
+  return 0;
+}

launch_chffrplus.sh

@@ -1,24 +1,122 @@
 #!/usr/bin/bash
 
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export NUMEXPR_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export VECLIB_MAXIMUM_THREADS=1
+
+if [ -z "$BASEDIR" ]; then
+  BASEDIR="/data/openpilot"
+fi
+
 if [ -z "$PASSIVE" ]; then
   export PASSIVE="1"
 fi
 
+STAGING_ROOT="/data/safe_staging"
+
 function launch {
-  # apply update
-  if [ "$(git rev-parse HEAD)" != "$(git rev-parse @{u})" ]; then
-     git reset --hard @{u} &&
-     git clean -xdf &&
-     exec "${BASH_SOURCE[0]}"
+  # Wifi scan
+  wpa_cli IFNAME=wlan0 SCAN
+
+  # Check to see if there's a valid overlay-based update available. Conditions
+  # are as follows:
+  #
+  # 1. The BASEDIR init file has to exist, with a newer modtime than anything in
+  #    the BASEDIR Git repo. This checks for local development work or the user
+  #    switching branches/forks, which should not be overwritten.
+  # 2. The FINALIZED consistent file has to exist, indicating there's an update
+  #    that completed successfully and synced to disk.
+
+  if [ -f "${BASEDIR}/.overlay_init" ]; then
+    find ${BASEDIR}/.git -newer ${BASEDIR}/.overlay_init | grep -q '.' 2> /dev/null
+    if [ $? -eq 0 ]; then
+      echo "${BASEDIR} has been modified, skipping overlay update installation"
+    else
+      if [ -f "${STAGING_ROOT}/finalized/.overlay_consistent" ]; then
+        if [ ! -d /data/safe_staging/old_openpilot ]; then
+          echo "Valid overlay update found, installing"
+          LAUNCHER_LOCATION="${BASH_SOURCE[0]}"
+
+          mv $BASEDIR /data/safe_staging/old_openpilot
+          mv "${STAGING_ROOT}/finalized" $BASEDIR
+          cd $BASEDIR
+
+          # Partial mitigation for symlink-related filesystem corruption
+          # Ensure all files match the repo versions after update
+          git reset --hard
+          git submodule foreach --recursive git reset --hard
+
+          echo "Restarting launch script ${LAUNCHER_LOCATION}"
+          exec "${LAUNCHER_LOCATION}"
+        else
+          echo "openpilot backup found, not updating"
+          # TODO: restore backup? This means the updater didn't start after swapping
+        fi
+      fi
+    fi
   fi
 
-  # no cpu rationing for now
-  echo 0-3 > /dev/cpuset/background/cpus
-  echo 0-3 > /dev/cpuset/system-background/cpus
-  echo 0-3 > /dev/cpuset/foreground/boost/cpus
-  echo 0-3 > /dev/cpuset/foreground/cpus
-  echo 0-3 > /dev/cpuset/android/cpus
+  # Android and other system processes are not permitted to run on CPU 3
+  # NEOS installed app processes can run anywhere
+  echo 0-2 > /dev/cpuset/background/cpus
+  echo 0-2 > /dev/cpuset/system-background/cpus
+  [ -d "/dev/cpuset/foreground/boost/cpus" ] && echo 0-2 > /dev/cpuset/foreground/boost/cpus  # Not present in < NEOS 15
+  echo 0-2 > /dev/cpuset/foreground/cpus
+  echo 0-2 > /dev/cpuset/android/cpus
+  echo 0-3 > /dev/cpuset/app/cpus
 
+  # Collect RIL and other possibly long-running I/O interrupts onto CPU 1
+  echo 1 > /proc/irq/78/smp_affinity_list # qcom,smd-modem (LTE radio)
+  echo 1 > /proc/irq/33/smp_affinity_list # ufshcd (flash storage)
+  echo 1 > /proc/irq/35/smp_affinity_list # wifi (wlan_pci)
+  # USB traffic needs realtime handling on cpu 3
+  [ -d "/proc/irq/733" ] && echo 3 > /proc/irq/733/smp_affinity_list # USB for LeEco
+  [ -d "/proc/irq/736" ] && echo 3 > /proc/irq/736/smp_affinity_list # USB for OP3T
+
+  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+  # Remove old NEOS update file
+  # TODO: move this code to the updater
+  if [ -d /data/neoupdate ]; then
+    rm -rf /data/neoupdate
+  fi
+
+  # Check for NEOS update
+  if [ $(< /VERSION) != "14" ]; then
+    if [ -f "$DIR/scripts/continue.sh" ]; then
+      cp "$DIR/scripts/continue.sh" "/data/data/com.termux/files/continue.sh"
+    fi
+
+    if [ ! -f "$BASEDIR/prebuilt" ]; then
+      echo "Clearing build products and resetting scons state prior to NEOS update"
+      cd $BASEDIR && scons --clean
+      rm -rf /tmp/scons_cache
+      rm -r $BASEDIR/.sconsign.dblite
+    fi
+    "$DIR/installer/updater/updater" "file://$DIR/installer/updater/update.json"
+  else
+    if [[ $(uname -v) == "#1 SMP PREEMPT Wed Jun 10 12:40:53 PDT 2020" ]]; then
+      "$DIR/installer/updater/updater" "file://$DIR/installer/updater/update_kernel.json"
+    fi
+  fi
+
+  # One-time fix for a subset of OP3T with gyro orientation offsets.
+  # Remove and regenerate qcom sensor registry. Only done on OP3T mainboards.
+  # Performed exactly once. The old registry is preserved just-in-case, and
+  # doubles as a flag denoting we've already done the reset.
+  # TODO: we should really grow per-platform detect and setup routines
+  if ! $(grep -q "letv" /proc/cmdline) && [ ! -f "/persist/comma/op3t-sns-reg-backup" ]; then
+    echo "Performing OP3T sensor registry reset"
+    mv /persist/sensors/sns.reg /persist/comma/op3t-sns-reg-backup &&
+      rm -f /persist/sensors/sensors_settings /persist/sensors/error_log /persist/sensors/gyro_sensitity_cal &&
+      echo "restart" > /sys/kernel/debug/msm_subsys/slpi &&
+      sleep 5  # Give Android sensor subsystem a moment to recover
+  fi
+
+  # handle pythonpath
+  ln -sfn $(pwd) /data/pythonpath
   export PYTHONPATH="$PWD"
 
   # start manager

opendbc/.gitignore

@@ -1,2 +1,17 @@
+.mypy_cache/
 *.pyc
+*.os
+*.tmp
+*.dylib
 .*.swp
+.DS_Store
+.sconsign.dblite
+
+can/*.so
+can/build/
+can/obj/
+can/packer_pyx.cpp
+can/parser_pyx.cpp
+can/packer_pyx.html
+can/parser_pyx.html
+can/packer_impl.cpp

opendbc/README.md

@@ -1,53 +0,0 @@
-opendbc
-======
-
-The project to democratize access to the decoder ring of your car.
-
-
-
-### DBC file basics
-
-A DBC file encodes, in a humanly readable way, the information needed to understand a vehicle's CAN bus traffic. A vehicle might have multiple CAN buses and every CAN bus is represented by its own dbc file.
-Wondering what's the DBC file format? [Here](http://www.socialledge.com/sjsu/index.php?title=DBC_Format) and [Here](https://github.com/stefanhoelzl/CANpy/blob/master/docs/DBC_Specification.md) a couple of good overviews.
-
-### How to start reverse engineering cars
-
-[opendbc](https://github.com/commaai/opendbc) is integrated with [cabana](https://community.comma.ai/cabana/).
-
-Use [panda](https://github.com/commaai/panda) to connect your car to a computer.
-
-### DBC file preprocessor
-
-DBC files for different models of the same brand have a lot of overlap. Therefore, we wrote a preprocessor to create DBC files from a brand DBC file and a model specific DBC file. The source DBC files can be found in the generator folder. After changing one of the files run the generator.py script to regenerate the output files. These output files will be placed in the root of the opendbc repository and are suffixed by _generated.
-
-### Good practices for contributing to opendbc
-
-- Comments: the best way to store comments is to add them directly to the DBC files. For example:
-    ```
-    CM_ SG_ 490 LONG_ACCEL "wheel speed derivative, noisy and zero snapping";
-    ```
-    is a comment that refers to signal `LONG_ACCEL` in message `490`. Using comments is highly recommended, especially for doubts and uncertainties. [cabana](https://community.comma.ai/cabana/) can easily display/add/edit comments to signals and messages.
-
-- Units: when applicable, it's recommended to convert signals into physical units, by using a proper signal factor. Using a SI unit is preferred, unless a non-SI unit rounds the signal factor much better.
-For example:
-    ```
-    SG_ VEHICLE_SPEED : 7|15@0+ (0.00278,0) [0|70] "m/s" PCM
-    ```
-    is better than:
-    ```
-    SG_ VEHICLE_SPEED : 7|15@0+ (0.00620,0) [0|115] "mph" PCM
-    ```
-    However, the cleanest option is really:
-    ```
-    SG_ VEHICLE_SPEED : 7|15@0+ (0.01,0) [0|250] "kph" PCM
-    ```
-
-- Signal's size: always use the smallest amount of bits possible. For example, let's say I'm reverse engineering the gas pedal position and I've determined that it's in a 3 bytes message. For 0% pedal position I read a message value of `0x00 0x00 0x00`, while for 100% of pedal position I read `0x64 0x00 0x00`: clearly, the gas pedal position is within the first byte of the message and I might be tempted to define the signal `GAS_POS` as:
-    ```
-    SG_ GAS_POS : 7|8@0+ (1,0) [0|100] "%" PCM
-    ```
-    However, I can't be sure that the very first bit of the message is referred to the pedal position: I haven't seen it changing! Therefore, a safer way of defining the signal is:
-    ```
-    SG_ GAS_POS : 6|7@0+ (1,0) [0|100] "%" PCM
-    ```
-    which leaves the first bit unallocated. This prevents from very erroneous reading of the gas pedal position, in case the first bit is indeed used for something else.

opendbc/acura_ilx_2016_can_generated.dbc

@@ -6,15 +6,15 @@ BO_ 512 GAS_COMMAND: 6 EON
  SG_ GAS_COMMAND : 7|16@0+ (0.253984064,-83.3) [0|1] "" INTERCEPTOR
  SG_ GAS_COMMAND2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" INTERCEPTOR
  SG_ ENABLE : 39|1@0+ (1,0) [0|1] "" INTERCEPTOR
- SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" INTERCEPTOR
- SG_ CHECKSUM : 43|4@0+ (1,0) [0|3] "" INTERCEPTOR
+ SG_ COUNTER_PEDAL : 35|4@0+ (1,0) [0|15] "" INTERCEPTOR
+ SG_ CHECKSUM_PEDAL : 47|8@0+ (1,0) [0|255] "" INTERCEPTOR
 
 BO_ 513 GAS_SENSOR: 6 INTERCEPTOR
  SG_ INTERCEPTOR_GAS : 7|16@0+ (0.253984064,-83.3) [0|1] "" EON
  SG_ INTERCEPTOR_GAS2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" EON
- SG_ STATE : 39|8@0+ (1,0) [0|255] "" EON
- SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" EON
- SG_ CHECKSUM : 43|4@0+ (1,0) [0|3] "" EON
+ SG_ STATE : 39|4@0+ (1,0) [0|15] "" EON
+ SG_ COUNTER_PEDAL : 35|4@0+ (1,0) [0|15] "" EON
+ SG_ CHECKSUM_PEDAL : 47|8@0+ (1,0) [0|255] "" EON
 
 VAL_ 513 STATE 5 "FAULT_TIMEOUT" 4 "FAULT_STARTUP" 3 "FAULT_SCE" 2 "FAULT_SEND" 1 "FAULT_BAD_CHECKSUM" 0 "NO_FAULT" ;
 
@@ -59,9 +59,9 @@ BO_ 344 ENGINE_DATA: 8 PCM
  SG_ XMISSION_SPEED : 7|16@0+ (0.01,0) [0|250] "kph" EON
  SG_ ENGINE_RPM : 23|16@0+ (1,0) [0|15000] "rpm" EON
  SG_ XMISSION_SPEED2 : 39|16@0+ (0.01,0) [0|250] "kph" EON
+ SG_ ODOMETER : 55|8@0+ (10,0) [0|2550] "m" XXX
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
- SG_ ODOMETER : 55|8@0+ (0.010588,0) [0|255] "km" XXX
 
 BO_ 380 POWERTRAIN_DATA: 8 PCM
  SG_ PEDAL_GAS : 7|8@0+ (1,0) [0|255] "" EON
@@ -76,6 +76,23 @@ BO_ 380 POWERTRAIN_DATA: 8 PCM
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
 
+BO_ 420 VSA_STATUS: 8 VSA
+ SG_ USER_BRAKE : 7|16@0+ (0.015625,-1.609375) [0|1000] "" EON
+ SG_ COMPUTER_BRAKING : 23|1@0+ (1,0) [0|1] "" EON
+ SG_ ESP_DISABLED : 28|1@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_HOLD_RELATED : 52|1@0+ (1,0) [0|1] "" XXX
+ SG_ BRAKE_HOLD_ACTIVE : 46|1@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_HOLD_ENABLED : 45|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 427 STEER_MOTOR_TORQUE: 3 EPS
+ SG_ CONFIG_VALID : 7|1@0+ (1,0) [0|1] "" EON
+ SG_ MOTOR_TORQUE : 1|10@0+ (1,0) [0|256] "" EON
+ SG_ OUTPUT_DISABLED : 22|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 21|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 19|4@0+ (1,0) [0|15] "" EON
+
 BO_ 432 STANDSTILL: 7 VSA
  SG_ CONTROLLED_STANDSTILL : 0|1@0+ (1,0) [0|1] "" EON
  SG_ WHEELS_MOVING : 12|1@0+ (1,0) [0|1] "" EON
@@ -98,21 +115,24 @@ BO_ 490 VEHICLE_DYNAMICS: 8 VSA
 
 BO_ 506 BRAKE_COMMAND: 8 ADAS
  SG_ COMPUTER_BRAKE : 7|10@0+ (1,0) [0|1] "" EBCM
- SG_ ZEROS_BOH : 13|5@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00 : 13|5@0+ (1,0) [0|1] "" EBCM
  SG_ BRAKE_PUMP_REQUEST : 8|1@0+ (1,0) [0|1] "" EBCM
- SG_ CRUISE_BOH2 : 23|3@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00_2 : 23|3@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_OVERRIDE : 20|1@0+ (1,0) [0|1] "" EBCM
- SG_ CRUISE_BOH3 : 19|1@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00_3 : 19|1@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_FAULT_CMD : 18|1@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_CANCEL_CMD : 17|1@0+ (1,0) [0|1] "" EBCM
  SG_ COMPUTER_BRAKE_REQUEST : 16|1@0+ (1,0) [0|1] "" EBCM
- SG_ SET_ME_0X80 : 31|8@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_1 : 31|1@0+ (1,0) [0|1] "" EBCM
+ SG_ AEB_REQ_1 : 29|1@0+ (1,0) [0|1] "" XXX
+ SG_ AEB_REQ_2 : 26|3@0+ (1,0) [0|7] "" XXX
  SG_ BRAKE_LIGHTS : 39|1@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_STATES : 38|7@0+ (1,0) [0|1] "" EBCM
  SG_ CHIME : 47|3@0+ (1,0) [0|7] "" EBCM
- SG_ ZEROS_BOH6 : 44|1@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00_4 : 44|1@0+ (1,0) [0|1] "" EBCM
  SG_ FCW : 43|2@0+ (1,0) [0|3] "" EBCM
- SG_ ZEROS_BOH4 : 55|8@0+ (1,0) [0|0] "" EBCM
+ SG_ AEB_STATUS : 41|2@0+ (1,0) [0|3] "" XXX
+ SG_ SET_ME_X00_5 : 55|8@0+ (1,0) [0|0] "" EBCM
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EBCM
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EBCM
 
@@ -151,19 +171,22 @@ BO_ 780 ACC_HUD: 8 ADAS
  SG_ BOH : 38|1@0+ (1,0) [0|1] "" BDY
  SG_ ACC_PROBLEM : 37|1@0+ (1,0) [0|1] "" BDY
  SG_ FCM_OFF : 36|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH_2 : 35|1@0+ (1,0) [0|1] "" BDY
+ SG_ FCM_OFF_2 : 35|1@0+ (1,0) [0|1] "" BDY
  SG_ FCM_PROBLEM : 34|1@0+ (1,0) [0|1] "" BDY
  SG_ RADAR_OBSTRUCTED : 33|1@0+ (1,0) [0|1] "" BDY
  SG_ ENABLE_MINI_CAR : 32|1@0+ (1,0) [0|1] "" BDY
- SG_ SET_ME_X03 : 47|2@0+ (1,0) [0|3] "" BDY
+ SG_ HUD_DISTANCE : 47|2@0+ (1,0) [0|3] "" BDY
  SG_ HUD_LEAD : 45|2@0+ (1,0) [0|3] "" BDY
  SG_ BOH_3 : 43|1@0+ (1,0) [0|3] "" BDY
  SG_ BOH_4 : 42|1@0+ (1,0) [0|3] "" BDY
  SG_ BOH_5 : 41|1@0+ (1,0) [0|3] "" BDY
  SG_ CRUISE_CONTROL_LABEL : 40|1@0+ (1,0) [0|3] "" BDY
+ SG_ SET_ME_X01_2 : 55|1@0+ (1,0) [0|1] "" BDY
+ SG_ IMPERIAL_UNIT : 54|1@0+ (1,0) [0|1] "" BDY
  SG_ HUD_DISTANCE_3 : 52|1@0+ (1,0) [0|1] "" BDY
- SG_ SET_ME_X03_2 : 55|2@0+ (1,0) [0|3] "" BDY
+ SG_ CHIME : 51|3@0+ (1,0) [0|1] "" BDY
  SG_ SET_ME_X01 : 48|1@0+ (1,0) [0|1] "" BDY
+ SG_ ICONS : 63|2@0+ (1,0) [0|1] "" BDY
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" BDY
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" BDY
 
@@ -213,6 +236,7 @@ BO_ 1029 DOORS_STATUS: 8 BDY
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
 
 CM_ SG_ 490 LONG_ACCEL "wheel speed derivative, noisy and zero snapping";
+CM_ SG_ 506 AEB_REQ_1 "set for duration of suspected AEB event";
 CM_ SG_ 773 PASS_AIRBAG_ON "Might just be indicator light";
 CM_ SG_ 773 PASS_AIRBAG_OFF "Might just be indicator light";
 CM_ SG_ 780 CRUISE_SPEED "255 = no speed";
@@ -222,6 +246,7 @@ CM_ SG_ 829 BEEP "beeps are pleasant, chimes are for warnngs etc...";
 
 VAL_ 506 FCW 3 "fcw" 2 "fcw" 1 "fcw" 0 "no_fcw" ;
 VAL_ 506 CHIME 4 "double_chime" 3 "single_chime" 2 "continuous_chime" 1 "repeating_chime" 0 "no_chime" ;
+VAL_ 506 AEB_STATUS 3 "aeb_prepare" 2 "aeb_ready" 1 "aeb_braking" 0 "no_aeb" ;
 VAL_ 780 CRUISE_SPEED 255 "no_speed" 252 "stopped" ;
 VAL_ 780 HUD_LEAD 3 "acc_off" 2 "solid_car" 1 "dashed_car" 0 "no_car" ;
 VAL_ 829 BEEP 3 "single_beep" 2 "triple_beep" 1 "repeated_beep" 0 "no_beep" ;
@@ -255,10 +280,11 @@ BO_ 342 STEERING_SENSORS: 6 EPS
  SG_ CHECKSUM : 43|4@0+ (1,0) [0|3] "" EON
 
 BO_ 399 STEER_STATUS: 7 EPS
- SG_ STEER_TORQUE_SENSOR : 7|16@0- (1,0) [-31000|31000] "tbd" EON
- SG_ STEER_TORQUE_MOTOR : 23|16@0- (1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_TORQUE_SENSOR : 7|16@0- (-1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_ANGLE_RATE : 23|16@0- (-0.1,0) [-31000|31000] "deg/s" EON
  SG_ STEER_STATUS : 39|4@0+ (1,0) [0|15] "" EON
  SG_ STEER_CONTROL_ACTIVE : 35|1@0+ (1,0) [0|1] "" EON
+ SG_ STEER_CONFIG_INDEX : 43|4@0+ (1,0) [0|15] "" EON
  SG_ COUNTER : 53|2@0+ (1,0) [0|3] "" EON
  SG_ CHECKSUM : 51|4@0+ (1,0) [0|3] "" EON
 
@@ -268,12 +294,6 @@ BO_ 419 GEARBOX: 8 PCM
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
 
-BO_ 420 VSA_STATUS: 8 VSA
- SG_ USER_BRAKE : 7|16@0+ (0.015625,-1.609375) [0|1000] "" EON
- SG_ ESP_DISABLED : 28|1@0+ (1,0) [0|1] "" EON
- SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
- SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
-
 BO_ 422 SCM_BUTTONS: 8 SCM
  SG_ CRUISE_BUTTONS : 7|3@0+ (1,0) [0|7] "" EON
  SG_ LIGHTS_SETTING : 1|2@0+ (1,0) [0|3] "" EON
@@ -293,7 +313,7 @@ CM_ SG_ 780 CRUISE_SPEED "255 = no speed";
 CM_ SG_ 804 CRUISE_SPEED_PCM "255 = no speed";
 CM_ SG_ 829 BEEP "beeps are pleasant, chimes are for warnngs etc...";
 
-VAL_ 399 STEER_STATUS 5 "fault" 4 "no_torque_alert_2" 2 "no_torque_alert_1" 0 "normal" ;
+VAL_ 399 STEER_STATUS 6 "tmp_fault" 5 "fault_1" 4 "no_torque_alert_2" 3 "low_speed_lockout" 2 "no_torque_alert_1" 0 "normal" ;
 VAL_ 419 GEAR_SHIFTER 10 "S" 4 "D" 3 "N" 2 "R" 1 "P" ;
 VAL_ 422 CRUISE_BUTTONS 7 "tbd" 6 "tbd" 5 "tbd" 4 "accel_res" 3 "decel_set" 2 "cancel" 1 "main" 0 "none" ;
 VAL_ 422 LIGHTS_SETTING 3 "high_beam" 2 "low_beam" 1 "position" 0 "no_lights" ;

opendbc/acura_ilx_2016_nidec.dbc

@@ -182,4 +182,4 @@ BO_TX_BU_ 769 : NEO,ADAS;
 
 
 CM_ SG_ 1024 RADAR_STATE "need to find out more diagnostic values";
-VAL_ 1024 RADAR_STATE 121 "ok" 110 "faulted";
+VAL_ 1024 RADAR_STATE 121 "ok" 110 "faulted" 105 "wrong_config";

opendbc/acura_rdx_2018_can_generated.dbc

@@ -6,15 +6,15 @@ BO_ 512 GAS_COMMAND: 6 EON
  SG_ GAS_COMMAND : 7|16@0+ (0.253984064,-83.3) [0|1] "" INTERCEPTOR
  SG_ GAS_COMMAND2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" INTERCEPTOR
  SG_ ENABLE : 39|1@0+ (1,0) [0|1] "" INTERCEPTOR
- SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" INTERCEPTOR
- SG_ CHECKSUM : 43|4@0+ (1,0) [0|3] "" INTERCEPTOR
+ SG_ COUNTER_PEDAL : 35|4@0+ (1,0) [0|15] "" INTERCEPTOR
+ SG_ CHECKSUM_PEDAL : 47|8@0+ (1,0) [0|255] "" INTERCEPTOR
 
 BO_ 513 GAS_SENSOR: 6 INTERCEPTOR
  SG_ INTERCEPTOR_GAS : 7|16@0+ (0.253984064,-83.3) [0|1] "" EON
  SG_ INTERCEPTOR_GAS2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" EON
- SG_ STATE : 39|8@0+ (1,0) [0|255] "" EON
- SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" EON
- SG_ CHECKSUM : 43|4@0+ (1,0) [0|3] "" EON
+ SG_ STATE : 39|4@0+ (1,0) [0|15] "" EON
+ SG_ COUNTER_PEDAL : 35|4@0+ (1,0) [0|15] "" EON
+ SG_ CHECKSUM_PEDAL : 47|8@0+ (1,0) [0|255] "" EON
 
 VAL_ 513 STATE 5 "FAULT_TIMEOUT" 4 "FAULT_STARTUP" 3 "FAULT_SCE" 2 "FAULT_SEND" 1 "FAULT_BAD_CHECKSUM" 0 "NO_FAULT" ;
 
@@ -59,9 +59,9 @@ BO_ 344 ENGINE_DATA: 8 PCM
  SG_ XMISSION_SPEED : 7|16@0+ (0.01,0) [0|250] "kph" EON
  SG_ ENGINE_RPM : 23|16@0+ (1,0) [0|15000] "rpm" EON
  SG_ XMISSION_SPEED2 : 39|16@0+ (0.01,0) [0|250] "kph" EON
+ SG_ ODOMETER : 55|8@0+ (10,0) [0|2550] "m" XXX
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
- SG_ ODOMETER : 55|8@0+ (0.010588,0) [0|255] "km" XXX
 
 BO_ 380 POWERTRAIN_DATA: 8 PCM
  SG_ PEDAL_GAS : 7|8@0+ (1,0) [0|255] "" EON
@@ -76,6 +76,23 @@ BO_ 380 POWERTRAIN_DATA: 8 PCM
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
 
+BO_ 420 VSA_STATUS: 8 VSA
+ SG_ USER_BRAKE : 7|16@0+ (0.015625,-1.609375) [0|1000] "" EON
+ SG_ COMPUTER_BRAKING : 23|1@0+ (1,0) [0|1] "" EON
+ SG_ ESP_DISABLED : 28|1@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_HOLD_RELATED : 52|1@0+ (1,0) [0|1] "" XXX
+ SG_ BRAKE_HOLD_ACTIVE : 46|1@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_HOLD_ENABLED : 45|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 427 STEER_MOTOR_TORQUE: 3 EPS
+ SG_ CONFIG_VALID : 7|1@0+ (1,0) [0|1] "" EON
+ SG_ MOTOR_TORQUE : 1|10@0+ (1,0) [0|256] "" EON
+ SG_ OUTPUT_DISABLED : 22|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 21|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 19|4@0+ (1,0) [0|15] "" EON
+
 BO_ 432 STANDSTILL: 7 VSA
  SG_ CONTROLLED_STANDSTILL : 0|1@0+ (1,0) [0|1] "" EON
  SG_ WHEELS_MOVING : 12|1@0+ (1,0) [0|1] "" EON
@@ -98,21 +115,24 @@ BO_ 490 VEHICLE_DYNAMICS: 8 VSA
 
 BO_ 506 BRAKE_COMMAND: 8 ADAS
  SG_ COMPUTER_BRAKE : 7|10@0+ (1,0) [0|1] "" EBCM
- SG_ ZEROS_BOH : 13|5@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00 : 13|5@0+ (1,0) [0|1] "" EBCM
  SG_ BRAKE_PUMP_REQUEST : 8|1@0+ (1,0) [0|1] "" EBCM
- SG_ CRUISE_BOH2 : 23|3@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00_2 : 23|3@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_OVERRIDE : 20|1@0+ (1,0) [0|1] "" EBCM
- SG_ CRUISE_BOH3 : 19|1@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00_3 : 19|1@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_FAULT_CMD : 18|1@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_CANCEL_CMD : 17|1@0+ (1,0) [0|1] "" EBCM
  SG_ COMPUTER_BRAKE_REQUEST : 16|1@0+ (1,0) [0|1] "" EBCM
- SG_ SET_ME_0X80 : 31|8@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_1 : 31|1@0+ (1,0) [0|1] "" EBCM
+ SG_ AEB_REQ_1 : 29|1@0+ (1,0) [0|1] "" XXX
+ SG_ AEB_REQ_2 : 26|3@0+ (1,0) [0|7] "" XXX
  SG_ BRAKE_LIGHTS : 39|1@0+ (1,0) [0|1] "" EBCM
  SG_ CRUISE_STATES : 38|7@0+ (1,0) [0|1] "" EBCM
  SG_ CHIME : 47|3@0+ (1,0) [0|7] "" EBCM
- SG_ ZEROS_BOH6 : 44|1@0+ (1,0) [0|1] "" EBCM
+ SG_ SET_ME_X00_4 : 44|1@0+ (1,0) [0|1] "" EBCM
  SG_ FCW : 43|2@0+ (1,0) [0|3] "" EBCM
- SG_ ZEROS_BOH4 : 55|8@0+ (1,0) [0|0] "" EBCM
+ SG_ AEB_STATUS : 41|2@0+ (1,0) [0|3] "" XXX
+ SG_ SET_ME_X00_5 : 55|8@0+ (1,0) [0|0] "" EBCM
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EBCM
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EBCM
 
@@ -151,19 +171,22 @@ BO_ 780 ACC_HUD: 8 ADAS
  SG_ BOH : 38|1@0+ (1,0) [0|1] "" BDY
  SG_ ACC_PROBLEM : 37|1@0+ (1,0) [0|1] "" BDY
  SG_ FCM_OFF : 36|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH_2 : 35|1@0+ (1,0) [0|1] "" BDY
+ SG_ FCM_OFF_2 : 35|1@0+ (1,0) [0|1] "" BDY
  SG_ FCM_PROBLEM : 34|1@0+ (1,0) [0|1] "" BDY
  SG_ RADAR_OBSTRUCTED : 33|1@0+ (1,0) [0|1] "" BDY
  SG_ ENABLE_MINI_CAR : 32|1@0+ (1,0) [0|1] "" BDY
- SG_ SET_ME_X03 : 47|2@0+ (1,0) [0|3] "" BDY
+ SG_ HUD_DISTANCE : 47|2@0+ (1,0) [0|3] "" BDY
  SG_ HUD_LEAD : 45|2@0+ (1,0) [0|3] "" BDY
  SG_ BOH_3 : 43|1@0+ (1,0) [0|3] "" BDY
  SG_ BOH_4 : 42|1@0+ (1,0) [0|3] "" BDY
  SG_ BOH_5 : 41|1@0+ (1,0) [0|3] "" BDY
  SG_ CRUISE_CONTROL_LABEL : 40|1@0+ (1,0) [0|3] "" BDY
+ SG_ SET_ME_X01_2 : 55|1@0+ (1,0) [0|1] "" BDY
+ SG_ IMPERIAL_UNIT : 54|1@0+ (1,0) [0|1] "" BDY
  SG_ HUD_DISTANCE_3 : 52|1@0+ (1,0) [0|1] "" BDY
- SG_ SET_ME_X03_2 : 55|2@0+ (1,0) [0|3] "" BDY
+ SG_ CHIME : 51|3@0+ (1,0) [0|1] "" BDY
  SG_ SET_ME_X01 : 48|1@0+ (1,0) [0|1] "" BDY
+ SG_ ICONS : 63|2@0+ (1,0) [0|1] "" BDY
  SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" BDY
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" BDY
 
@@ -213,6 +236,7 @@ BO_ 1029 DOORS_STATUS: 8 BDY
  SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
 
 CM_ SG_ 490 LONG_ACCEL "wheel speed derivative, noisy and zero snapping";
+CM_ SG_ 506 AEB_REQ_1 "set for duration of suspected AEB event";
 CM_ SG_ 773 PASS_AIRBAG_ON "Might just be indicator light";
 CM_ SG_ 773 PASS_AIRBAG_OFF "Might just be indicator light";
 CM_ SG_ 780 CRUISE_SPEED "255 = no speed";
@@ -222,6 +246,7 @@ CM_ SG_ 829 BEEP "beeps are pleasant, chimes are for warnngs etc...";
 
 VAL_ 506 FCW 3 "fcw" 2 "fcw" 1 "fcw" 0 "no_fcw" ;
 VAL_ 506 CHIME 4 "double_chime" 3 "single_chime" 2 "continuous_chime" 1 "repeating_chime" 0 "no_chime" ;
+VAL_ 506 AEB_STATUS 3 "aeb_prepare" 2 "aeb_ready" 1 "aeb_braking" 0 "no_aeb" ;
 VAL_ 780 CRUISE_SPEED 255 "no_speed" 252 "stopped" ;
 VAL_ 780 HUD_LEAD 3 "acc_off" 2 "solid_car" 1 "dashed_car" 0 "no_car" ;
 VAL_ 829 BEEP 3 "single_beep" 2 "triple_beep" 1 "repeated_beep" 0 "no_beep" ;
@@ -243,8 +268,8 @@ BO_ 392 GEARBOX: 6 XXX
  SG_ GEAR : 36|5@0+ (1,0) [0|31] "" EON
 
 BO_ 399 STEER_STATUS: 6 EPS
- SG_ STEER_TORQUE_SENSOR : 7|12@0- (1,0) [-2047.5|2047.5] "tbd" EON
- SG_ STEER_TORQUE_MOTOR : 23|16@0- (1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_TORQUE_SENSOR : 7|12@0- (-1,0) [-2047.5|2047.5] "tbd" EON
+ SG_ STEER_ANGLE_RATE : 23|16@0- (-0.1,0) [-31000|31000] "deg/s" EON
  SG_ STEER_STATUS : 35|4@0+ (1,0) [0|15] "" EON
  SG_ STEER_CONTROL_ACTIVE : 36|1@0+ (1,0) [0|1] "" EON
  SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" EON
@@ -257,12 +282,6 @@ BO_ 404 STEERING_CONTROL: 4 EON
  SG_ COUNTER : 29|2@0+ (1,0) [0|15] "" EPS
  SG_ CHECKSUM : 27|4@0+ (1,0) [0|3] "" EPS
 
-BO_ 420 VSA_STATUS: 8 VSA
- SG_ USER_BRAKE : 7|16@0+ (0.015625,-1.609375) [0|1000] "" EON
- SG_ ESP_DISABLED : 28|1@0+ (1,0) [0|1] "" EON
- SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
- SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
-
 BO_ 422 SCM_BUTTONS: 8 SCM
  SG_ CRUISE_BUTTONS : 7|3@0+ (1,0) [0|7] "" EON
  SG_ LIGHTS_SETTING : 1|2@0+ (1,0) [0|3] "" EON
@@ -288,6 +307,7 @@ BO_ 660 SCM_FEEDBACK: 8 SCM
 CM_ SG_ 422 PARKING_BRAKE_LIGHT "Believe this is just the dash light for the parking break";
 VAL_ 392 GEAR_SHIFTER 0 "S" 1 "P" 2 "R" 4 "N" 8 "D" ;
 VAL_ 392 GEAR 26 "S" 4 "D" 3 "N" 2 "R" 1 "P" ;
+VAL_ 399 STEER_STATUS 6 "tmp_fault" 5 "fault_1" 4 "no_torque_alert_2" 3 "low_speed_lockout" 2 "no_torque_alert_1" 0 "normal" ;
 VAL_ 422 CRUISE_BUTTONS 7 "tbd" 6 "tbd" 5 "tbd" 4 "accel_res" 3 "decel_set" 2 "cancel" 1 "main" 0 "none" ;
 
 CM_ "CHFFR_METRIC 342 STEER_ANGLE STEER_ANGLE 0.36 180; CHFFR_METRIC 380 ENGINE_RPM ENGINE_RPM 1 0; CHFFR_METRIC 804 ENGINE_TEMPERATURE ENGINE_TEMPERATURE 1 0";

opendbc/cadillac_ct6_chassis.dbc

@@ -1,95 +0,0 @@
-VERSION ""
-
-
-NS_ :
-	NS_DESC_
-	CM_
-	BA_DEF_
-	BA_
-	VAL_
-	CAT_DEF_
-	CAT_
-	FILTER
-	BA_DEF_DEF_
-	EV_DATA_
-	ENVVAR_DATA_
-	SGTYPE_
-	SGTYPE_VAL_
-	BA_DEF_SGTYPE_
-	BA_SGTYPE_
-	SIG_TYPE_REF_
-	VAL_TABLE_
-	SIG_GROUP_
-	SIG_VALTYPE_
-	SIGTYPE_VALTYPE_
-	BO_TX_BU_
-	BA_DEF_REL_
-	BA_REL_
-	BA_DEF_DEF_REL_
-	BU_SG_REL_
-	BU_EV_REL_
-	BU_BO_REL_
-	SG_MUL_VAL_
-
-BS_:
-
-BU_: K182_PACM K43_PSCM K17_EBCM NEO K124_ASCM
-
-
-
-BO_ 823 PACMParkAssitCmd: 7 NEO
- SG_ RollingCounter : 35|2@0+ (1,0) [0|0] ""  NEO
- SG_ SteeringWheelChecksum : 47|16@0+ (1,0) [0|0] ""  NEO
- SG_ SteeringWheelCmd : 23|16@0+ (1,0) [0|0] ""  NEO
-
-BO_ 560 EBCMRegen: 6 K17_EBCM
- SG_ Regen : 1|10@0+ (1,0) [0|0] ""  NEO
-
-BO_ 338 ASCMLKASteeringCmd: 6 NEO
- SG_ LKASteeringCmdActive : 7|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmd : 5|14@0- (1,0) [0|0] ""  NEO
- SG_ RollingCounter : 23|2@0+ (1,0) [0|0] ""  NEO
- SG_ SetMe1 : 21|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASVehicleSpeed : 20|13@0+ (0.22,0) [0|0] "kph" NEO
- SG_ LKASMode : 36|2@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmdChecksum : 33|10@0+ (1,0) [0|0] ""  NEO
-
-BO_ 340 ASCMBLKASteeringCmd: 6 NEO
- SG_ LKASteeringCmdActive : 7|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmd : 5|14@0- (1,0) [0|0] ""  NEO
- SG_ RollingCounter : 23|2@0+ (1,0) [0|0] ""  NEO
- SG_ SetMe1 : 21|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASVehicleSpeed : 20|13@0+ (0.22,0) [0|0] "kph" NEO
- SG_ LKASteeringCmdActive2 : 35|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmdChecksum : 33|10@0+ (1,0) [0|0] ""  NEO
-
-BO_ 368 EBCMFrictionBrakeStatus: 8 K17_EBCM
- SG_ FrictionBrakePressure : 23|16@0+ (1,0) [0|0] ""  NEO
-
-BO_ 789 EBCMFrictionBrakeCmd: 5 K17_EBCM
- SG_ RollingCounter : 37|6@0+ (1,0) [0|0] ""  NEO
- SG_ FrictionBrakeMode : 7|4@0+ (1,0) [0|0] ""  NEO
- SG_ FrictionBrakeChecksum : 23|16@0+ (1,0) [0|0] ""  NEO
- SG_ FrictionBrakeCmd : 3|12@0- (1,0) [0|0] ""  NEO
-
-BO_TX_BU_ 823 : K43_PSCM,NEO;
-BO_TX_BU_ 789 : NEO,K17_EBCM;
-
-
-CM_ BU_ K182_PACM "Parking Assist Control Module";
-CM_ BU_ K43_PSCM "Power Steering Control Module";
-CM_ BU_ K17_EBCM "Electronic Brake Control Module";
-CM_ BU_ NEO "Comma NEO";
-CM_ BU_ K124_ASCM "Active Safety Control Module";
-BA_DEF_  "UseGMParameterIDs" INT 0 0;
-BA_DEF_  "ProtocolType" STRING ;
-BA_DEF_  "BusType" STRING ;
-BA_DEF_DEF_  "UseGMParameterIDs" 1;
-BA_DEF_DEF_  "ProtocolType" "GMLAN";
-BA_DEF_DEF_  "BusType" "";
-BA_ "UseGMParameterIDs" 0;
-BA_ "BusType" "CAN";
-BA_ "ProtocolType" "GMLAN";
-
-VAL_ 338 LKASteeringCmdActive 1 "Active" 0 "Inactive" ;
-VAL_ 338 LKASMode 2 "supercruise" 1 "lkas" 0 "Inactive" ;

opendbc/cadillac_ct6_powertrain.dbc

@@ -1,244 +0,0 @@
-VERSION ""
-
-
-NS_ :
-	NS_DESC_
-	CM_
-	BA_DEF_
-	BA_
-	VAL_
-	CAT_DEF_
-	CAT_
-	FILTER
-	BA_DEF_DEF_
-	EV_DATA_
-	ENVVAR_DATA_
-	SGTYPE_
-	SGTYPE_VAL_
-	BA_DEF_SGTYPE_
-	BA_SGTYPE_
-	SIG_TYPE_REF_
-	VAL_TABLE_
-	SIG_GROUP_
-	SIG_VALTYPE_
-	SIGTYPE_VALTYPE_
-	BO_TX_BU_
-	BA_DEF_REL_
-	BA_REL_
-	BA_DEF_DEF_REL_
-	BU_SG_REL_
-	BU_EV_REL_
-	BU_BO_REL_
-	SG_MUL_VAL_
-
-BS_:
-
-BU_: K16_BECM K73_TCIC K9_BCM K43_PSCM K17_EBCM K20_ECM K114B_HPCM NEO K124_ASCM
-VAL_TABLE_ TurnSignals 2 "Right Turn" 1 "Left Turn" 0 "None" ;
-VAL_TABLE_ ACCLeadCar 1 "Present" 0 "Not Present" ;
-VAL_TABLE_ ACCCmdActive 1 "Active" 0 "Inactive" ;
-VAL_TABLE_ BrakePedalPressed 1 "Pressed" 0 "Depressed" ;
-VAL_TABLE_ DistanceButton 1 "Active" 0 "Inactive" ;
-VAL_TABLE_ LKAButton 1 "Active" 0 "Inactive" ;
-VAL_TABLE_ ACCButtons 6 "Cancel" 5 "Main" 3 "Set" 2 "Resume" 1 "None" ;
-VAL_TABLE_ PRNDL 3 "Reverse" 2 "Drive" 1 "Neutral" 0 "Park" ;
-VAL_TABLE_ DoorStatus 1 "Opened" 0 "Closed" ;
-VAL_TABLE_ SeatBeltStatus 1 "Latched" 0 "Unlatched" ;
-VAL_TABLE_ LKASteeringCmdActive 1 "Active" 0 "Inactive" ;
-VAL_TABLE_ ACCGapLevel 3 "Far" 2 "Med" 1 "Near" 0 "Inactive" ;
-VAL_TABLE_ GasRegenCmdActiveInv 1 "Inactive" 0 "Active" ;
-VAL_TABLE_ GasRegenCmdActive 1 "Active" 0 "Inactive" ;
-VAL_TABLE_ LKATorqueDeliveredStatus 3 "Failed" 2 "Temp. Limited" 1 "Active" 0 "Inactive" ;
-VAL_TABLE_ HandsOffSWDetectionStatus 1 "Hands On" 0 "Hands Off" ;
-VAL_TABLE_ HandsOffSWDetectionMode 2 "Failed" 1 "Enabled" 0 "Disabled" ;
-
-
-BO_ 717 ASCM_2CD: 5 K124_ASCM
-
-BO_ 869 ASCM_365: 4 K124_ASCM
-
-BO_ 1034 ASCM_40A: 7 K124_ASCM
-
-BO_ 1296 ASCM_510: 4 K124_ASCM
-
-BO_ 1930 ASCM_78A: 7 K124_ASCM
-
-BO_ 190 ECMAcceleratorPos: 6 K20_ECM
- SG_ BrakePedalPos : 15|8@0+ (1,0) [0|0] "sticky"  NEO
- SG_ GasPedalAndAcc : 23|8@0+ (1,0) [0|0] ""  NEO
-
-BO_ 201 ECMEngineStatus: 8 K20_ECM
- SG_ EngineTPS : 39|8@0+ (0.392156863,0) [0|100.000000065] "%"  NEO
- SG_ EngineRPM : 15|16@0+ (0.25,0) [0|0] "RPM"  NEO
-
-BO_ 209 EBCMBrakePedalTorque: 7 K17_EBCM
- SG_ BrakePedalTorque : 3|12@0+ (1,0) [0|0] ""  NEO
-
-BO_ 241 EBCMBrakePedalPosition: 6 K17_EBCM
- SG_ BrakePedalPosition : 15|8@0+ (1,0) [0|255] ""  NEO
-
-BO_ 298 BCMDoorBeltStatus: 8 K9_BCM
- SG_ RearLeftDoor : 8|1@0+ (1,0) [0|0] ""  NEO
- SG_ FrontLeftDoor : 9|1@0+ (1,0) [0|0] ""  NEO
- SG_ FrontRightDoor : 10|1@0+ (1,0) [0|0] ""  NEO
- SG_ RearRightDoor : 23|1@0+ (1,0) [0|0] ""  NEO
- SG_ LeftSeatBelt : 12|1@0+ (1,0) [0|0] ""  NEO
- SG_ RightSeatBelt : 53|1@0+ (1,0) [0|0] ""  NEO
-
-BO_ 309 ECMPRDNL: 8 K20_ECM
- SG_ PRNDL : 2|3@0+ (1,0) [0|0] ""  NEO
-
-BO_ 320 BCMTurnSignals: 3 K9_BCM
- SG_ TurnSignals : 19|2@0+ (1,0) [0|0] ""  NEO
-
-BO_ 336 ASCMLKASStatus: 1 NEO
- SG_ Available : 7|1@0+ (1,0) [0|0] ""  NEO
-
-BO_ 338 ASCMLKASteeringCmd: 6 NEO
- SG_ LKASteeringCmdActive : 7|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmd : 5|14@0- (1,0) [0|0] ""  NEO
- SG_ RollingCounter : 23|2@0+ (1,0) [0|0] ""  NEO
- SG_ SetMe1 : 21|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASVehicleSpeed : 20|13@0+ (0.22,0) [0|0] "kph" NEO
- SG_ LKASMode : 36|2@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmdChecksum : 33|10@0+ (1,0) [0|0] ""  NEO
-
-BO_ 340 ASCMBLKASteeringCmd: 6 NEO
- SG_ LKASteeringCmdActive : 7|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmd : 5|14@0- (1,0) [0|0] ""  NEO
- SG_ RollingCounter : 23|2@0+ (1,0) [0|0] ""  NEO
- SG_ SetMe1 : 21|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKASVehicleSpeed : 20|13@0+ (0.22,0) [0|0] "kph" NEO
- SG_ LKASMode : 36|2@0+ (1,0) [0|0] ""  NEO
- SG_ LKASteeringCmdChecksum : 33|10@0+ (1,0) [0|0] ""  NEO
-
-BO_ 381 MSG_17D: 8 K20_ECM
- SG_ MSG17D_AccPower : 35|12@0- (1,0) [0|0] ""  NEO
-
-BO_ 356 PSCMStatus: 8 K43_PSCM
- SG_ LKATorqueDeliveredStatus : 7|3@0+ (1,0) [0|7] ""  NEO
- SG_ LKADriverAppldTrq : 2|11@0- (0.01,0) [-10.24|10.23] "Nm"  NEO
- SG_ LKATBDTorque : 21|14@0- (-0.005,0) [-10.24|10.23] "Nm"  NEO
- SG_ RollingCounter : 39|2@0+ (1,0) [0|0] ""  NEO
- SG_ LKATotalTorqueDelivered : 37|14@0- (0.01,0) [-10.24|10.23] "Nm"  NEO
-
-BO_ 417 AcceleratorPedal: 7 XXX
- SG_ AcceleratorPedal : 55|8@0+ (1,0) [0|0] ""  NEO
-
-BO_ 451 GasAndAcc: 8 XXX
- SG_ GasPedalAndAcc2 : 55|8@0+ (1,0) [0|0] ""  NEO
-
-BO_ 452 AcceleratorPedal2: 8 XXX
- SG_ AcceleratorPedal2 : 47|8@0+ (1,0) [0|0] ""  NEO
-
-BO_ 481 ASCMSteeringButton: 7 K124_ASCM
- SG_ DistanceButton : 22|1@0+ (1,0) [0|0] ""  NEO
- SG_ LKAButton : 23|1@0+ (1,0) [0|0] ""  NEO
- SG_ ACCButtons : 46|3@0+ (1,0) [0|0] ""  NEO
-
-BO_ 485 PSCMSteeringAngle: 8 K43_PSCM
- SG_ SteeringWheelAngle : 15|16@0- (0.0625,0) [-540|540] "deg"  NEO
- SG_ SteeringWheelRate : 27|12@0- (0.5,0) [-100|100] "deg/s"  NEO
-
-BO_ 489 EBCMVehicleDynamic: 8 K17_EBCM
- SG_ YawRate : 51|12@0- (0.0625,0) [-2047|2047] "grad/s"  NEO
- SG_ LateralAcceleration : 3|12@0- (0.0161,0) [-2047|2047] "m/s2"  NEO
- SG_ BrakePedalPressed : 6|1@0+ (1,0) [0|0] ""  NEO
-
-BO_ 711 BECMBatteryVoltageCurrent: 6 K17_EBCM
- SG_ HVBatteryVoltage : 31|12@0+ (0.125,0) [0|511.875] "V"  NEO
- SG_ HVBatteryCurrent : 12|13@0- (0.15,0) [-614.4|614.25] "A"  NEO
-
-BO_ 715 ASCMGasRegenCmd: 8 K124_ASCM
- SG_ GasRegenAlwaysOne : 9|1@0+ (1,1) [1|1] ""  NEO
- SG_ GasRegenAlwaysThree : 15|2@0+ (1,1) [1|1] ""  NEO
- SG_ GasRegenChecksum : 47|24@0+ (1,0) [0|0] ""  NEO
- SG_ GasRegenCmdActiveInv : 32|1@0+ (1,0) [0|0] ""  NEO
- SG_ GasRegenFullStopActive : 13|1@0+ (1,0) [0|0] ""  NEO
- SG_ GasRegenCmdActive : 0|1@0+ (1,0) [0|0] ""  NEO
- SG_ RollingCounter : 7|2@0+ (1,0) [0|0] ""  NEO
- SG_ RollingCounter2 : 36|4@0+ (1,0) [0|0] ""  NEO
- SG_ GasRegenAlwaysOne2 : 23|1@0+ (1,0) [0|1] ""  NEO
- SG_ GasRegenCmd : 22|15@0+ (1,0) [0|0] ""  NEO
-
-BO_ 810 TCICOnStarGPSPosition: 8 K73_TCIC
- SG_ GPSLongitude : 39|32@0+ (1,0) [0|0] "milliarcsecond"  NEO
- SG_ GPSLatitude : 7|32@0+ (1,0) [0|0] "milliarcsecond"  NEO
-
-BO_ 840 EBCMWheelSpdFront: 4 K17_EBCM
- SG_ FLWheelSpd : 7|16@0+ (0.0311,0) [0|255] "km/h"  NEO
- SG_ FRWheelSpd : 23|16@0+ (0.0311,0) [0|255] "km/h"  NEO
-
-BO_ 842 EBCMWheelSpdRear: 4 K17_EBCM
- SG_ RLWheelSpd : 7|16@0+ (0.0311,0) [0|255] "km/h"  NEO
- SG_ RRWheelSpd : 23|16@0+ (0.0311,0) [0|255] "km/h"  NEO
-
-BO_ 880 ASCMActiveCruiseControlStatus: 6 K124_ASCM
- SG_ ACCLeadCar : 44|1@0+ (1,0) [0|0] "" Vector__XXX
- SG_ ACCAlwaysOne2 : 32|1@0+ (1,0) [0|0] "" Vector__XXX
- SG_ ACCAlwaysOne : 0|1@0+ (1,0) [0|0] "" Vector__XXX
- SG_ ACCSpeedSetpoint : 19|12@0+ (1,0) [0|0] "km/h"  NEO
- SG_ ACCGapLevel : 21|2@0+ (1,0) [0|0] ""  NEO
- SG_ ACCResumeButton : 1|1@0+ (1,0) [0|0] ""  NEO
- SG_ ACCCmdActive : 23|1@0+ (1,0) [0|0] ""  NEO
-
-BO_ 1001 ECMVehicleSpeed: 8 K20_ECM
- SG_ VehicleSpeed : 7|16@0+ (0.01,0) [0|0] "mph"  NEO
-
-BO_ 1033 ASCMKeepAlive: 7 NEO
- SG_ ASCMKeepAliveAllZero : 7|56@0+ (1,0) [0|0] ""  NEO
-
-BO_ 1217 ECMEngineCoolantTemp: 8 K20_ECM
- SG_ EngineCoolantTemp : 23|8@0+ (1,-40) [0|0] "°C"  NEO
-
-BO_ 1249 VIN_Part2: 8 K20_ECM
- SG_ VINPart2 : 7|64@0+ (1,0) [0|0] ""  NEO
-
-BO_ 1300 VIN_Part1: 8 K20_ECM
- SG_ VINPart1 : 7|64@0+ (1,0) [0|0] ""  NEO
-
-BO_ 1912 PSCM_778: 8 K43_PSCM
-
-BO_TX_BU_ 338 : K124_ASCM,NEO;
-BO_TX_BU_ 880 : NEO,K124_ASCM;
-BO_TX_BU_ 1033 : K124_ASCM,NEO;
-BO_TX_BU_ 715 : NEO,K124_ASCM;
-
-
-CM_ BU_ K16_BECM "Battery Energy Control Module";
-CM_ BU_ K73_TCIC "Telematics Communication Control Module";
-CM_ BU_ K9_BCM "Body Control Module";
-CM_ BU_ K43_PSCM "Power Steering Control Module";
-CM_ BU_ K17_EBCM "Electronic Brake Control Module";
-CM_ BU_ K20_ECM "Engine Control Module";
-CM_ BU_ K114B_HPCM "Hybrid Powertrain Control Module";
-CM_ BU_ NEO "Comma NEO";
-CM_ BU_ K124_ASCM "Active Safety Control Module";
-CM_ SG_ 381 MSG17D_AccPower "Need to investigate";
-CM_ SG_ 190 GasPedalAndAcc "ACC baseline is 62";
-CM_ SG_ 451 GasPedalAndAcc2 "ACC baseline is 62";
-CM_ SG_ 715 RollingCounter2 "Values cycle between 0, 7, 10, 13";
-BA_DEF_  "UseGMParameterIDs" INT 0 0;
-BA_DEF_  "ProtocolType" STRING ;
-BA_DEF_  "BusType" STRING ;
-BA_DEF_DEF_  "UseGMParameterIDs" 1;
-BA_DEF_DEF_  "ProtocolType" "GMLAN";
-BA_DEF_DEF_  "BusType" "";
-BA_ "BusType" "CAN";
-BA_ "ProtocolType" "GMLAN";
-BA_ "UseGMParameterIDs" 0;
-VAL_ 481 DistanceButton 1 "Active" 0 "Inactive" ;
-VAL_ 481 LKAButton 1 "Active" 0 "Inactive" ;
-VAL_ 481 ACCButtons 6 "Cancel" 5 "Main" 3 "Set" 2 "Resume" 1 "None" ;
-VAL_ 309 PRNDL 3 "Reverse" 2 "Drive" 1 "Neutral" 0 "Park" ;
-VAL_ 338 LKASteeringCmdActive 1 "Active" 0 "Inactive" ;
-VAL_ 338 LKASMode 2 "supercruise" 1 "lkas" 0 "Inactive" ;
-VAL_ 880 ACCLeadCar 1 "Present" 0 "Not Present" ;
-VAL_ 880 ACCGapLevel 3 "Far" 2 "Med" 1 "Near" 0 "Inactive" ;
-VAL_ 880 ACCResumeButton 1 "Pressed" 0 "Depressed" ;
-VAL_ 880 ACCCmdActive 1 "Active" 0 "Inactive" ;
-VAL_ 356 LKATorqueDeliveredStatus 7 "Override Fault" 6 "LKAS Fault but Responsive" 5 "TBD but Responsive" 4 "TBD but Responsive" 3 "Fault" 1 "Active" 0 "Inactive" ;
-VAL_ 489 BrakePedalPressed 1 "Pressed" 0 "Depressed" ;
-VAL_ 715 GasRegenCmdActiveInv 1 "Inactive" 0 "Active" ;
-VAL_ 715 GasRegenCmdActive 1 "Active" 0 "Inactive" ;
-

opendbc/can/SConscript

@@ -0,0 +1,23 @@
+Import('env', 'cereal')
+
+import os
+from opendbc.can.process_dbc import process
+
+dbcs = []
+for x in sorted(os.listdir('../')):
+  if x.endswith(".dbc"):
+    def compile_dbc(target, source, env):
+      process(source[0].path, target[0].path)
+    in_fn = [os.path.join('../', x), 'dbc_template.cc']
+    out_fn = os.path.join('dbc_out', x.replace(".dbc", ".cc"))
+    dbc = env.Command(out_fn, in_fn, compile_dbc)
+    dbcs.append(dbc)
+
+
+libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
+
+# Build packer and parser
+
+env.Command(['packer_pyx.so', 'parser_pyx.so'],
+  [libdbc, cereal, 'common_pyx_setup.py', 'packer_pyx.pyx', 'parser_pyx.pyx', 'common.pxd'],
+  "cd opendbc/can && python3 common_pyx_setup.py build_ext --inplace")