openpilot v0.8.3 release

Co-authored-by: robbederks <robbe.derks@gmail.com>
Co-authored-by: Willem Melching <willem.melching@gmail.com>

.gitignore

@@ -1,4 +1,5 @@
 venv/
+.clang-format
 .DS_Store
 .tags
 .ipynb_checkpoints
@@ -6,7 +7,7 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
-.vscode
+.vscode*
 model2.png
 a.out
 
@@ -31,6 +32,7 @@ a.out
 *.vcd
 config.json
 clcache
+compile_commands.json
 
 persist
 board/obj/
@@ -42,9 +44,11 @@ selfdrive/ui/_ui
 selfdrive/test/longitudinal_maneuvers/out
 selfdrive/visiond/visiond
 selfdrive/loggerd/loggerd
+selfdrive/loggerd/bootlog
 selfdrive/sensord/_gpsd
 selfdrive/sensord/_sensord
 selfdrive/camerad/camerad
+selfdrive/camerad/test/ae_gray_test
 selfdrive/modeld/_modeld
 selfdrive/modeld/_dmonitoringmodeld
 /src/
@@ -53,9 +57,8 @@ one
 openpilot
 notebooks
 xx
+hyperthneed
 panda_jungle
-apks
-openpilot-apks
 
 .coverage*
 coverage.xml
@@ -66,3 +69,7 @@ pandaextra
 flycheck_*
 
 cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/thneed/compile
+models/*.thneed

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Our software is open source so you can solve your own problems without needing help from others. And if you solve a problem and are so kind, you can upstream it for the rest of the world to use.
 
-Most open source development activity is coordinated through our [Discord](https://discord.comma.ai). A lot of documentation is available on our [medium](https://medium.com/@comma_ai/).
+Most open source development activity is coordinated through our [GitHub Discussions](https://github.com/commaai/openpilot/discussions) and [Discord](https://discord.comma.ai). A lot of documentation is available on our [medium](https://medium.com/@comma_ai/).
 
 ## Getting Started
 
@@ -12,23 +12,19 @@ Most open source development activity is coordinated through our [Discord](https
 
 ## Testing
 
-### Local Testing
-
-You can test your changes on your machine by running `run_docker_tests.sh`. This will run some automated tests in docker against your code.
-
 ### Automated Testing
 
-All PRs and commits are automatically checked by Github Actions. Check out `.github/workflows/` for what Github Actions runs. Any new tests sould be added to Github Actions.
+All PRs and commits are automatically checked by GitHub Actions. Check out `.github/workflows/` for what GitHub Actions runs. Any new tests should be added to GitHub Actions.
 
 ### Code Style and Linting
 
-Code is automatically checked for style by Github Actions as part of the automated tests. You can also run these tests yourself by running `pre-commit run --all`.
+Code is automatically checked for style by GitHub Actions as part of the automated tests. You can also run these tests yourself by running `pre-commit run --all`.
 
 ## Car Ports (openpilot)
 
 We've released a [Model Port guide](https://medium.com/@comma_ai/openpilot-port-guide-for-toyota-models-e5467f4b5fe6) for porting to Toyota/Lexus models.
 
-If you port openpilot to a substantially new car brand, see this more generic [Brand Port guide](https://medium.com/@comma_ai/how-to-write-a-car-port-for-openpilot-7ce0785eda84). You might also be eligible for a bounty. See our bounties at [comma.ai/bounties.html](https://comma.ai/bounties.html)
+If you port openpilot to a substantially new car brand, see this more generic [Brand Port guide](https://medium.com/@comma_ai/how-to-write-a-car-port-for-openpilot-7ce0785eda84).
 
 ## Pull Requests
 
@@ -38,12 +34,10 @@ git clone https://github.com/commaai/openpilot.git --recursive
 ```
 Or alternatively, when on the master branch:
 ```
-git submodule init
-git submodule update
+git submodule update --init
 ```
 The reasons for having submodules on a dedicated repository and our new development philosophy can be found in our [post about externalization](https://medium.com/@comma_ai/a-2020-theme-externalization-13b33326d8b3).
 Modules that are in seperate repositories include:
-* apks
 * cereal
 * laika
 * opendbc

Jenkinsfile

@@ -1,89 +1,213 @@
-pipeline {
-  agent {
-    docker {
-      image 'python:3.7.3'
-      args '--user=root'
+def phone(String ip, String step_label, String cmd) {
+  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
+    def ssh_cmd = """
+ssh -tt -o StrictHostKeyChecking=no -i ${key_file} -p 8022 'comma@${ip}' /usr/bin/bash <<'EOF'
+
+set -e
+
+export CI=1
+export TEST_DIR=${env.TEST_DIR}
+export GIT_BRANCH=${env.GIT_BRANCH}
+export GIT_COMMIT=${env.GIT_COMMIT}
+
+source ~/.bash_profile
+if [ -f /TICI ]; then
+  source /etc/profile
+fi
+
+ln -snf ${env.TEST_DIR} /data/pythonpath
+
+if [ -f /EON ]; then
+  echo \$\$ > /dev/cpuset/app/tasks || true
+  echo \$PPID > /dev/cpuset/app/tasks || true
+  mkdir -p /dev/shm
+  chmod 777 /dev/shm
+fi
+
+cd ${env.TEST_DIR} || true
+${cmd}
+exit 0
+
+EOF"""
+
+    sh script: ssh_cmd, label: step_label
+  }
+}
+
+def phone_steps(String device_type, steps) {
+  lock(resource: "", label: device_type, inversePrecedence: true, variable: 'device_ip', quantity: 1) {
+    timeout(time: 60, unit: 'MINUTES') {
+      phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),)
+      steps.each { item ->
+        phone(device_ip, item[0], item[1])
+      }
     }
   }
+}
+
+pipeline {
+  agent none
   environment {
     COMMA_JWT = credentials('athena-test-jwt')
+    TEST_DIR = "/data/openpilot"
+  }
+  options {
+      timeout(time: 1, unit: 'HOURS')
   }
 
   stages {
 
-    stage('Release Build') {
+    stage('Build release2') {
+      agent {
+        docker {
+          image 'python:3.7.3'
+          args '--user=root'
+        }
+      }
       when {
         branch 'devel-staging'
       }
       steps {
-        lock(resource: "", label: 'eon-build', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
-          timeout(time: 60, unit: 'MINUTES') {
-            dir(path: 'selfdrive/test') {
-              sh 'pip install paramiko'
-              sh 'python phone_ci.py "cd release && PUSH=1 ./build_release2.sh"'
-            }
-          }
-        }
+        phone_steps("eon-build", [
+          ["build release2-staging and dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"],
+        ])
       }
     }
 
-    stage('On-device Tests') {
+    stage('openpilot tests') {
       when {
         not {
           anyOf {
-            branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging'
+            branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging'; branch 'testing-closet*'
           }
         }
       }
 
-      parallel {
 
-        stage('Build') {
-          environment {
-            CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
+      stages {
+
+        /*
+        stage('PC tests') {
+          agent {
+            dockerfile {
+              filename 'Dockerfile.openpilotci'
+              args '--privileged --shm-size=1G --user=root'
+            }
           }
-
-          steps {
-            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
-              timeout(time: 60, unit: 'MINUTES') {
-                dir(path: 'selfdrive/test') {
-                  sh 'pip install paramiko'
-                  sh 'python phone_ci.py "cd release && ./build_devel.sh"'
-                }
+          stages {
+            stage('Build') {
+              steps {
+                sh 'scons -j$(nproc)'
               }
             }
           }
+          post {
+            always {
+              // fix permissions since docker runs as another user
+              sh "chmod -R 777 ."
+            }
+          }
         }
+        */
 
-        stage('Replay Tests') {
-          steps {
-            lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
-              timeout(time: 60, unit: 'MINUTES') {
-                dir(path: 'selfdrive/test') {
-                  sh 'pip install paramiko'
-                  sh 'python phone_ci.py "cd selfdrive/test/process_replay && ./camera_replay.py"'
+        stage('On-device Tests') {
+          agent {
+            docker {
+              image 'python:3.7.3'
+              args '--user=root'
+            }
+          }
+
+          stages {
+            stage('parallel tests') {
+              parallel {
+                stage('Devel Build') {
+                  environment {
+                    CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ' '}"
+                  }
+                  steps {
+                    phone_steps("eon-build", [
+                      ["build", "SCONS_CACHE=1 scons -j4"],
+                      ["test athena", "nosetests -s selfdrive/athena/tests/test_athenad_old.py"],
+                      ["test manager", "python selfdrive/manager/test/test_manager.py"],
+                      ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
+                      ["build devel", "cd release && CI_PUSH=${env.CI_PUSH} ./build_devel.sh"],
+                      ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
+                    ])
+                  }
                 }
+
+                stage('Replay Tests') {
+                  steps {
+                    phone_steps("eon2", [
+                      ["build QCOM_REPLAY", "SCONS_CACHE=1 QCOM_REPLAY=1 scons -j4"],
+                      ["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"],
+                    ])
+                  }
+                }
+
+                stage('HW + Unit Tests') {
+                  steps {
+                    phone_steps("eon", [
+                      ["build", "SCONS_CACHE=1 scons -j4"],
+                      ["test sounds", "nosetests -s selfdrive/test/test_sounds.py"],
+                      ["test boardd loopback", "nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"],
+                      ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
+                      ["test encoder", "python selfdrive/loggerd/tests/test_encoder.py"],
+                      ["test logcatd", "python selfdrive/logcatd/tests/test_logcatd_android.py"],
+                      //["test updater", "python installer/updater/test_updater.py"],
+                    ])
+                  }
+                }
+
+                stage('Tici Build') {
+                  environment {
+                    R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"
+                  }
+                  steps {
+                    phone_steps("tici", [
+                      ["build", "SCONS_CACHE=1 scons -j16"],
+                      ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
+                      ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python selfdrive/loggerd/tests/test_encoder.py"],
+                      //["build release3-staging", "cd release && PUSH=${env.R3_PUSH} ./build_release3.sh"],
+                    ])
+                  }
+                }
+
+                stage('camerad') {
+                  steps {
+                    phone_steps("eon-party", [
+                      ["build", "SCONS_CACHE=1 scons -j16"],
+                      ["test camerad", "python selfdrive/camerad/test/test_camerad.py"],
+                      ["test exposure", "python selfdrive/camerad/test/test_exposure.py"],
+                    ])
+                  }
+                }
+
+                stage('Tici camerad') {
+                  steps {
+                    phone_steps("tici-party", [
+                      ["build", "SCONS_CACHE=1 scons -j16"],
+                      ["test camerad", "python selfdrive/camerad/test/test_camerad.py"],
+                      ["test exposure", "python selfdrive/camerad/test/test_exposure.py"],
+                    ])
+                  }
+                }
+
               }
             }
           }
-        }
 
-        stage('HW Tests') {
-          steps {
-            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
-              timeout(time: 60, unit: 'MINUTES') {
-                dir(path: 'selfdrive/test') {
-                  sh 'pip install paramiko'
-                  sh 'python phone_ci.py "SCONS_CACHE=1 scons -j3 cereal/ && \
-                                          nosetests -s selfdrive/test/test_sounds.py && \
-                                          nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"'
-                }
-              }
+          post {
+            always {
+              cleanWs()
             }
           }
+
         }
 
       }
     }
   }
 }
+

README.md

@@ -64,58 +64,62 @@ Supported Cars
 
 | Make      | Model (US Market Reference)   | Supported Package | ACC              | No ACC accel below | No ALC below      |
 | ----------| ------------------------------| ------------------| -----------------| -------------------| ------------------|
-| Acura     | ILX 2016-18                   | AcuraWatch Plus   | openpilot        | 25mph<sup>1</sup>  | 25mph             |
+| Acura     | ILX 2016-19                   | AcuraWatch Plus   | openpilot        | 25mph<sup>1</sup>  | 25mph             |
 | Acura     | RDX 2016-18                   | AcuraWatch Plus   | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Honda     | Accord 2018-19                | All               | Stock            | 0mph               | 3mph              |
+| Acura     | RDX 2020-21                   | All               | Stock            | 0mph               | 3mph              |
+| Honda     | Accord 2018-20                | All               | Stock            | 0mph               | 3mph              |
 | Honda     | Accord Hybrid 2018-20         | All               | Stock            | 0mph               | 3mph              |
-| Honda     | Civic Hatchback 2017-19       | Honda Sensing     | Stock            | 0mph               | 12mph             |
+| Honda     | Civic Hatchback 2017-21       | Honda Sensing     | Stock            | 0mph               | 12mph             |
 | Honda     | Civic Sedan/Coupe 2016-18     | Honda Sensing     | openpilot        | 0mph               | 12mph             |
-| Honda     | Civic Sedan/Coupe 2019-20     | Honda Sensing     | Stock            | 0mph               | 2mph<sup>2</sup>  |
+| Honda     | Civic Sedan/Coupe 2019-20     | All               | Stock            | 0mph               | 2mph<sup>2</sup>  |
 | Honda     | CR-V 2015-16                  | Touring           | openpilot        | 25mph<sup>1</sup>  | 12mph             |
 | Honda     | CR-V 2017-20                  | Honda Sensing     | Stock            | 0mph               | 12mph             |
 | Honda     | CR-V Hybrid 2017-2019         | Honda Sensing     | Stock            | 0mph               | 12mph             |
 | Honda     | Fit 2018-19                   | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Honda     | HR-V 2019                     | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Honda     | Insight 2019                  | Honda Sensing     | Stock            | 0mph               | 3mph              |
+| Honda     | HR-V 2019-20                  | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Insight 2019-21               | All               | Stock            | 0mph               | 3mph              |
+| Honda     | Inspire 2018                  | All               | Stock            | 0mph               | 3mph              |
 | Honda     | Odyssey 2018-20               | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 0mph              |
 | Honda     | Passport 2019                 | All               | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Honda     | Pilot 2016-18                 | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Honda     | Pilot 2019                    | All               | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Honda     | Ridgeline 2017-20             | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
-| Hyundai   | Sonata 2020                   | All               | Stock            | 0mph               | 0mph              |
-| Lexus     | CT Hybrid 2017-18             | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Lexus     | ES 2019                       | All               | openpilot        | 0mph               | 0mph              |
+| Honda     | Pilot 2016-19                 | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Honda     | Ridgeline 2017-21             | Honda Sensing     | openpilot        | 25mph<sup>1</sup>  | 12mph             |
+| Hyundai   | Palisade 2020-21              | All               | Stock            | 0mph               | 0mph              |
+| Hyundai   | Sonata 2020-21                | All               | Stock            | 0mph               | 0mph              |
+| Lexus     | CT Hybrid 2017-18             | LSS               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Lexus     | ES 2019-20                    | All               | openpilot        | 0mph               | 0mph              |
+| Lexus     | ES Hybrid 2018                | LSS               | Stock<sup>3</sup>| 0mph               | 0mph              |
 | Lexus     | ES Hybrid 2019                | All               | openpilot        | 0mph               | 0mph              |
 | Lexus     | IS 2017-2019                  | All               | Stock            | 22mph              | 0mph              |
 | Lexus     | IS Hybrid 2017                | All               | Stock            | 0mph               | 0mph              |
+| Lexus     | NX 2018                       | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
 | Lexus     | NX Hybrid 2018                | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Lexus     | RX 2016-17                    | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Lexus     | RX 2020                       | All               | openpilot        | 0mph               | 0mph              |
+| Lexus     | RX 2016-18                    | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Lexus     | RX 2020-21                    | All               | openpilot        | 0mph               | 0mph              |
 | Lexus     | RX Hybrid 2016-19             | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
 | Lexus     | RX Hybrid 2020                | All               | openpilot        | 0mph               | 0mph              |
-| Toyota    | Avalon 2016                   | TSS-P             | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
-| Toyota    | Avalon 2017-18                | All               | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Avalon 2016-18, 2020-21       | TSS-P             | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
 | Toyota    | Camry 2018-20                 | All               | Stock            | 0mph<sup>4</sup>   | 0mph              |
-| Toyota    | Camry Hybrid 2018-19          | All               | Stock            | 0mph<sup>4</sup>   | 0mph              |
-| Toyota    | C-HR 2017-19                  | All               | Stock            | 0mph               | 0mph              |
+| Toyota    | Camry 2021                    | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Camry Hybrid 2018-20          | All               | Stock            | 0mph<sup>4</sup>   | 0mph              |
+| Toyota    | Camry Hybrid 2021             | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | C-HR 2017-20                  | All               | Stock            | 0mph               | 0mph              |
 | Toyota    | C-HR Hybrid 2017-19           | All               | Stock            | 0mph               | 0mph              |
 | Toyota    | Corolla 2017-19               | All               | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
-| Toyota    | Corolla 2020                  | All               | openpilot        | 0mph               | 0mph              |
-| Toyota    | Corolla Hatchback 2019-20     | All               | openpilot        | 0mph               | 0mph              |
-| Toyota    | Corolla Hybrid 2020           | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Corolla 2020-21               | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Corolla Hatchback 2019-21     | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Corolla Hybrid 2020-21        | All               | openpilot        | 0mph               | 0mph              |
 | Toyota    | Highlander 2017-19            | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Highlander 2020-21            | All               | openpilot        | 0mph               | 0mph              |
 | Toyota    | Highlander Hybrid 2017-19     | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Toyota    | Highlander 2020               | All               | openpilot        | 0mph               | 0mph              |
-| Toyota    | Highlander Hybrid 2020        | All               | openpilot        | 0mph               | 0mph              |
-| Toyota    | Prius 2016                    | TSS-P             | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Toyota    | Prius 2017-20                 | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Highlander Hybrid 2020-21     | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Prius 2016-20                 | TSS-P             | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Prius 2021                    | All               | openpilot        | 0mph               | 0mph              |
 | Toyota    | Prius Prime 2017-20           | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Toyota    | Rav4 2016                     | TSS-P             | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
-| Toyota    | Rav4 2017-18                  | All               | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
-| Toyota    | Rav4 2019-20                  | All               | openpilot        | 0mph               | 0mph              |
-| Toyota    | Rav4 Hybrid 2016              | TSS-P             | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Toyota    | Rav4 Hybrid 2017-18           | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
-| Toyota    | Rav4 Hybrid 2019-20           | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Prius Prime 2021              | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Rav4 2016-18                  | TSS-P             | Stock<sup>3</sup>| 20mph<sup>1</sup>  | 0mph              |
+| Toyota    | Rav4 2019-21                  | All               | openpilot        | 0mph               | 0mph              |
+| Toyota    | Rav4 Hybrid 2016-18           | TSS-P             | Stock<sup>3</sup>| 0mph               | 0mph              |
+| Toyota    | Rav4 Hybrid 2019-21           | All               | openpilot        | 0mph               | 0mph              |
 | Toyota    | Sienna 2018-20                | All               | Stock<sup>3</sup>| 0mph               | 0mph              |
 
 <sup>1</sup>[Comma Pedal](https://github.com/commaai/openpilot/wiki/comma-pedal) is used to provide stop-and-go capability to some of the openpilot-supported cars that don't currently support stop-and-go. ***NOTE: The Comma Pedal is not officially supported by [comma](https://comma.ai).*** <br />
@@ -128,6 +132,7 @@ Community Maintained Cars and Features
 
 | Make      | Model (US Market Reference)   | Supported Package | ACC              | No ACC accel below | No ALC below |
 | ----------| ------------------------------| ------------------| -----------------| -------------------| -------------|
+| Audi      | A3 2015, 2017                 | Prestige          | Stock            | 0mph               | 0mph         |
 | Buick     | Regal 2018<sup>1</sup>        | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
 | Cadillac  | ATS 2018<sup>1</sup>          | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
 | Chevrolet | Malibu 2017<sup>1</sup>       | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
@@ -135,38 +140,55 @@ Community Maintained Cars and Features
 | Chrysler  | Pacifica 2017-18              | Adaptive Cruise   | Stock            | 0mph               | 9mph         |
 | Chrysler  | Pacifica 2020                 | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
 | Chrysler  | Pacifica Hybrid 2017-18       | Adaptive Cruise   | Stock            | 0mph               | 9mph         |
-| Chrysler  | Pacifica Hybrid 2019-20       | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
+| Chrysler  | Pacifica Hybrid 2019-21       | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
+| Genesis   | G70 2018                      | All               | Stock            | 0mph               | 0mph         |
 | Genesis   | G80 2018                      | All               | Stock            | 0mph               | 0mph         |
 | Genesis   | G90 2018                      | All               | Stock            | 0mph               | 0mph         |
-| GMC       | Acadia Denali 2018<sup>2</sup>| Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
+| GMC       | Acadia 2018<sup>1</sup>       | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
 | Holden    | Astra 2017<sup>1</sup>        | Adaptive Cruise   | openpilot        | 0mph               | 7mph         |
 | Hyundai   | Elantra 2017-19               | SCC + LKAS        | Stock            | 19mph              | 34mph        |
 | Hyundai   | Genesis 2015-16               | SCC + LKAS        | Stock            | 19mph              | 37mph        |
-| Hyundai   | Ioniq Electric Premium SE 2020| SCC + LKAS        | Stock            | 0mph               | 32mph        |
-| Hyundai   | Ioniq Electric Limited 2019   | SCC + LKAS        | Stock            | 0mph               | 32mph        |
-| Hyundai   | Kona 2017-19                  | SCC + LKAS        | Stock            | 22mph              | 0mph         |
+| Hyundai   | Ioniq Electric 2019           | SCC + LKAS        | Stock            | 0mph               | 32mph        |
+| Hyundai   | Ioniq Electric 2020           | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Hyundai   | Kona 2020                     | SCC + LKAS        | Stock            | 0mph               | 0mph         |
 | Hyundai   | Kona EV 2019                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
-| Hyundai   | Palisade 2020                 | All               | Stock            | 0mph               | 0mph         |
-| Hyundai   | Santa Fe 2019                 | All               | Stock            | 0mph               | 0mph         |
-| Hyundai   | Sonata 2019                   | All               | Stock            | 0mph               | 0mph         |
+| Hyundai   | Santa Fe 2019-20              | All               | Stock            | 0mph               | 0mph         |
+| Hyundai   | Sonata 2018-2019              | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Hyundai   | Veloster 2019                 | SCC + LKAS        | Stock            | 5mph               | 0mph         |
 | Jeep      | Grand Cherokee 2016-18        | Adaptive Cruise   | Stock            | 0mph               | 9mph         |
 | Jeep      | Grand Cherokee 2019-20        | Adaptive Cruise   | Stock            | 0mph               | 39mph        |
-| Kia       | Forte 2018-19                 | SCC + LKAS        | Stock            | 0mph               | 0mph         |
-| Kia       | Optima 2017                   | SCC + LKAS/LDWS   | Stock            | 0mph               | 32mph        |
+| Kia       | Forte 2018-19, 2021           | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Kia       | Niro EV 2020                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Kia       | Optima 2017                   | SCC + LKAS        | Stock            | 0mph               | 32mph        |
 | Kia       | Optima 2019                   | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Kia       | Seltos 2021                   | SCC + LKAS        | Stock            | 0mph               | 0mph         |
 | Kia       | Sorento 2018                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
 | Kia       | Stinger 2018                  | SCC + LKAS        | Stock            | 0mph               | 0mph         |
-| Nissan    | Leaf 2018-19<sup>2</sup>      | Propilot          | Stock            | 0mph               | 0mph         |
-| Nissan    | Rogue 2019<sup>2</sup>        | Propilot          | Stock            | 0mph               | 0mph         |
-| Nissan    | X-Trail 2017<sup>2</sup>      | Propilot          | Stock            | 0mph               | 0mph         |
+| Kia       | Ceed 2019                     | SCC + LKAS        | Stock            | 0mph               | 0mph         |
+| Nissan    | Altima 2020                   | ProPILOT          | Stock            | 0mph               | 0mph         |
+| Nissan    | Leaf 2018-20                  | ProPILOT          | Stock            | 0mph               | 0mph         |
+| Nissan    | Rogue 2018-19                 | ProPILOT          | Stock            | 0mph               | 0mph         |
+| Nissan    | X-Trail 2017                  | ProPILOT          | Stock            | 0mph               | 0mph         |
+| SEAT      | Ateca 2018                    | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Škoda     | Kodiaq 2018                   | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Škoda     | Scala 2020                    | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Škoda     | Superb 2018                   | Driver Assistance | Stock            | 0mph               | 0mph         |
 | Subaru    | Ascent 2019                   | EyeSight          | Stock            | 0mph               | 0mph         |
 | Subaru    | Crosstrek 2018-19             | EyeSight          | Stock            | 0mph               | 0mph         |
-| Subaru    | Forester 2019                 | EyeSight          | Stock            | 0mph               | 0mph         |
+| Subaru    | Forester 2019-20              | EyeSight          | Stock            | 0mph               | 0mph         |
 | Subaru    | Impreza 2017-19               | EyeSight          | Stock            | 0mph               | 0mph         |
+| Volkswagen| e-Golf 2014, 2020             | Driver Assistance | Stock            | 0mph               | 0mph         |
 | Volkswagen| Golf 2015-19                  | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Golf GTE 2016                 | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Golf GTI 2018-19              | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Golf R 2016-19                | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Golf SportsVan 2016           | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Jetta 2018-21                 | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Passat 2016-17<sup>2</sup>    | Driver Assistance | Stock            | 0mph               | 0mph         |
+| Volkswagen| Tiguan 2020                   | Driver Assistance | Stock            | 0mph               | 0mph         |
 
-<sup>1</sup>Requires an [OBD-II car harness](https://comma.ai/shop/products/comma-car-harness) and [community built giraffe](https://github.com/commaai/openpilot/wiki/GM). ***NOTE: disconnecting the ASCM disables Automatic Emergency Braking (AEB).*** <br />
-<sup>2</sup>Requires a custom connector for the developer [car harness](https://comma.ai/shop/products/car-harness) <br />
+<sup>1</sup>Requires an [OBD-II car harness](https://comma.ai/shop/products/comma-car-harness) and [community built ASCM harness](https://github.com/commaai/openpilot/wiki/GM#hardware). ***NOTE: disconnecting the ASCM disables Automatic Emergency Braking (AEB).*** <br />
+<sup>2</sup>Only includes the MQB Passat sold outside of North America. The NMS Passat made in Chattanooga TN is not yet supported.
 
 Although it's not upstream, there's a community of people getting openpilot to run on Tesla's [here](https://tinkla.us/)
 
@@ -242,7 +264,6 @@ Many factors can impact the performance of openpilot DM, causing it to be unable
 * Low light conditions, such as driving at night or in dark tunnels.
 * Bright light (due to oncoming headlights, direct sunlight, etc.).
 * The driver's face is partially or completely outside field of view of the driver facing camera.
-* Right hand driving vehicles.
 * The driver facing camera is obstructed, covered, or damaged.
 
 The list above does not represent an exhaustive list of situations that may interfere with proper operation of openpilot components. A driver should not rely on openpilot DM to assess their level of attention.
@@ -262,7 +283,7 @@ By using openpilot, you agree to [our Privacy Policy](https://my.comma.ai/privac
 Safety and Testing
 ----
 
-* openpilot observes ISO26262 guidelines, see [SAFETY.md](SAFETY.md) for more detail.
+* openpilot observes ISO26262 guidelines, see [SAFETY.md](SAFETY.md) for more details.
 * openpilot has software in the loop [tests](.github/workflows/test.yaml) that run on every commit.
 * The safety model code lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
 * panda has software in the loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
@@ -272,8 +293,23 @@ Safety and Testing
 
 Testing on PC
 ------
+For simplified development and experimentation, openpilot can be run in the CARLA driving simulator, which allows you to develop openpilot without a car. The whole setup should only take a few minutes.
+
+Steps:
+1) Start the CARLA server on first terminal
+```
+bash -c "$(curl https://raw.githubusercontent.com/commaai/openpilot/master/tools/sim/start_carla.sh)"
+```
+2) Start openpilot on second terminal
+```
+bash -c "$(curl https://raw.githubusercontent.com/commaai/openpilot/master/tools/sim/start_openpilot_docker.sh)"
+```
+3) Press 1 to engage openpilot
+
+See the full [README](tools/sim/README.md)
+
+You should also take a look at the tools directory in master: lots of tools you can use to replay driving data, test, and develop openpilot from your PC.
 
-Check out the tools directory in master: lots of tools you can use to replay driving data, test and develop openpilot from your pc.
 
 Community and Contributing
 ------
@@ -289,7 +325,6 @@ And [follow us on Twitter](https://twitter.com/comma_ai).
 Directory Structure
 ------
     .
-    ├── apk                 # The apk files used for the UI
     ├── cereal              # The messaging spec and libs used for all logs
     ├── common              # Library like functionality we've developed here
     ├── installer/updater   # Manages auto-updates of openpilot
@@ -298,7 +333,7 @@ Directory Structure
     ├── phonelibs           # Libraries used on NEOS devices
     ├── pyextra             # Libraries used on NEOS devices
     └── selfdrive           # Code needed to drive the car
-        ├── assets          # Fonts, images, and sounds for UI
+        ├── assets          # Fonts, images and sounds for UI
         ├── athena          # Allows communication with the app
         ├── boardd          # Daemon to talk to the board
         ├── camerad         # Driver to capture images from the camera sensors
@@ -315,8 +350,6 @@ Directory Structure
         ├── test            # Unit tests, system tests and a car simulator
         └── ui              # The UI
 
-To understand how the services interact, see `cereal/service_list.yaml`.
-
 Licensing
 ------
 

RELEASES.md

@@ -1,3 +1,81 @@
+Version 0.8.3 (2021-04-01)
+========================
+ * New model
+   * Trained on new diverse dataset from 2000+ users from 30+ countries
+   * Trained with improved segnet from the comma-pencil community project
+   * 🥬 Dramatically improved end-to-end lateral performance 🥬
+ * Toggle added to disable the use of lanelines
+ * NEOS update: update packages and support for new UI
+ * New offroad UI based on Qt
+ * Default SSH key only used for setup
+ * Kia Ceed 2019 support thanks to ZanZaD13!
+ * Kia Seltos 2021 support thanks to speedking456!
+ * Added support for many Volkswagen and Škoda models thanks to jyoung8607!
+
+Version 0.8.2 (2021-02-26)
+========================
+ * Use model points directly in MPC (no more polyfits), making lateral planning more accurate
+ * Use model heading prediction for smoother lateral control
+ * Smarter actuator delay compensation
+ * Improve qcamera resolution for improved video in explorer and connect
+ * Adjust maximum engagement speed to better fit the model's training distribution
+ * New driver monitoring model trained with 3x more diverse data
+ * Improved face detection with masks
+ * More predictable DM alerts when visibility is bad
+ * Rewritten video streaming between openpilot processes
+ * Improved longitudinal tuning on TSS2 Corolla and Rav4 thanks to briskspirit!
+ * Audi A3 2015 and 2017 support thanks to keeleysam!
+ * Nissan Altima 2020 support thanks to avolmensky!
+ * Lexus ES Hybrid 2018 support thanks to TheInventorMan!
+ * Toyota Camry Hybrid 2021 support thanks to alancyau!
+
+Version 0.8.1 (2020-12-21)
+========================
+ * Original EON is deprecated, upgrade to comma two
+ * Better model performance in heavy rain
+ * Better lane positioning in turns
+ * Fixed bug where model would cut turns on empty roads at night
+ * Fixed issue where some Toyotas would not completely stop thanks to briskspirit!
+ * Toyota Camry 2021 with TSS2.5 support
+ * Hyundai Ioniq Electric 2020 support thanks to baldwalker!
+
+Version 0.8.0 (2020-11-30)
+========================
+ * New driving model: fully 3D and improved cut-in detection
+ * UI draws 2 road edges, 4 lanelines and paths in 3D
+ * Major fixes to cut-in detection for openpilot longitudinal
+ * Grey panda is no longer supported, upgrade to comma two or black panda
+ * Lexus NX 2018 support thanks to matt12eagles!
+ * Kia Niro EV 2020 support thanks to nickn17!
+ * Toyota Prius 2021 support thanks to rav4kumar!
+ * Improved lane positioning with uncertain lanelines, wide lanes and exits
+ * Improved lateral control for Prius and Subaru
+
+Version 0.7.10 (2020-10-29)
+========================
+ * Grey panda is deprecated, upgrade to comma two or black panda
+ * NEOS update: update to Python 3.8.2 and lower CPU frequency
+ * Improved thermals due to reduced CPU frequency
+ * Update SNPE to 1.41.0
+ * Reduced offroad power consumption
+ * Various system stability improvements
+ * Acura RDX 2020 support thanks to csouers!
+
+Version 0.7.9 (2020-10-09)
+========================
+ * Improved car battery power management
+ * Improved updater robustness
+ * Improved realtime performance
+ * Reduced UI and modeld lags
+ * Increased torque on 2020 Hyundai Sonata and Palisade
+
+Version 0.7.8 (2020-08-19)
+========================
+ * New driver monitoring model: improved face detection and better compatibility with sunglasses
+ * Download NEOS operating system updates in the background
+ * Improved updater reliability and responsiveness
+ * Hyundai Kona 2020, Veloster 2019, and Genesis G70 2018 support thanks to xps-genesis!
+
 Version 0.7.7 (2020-07-20)
 ========================
  * White panda is no longer supported, upgrade to comma two or black panda
@@ -14,24 +92,24 @@ Version 0.7.7 (2020-07-20)
 
 Version 0.7.6.1 (2020-06-16)
 ========================
-* Hotfix: update kernel on some comma twos (orders #8570-#8680)
+ * Hotfix: update kernel on some comma twos (orders #8570-#8680)
 
 Version 0.7.6 (2020-06-05)
 ========================
-* White panda is deprecated, upgrade to comma two or black panda
-* 2017 Nissan X-Trail, 2018-19 Leaf and 2019 Rogue support thanks to avolmensky!
-* 2017 Mazda CX-5 support in dashcam mode thanks to Jafaral!
-* Huge CPU savings in modeld by using thneed!
-* Lots of code cleanup and refactors
+ * White panda is deprecated, upgrade to comma two or black panda
+ * 2017 Nissan X-Trail, 2018-19 Leaf and 2019 Rogue support thanks to avolmensky!
+ * 2017 Mazda CX-5 support in dashcam mode thanks to Jafaral!
+ * Huge CPU savings in modeld by using thneed!
+ * Lots of code cleanup and refactors
 
 Version 0.7.5 (2020-05-13)
 ========================
-* Right-Hand Drive support for both driving and driver monitoring!
-* New driving model: improved at sharp turns and lead speed estimation
-* New driver monitoring model: overall improvement on comma two
-* Driver camera preview in settings to improve mounting position
-* Added support for many Hyundai, Kia, Genesis models thanks to xx979xx!
-* Improved lateral tuning for 2020 Toyota Rav 4 (hybrid)
+ * Right-Hand Drive support for both driving and driver monitoring!
+ * New driving model: improved at sharp turns and lead speed estimation
+ * New driver monitoring model: overall improvement on comma two
+ * Driver camera preview in settings to improve mounting position
+ * Added support for many Hyundai, Kia, Genesis models thanks to xx979xx!
+ * Improved lateral tuning for 2020 Toyota Rav 4 (hybrid)
 
 Version 0.7.4 (2020-03-20)
 ========================
@@ -467,96 +545,96 @@ Version 0.3.4 (2017-07-28)
 
 Version 0.3.3  (2017-06-28)
 ===========================
-  * Improved model trained on more data
-  * Alpha CR-V support thanks to energee and johnnwvs!
-  * Using the opendbc project for DBC files
-  * Minor performance improvements
-  * UI update thanks to pjlao307
-  * Power off button
-  * 6% more torque on the Civic
+ * Improved model trained on more data
+ * Alpha CR-V support thanks to energee and johnnwvs!
+ * Using the opendbc project for DBC files
+ * Minor performance improvements
+ * UI update thanks to pjlao307
+ * Power off button
+ * 6% more torque on the Civic
 
 Version 0.3.2  (2017-05-22)
 ===========================
-  * Minor stability bugfixes
-  * Added metrics and rear view mirror disable to settings
-  * Update model with more crowdsourced data
+ * Minor stability bugfixes
+ * Added metrics and rear view mirror disable to settings
+ * Update model with more crowdsourced data
 
 Version 0.3.1  (2017-05-17)
 ===========================
-  * visiond stability bugfix
-  * Add logging for angle and flashing
+ * visiond stability bugfix
+ * Add logging for angle and flashing
 
 Version 0.3.0  (2017-05-12)
 ===========================
-  * Add CarParams struct to improve the abstraction layer
-  * Refactor visiond IPC to support multiple clients
-  * Add raw GPS and beginning support for navigation
-  * Improve model in visiond using crowdsourced data
-  * Add improved system logging to diagnose instability
-  * Rewrite baseui in React Native
-  * Moved calibration to the cloud
+ * Add CarParams struct to improve the abstraction layer
+ * Refactor visiond IPC to support multiple clients
+ * Add raw GPS and beginning support for navigation
+ * Improve model in visiond using crowdsourced data
+ * Add improved system logging to diagnose instability
+ * Rewrite baseui in React Native
+ * Moved calibration to the cloud
 
 Version 0.2.9  (2017-03-01)
 ===========================
-  * Retain compatibility with NEOS v1
+ * Retain compatibility with NEOS v1
 
 Version 0.2.8  (2017-02-27)
 ===========================
-  * Fix bug where frames were being dropped in minute 71
+ * Fix bug where frames were being dropped in minute 71
 
 Version 0.2.7  (2017-02-08)
 ===========================
-  * Better performance and pictures at night
-  * Fix ptr alignment issue in boardd
-  * Fix brake error light, fix crash if too cold
+ * Better performance and pictures at night
+ * Fix ptr alignment issue in boardd
+ * Fix brake error light, fix crash if too cold
 
 Version 0.2.6  (2017-01-31)
 ===========================
-  * Fix bug in visiond model execution
+ * Fix bug in visiond model execution
 
 Version 0.2.5  (2017-01-30)
 ===========================
-  * Fix race condition in manager
+ * Fix race condition in manager
 
 Version 0.2.4  (2017-01-27)
 ===========================
-  * OnePlus 3T support
-  * Enable installation as NEOS app
-  * Various minor bugfixes
+ * OnePlus 3T support
+ * Enable installation as NEOS app
+ * Various minor bugfixes
 
 Version 0.2.3  (2017-01-11)
 ===========================
-  * Reduce space usage by 80%
-  * Add better logging
-  * Add Travis CI
+ * Reduce space usage by 80%
+ * Add better logging
+ * Add Travis CI
 
 Version 0.2.2  (2017-01-10)
 ===========================
-  * Board triggers started signal on CAN messages
-  * Improved autoexposure
-  * Handle out of space, improve upload status
+ * Board triggers started signal on CAN messages
+ * Improved autoexposure
+ * Handle out of space, improve upload status
 
 Version 0.2.1  (2016-12-14)
 ===========================
-  * Performance improvements, removal of more numpy
-  * Fix boardd process priority
-  * Make counter timer reset on use of steering wheel
+ * Performance improvements, removal of more numpy
+ * Fix boardd process priority
+ * Make counter timer reset on use of steering wheel
 
 Version 0.2  (2016-12-12)
 =========================
-  * Car/Radar abstraction layers have shipped, see cereal/car.capnp
-  * controlsd has been refactored
-  * Shipped plant model and testing maneuvers
-  * visiond exits more gracefully now
-  * Hardware encoder in visiond should always init
-  * ui now turns off the screen after 30 seconds
-  * Switch to openpilot release branch for future releases
-  * Added preliminary Docker container to run tests on PC
+ * Car/Radar abstraction layers have shipped, see cereal/car.capnp
+ * controlsd has been refactored
+ * Shipped plant model and testing maneuvers
+ * visiond exits more gracefully now
+ * Hardware encoder in visiond should always init
+ * ui now turns off the screen after 30 seconds
+ * Switch to openpilot release branch for future releases
+ * Added preliminary Docker container to run tests on PC
 
 Version 0.1  (2016-11-29)
 =========================
-  * Initial release of openpilot
-  * Adaptive cruise control is working
-  * Lane keep assist is working
-  * Support for Acura ILX 2016 with AcuraWatch Plus
-  * Support for Honda Civic 2016 Touring Edition
+ * Initial release of openpilot
+ * Adaptive cruise control is working
+ * Lane keep assist is working
+ * Support for Acura ILX 2016 with AcuraWatch Plus
+ * Support for Honda Civic 2016 Touring Edition

SConstruct

@@ -2,7 +2,12 @@ import os
 import shutil
 import subprocess
 import sys
+import sysconfig
 import platform
+import numpy as np
+
+TICI = os.path.isfile('/TICI')
+Decider('MD5-timestamp')
 
 AddOption('--test',
           action='store_true',
@@ -12,20 +17,44 @@ AddOption('--asan',
           action='store_true',
           help='turn on ASAN')
 
-arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--compile_db',
+          action='store_true',
+          help='build clang compilation database')
+
+AddOption('--mpc-generate',
+          action='store_true',
+          help='regenerates the mpc sources')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-if arch == "aarch64" and not os.path.isdir("/system"):
+
+if arch == "aarch64" and TICI:
   arch = "larch64"
 
-webcam = bool(ARGUMENTS.get("use_webcam", 0))
+USE_WEBCAM = os.getenv("USE_WEBCAM") is not None
 QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None
 
+lenv = {
+  "PATH": os.environ['PATH'],
+}
+
 if arch == "aarch64" or arch == "larch64":
-  lenv = {
-    "LD_LIBRARY_PATH": '/data/data/com.termux/files/usr/lib',
-    "PATH": os.environ['PATH'],
-  }
+  lenv["LD_LIBRARY_PATH"] = '/data/data/com.termux/files/usr/lib'
 
   if arch == "aarch64":
     # android
@@ -37,8 +66,8 @@ if arch == "aarch64" or arch == "larch64":
   ]
 
   libpath = [
+    "/usr/local/lib",
     "/usr/lib",
-    "/data/data/com.termux/files/usr/lib",
     "/system/vendor/lib64",
     "/system/comma/usr/lib",
     "#phonelibs/nanovg",
@@ -56,26 +85,20 @@ if arch == "aarch64" or arch == "larch64":
   else:
     libpath += [
       "#phonelibs/snpe/aarch64",
-      "#phonelibs/libyuv/lib"
+      "#phonelibs/libyuv/lib",
+      "/system/vendor/lib64"
     ]
     cflags = ["-DQCOM", "-mcpu=cortex-a57"]
     cxxflags = ["-DQCOM", "-mcpu=cortex-a57"]
-    rpath = ["/system/vendor/lib64"]
+    rpath = []
 
     if QCOM_REPLAY:
       cflags += ["-DQCOM_REPLAY"]
       cxxflags += ["-DQCOM_REPLAY"]
-
 else:
   cflags = []
   cxxflags = []
-
-  lenv = {
-    "PATH": "#external/bin:" + os.environ['PATH'],
-  }
-  cpppath = [
-    "#external/tensorflow/include",
-  ]
+  cpppath = []
 
   if arch == "Darwin":
     libpath = [
@@ -83,15 +106,16 @@ else:
       "#cereal",
       "#selfdrive/common",
       "/usr/local/lib",
+      "/usr/local/opt/openssl/lib",
       "/System/Library/Frameworks/OpenGL.framework/Libraries",
     ]
     cflags += ["-DGL_SILENCE_DEPRECATION"]
     cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += ["/usr/local/opt/openssl/include"]
   else:
     libpath = [
       "#phonelibs/snpe/x86_64-linux-clang",
       "#phonelibs/libyuv/x64/lib",
-      "#external/tensorflow/lib",
       "#cereal",
       "#selfdrive/common",
       "/usr/lib",
@@ -99,7 +123,7 @@ else:
     ]
 
   rpath = [
-    "external/tensorflow/lib",
+    "phonelibs/snpe/x86_64-linux-clang",
     "cereal",
     "selfdrive/common"
   ]
@@ -108,11 +132,14 @@ else:
   rpath = [os.path.join(os.getcwd(), x) for x in rpath]
 
 if GetOption('asan'):
-  ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"]
-  ldflags_asan = ["-fsanitize=address"]
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
 else:
-  ccflags_asan = []
-  ldflags_asan = []
+  ccflags = []
+  ldflags = []
 
 # change pythonpath to this
 lenv["PYTHONPATH"] = Dir("#").path
@@ -125,19 +152,23 @@ env = Environment(
     "-O2",
     "-Wunused",
     "-Werror",
+    "-Wno-unknown-warning-option",
     "-Wno-deprecated-register",
+    "-Wno-register",
     "-Wno-inconsistent-missing-override",
-  ] + cflags + ccflags_asan,
+    "-Wno-c99-designator",
+    "-Wno-reorder-init-list",
+  ] + cflags + ccflags,
 
   CPPPATH=cpppath + [
     "#",
     "#selfdrive",
+    "#phonelibs/catch2/include",
     "#phonelibs/bzip2",
     "#phonelibs/libyuv/include",
     "#phonelibs/openmax/include",
     "#phonelibs/json11",
     "#phonelibs/curl/include",
-    #"#phonelibs/opencv/include", # use opencv4 instead
     "#phonelibs/libgralloc/include",
     "#phonelibs/android_frameworks_native/include",
     "#phonelibs/android_hardware_libhardware/include",
@@ -145,44 +176,52 @@ env = Environment(
     "#phonelibs/linux/include",
     "#phonelibs/snpe/include",
     "#phonelibs/nanovg",
+    "#phonelibs/qrcode",
+    "#selfdrive/boardd",
     "#selfdrive/common",
     "#selfdrive/camerad",
     "#selfdrive/camerad/include",
     "#selfdrive/loggerd/include",
     "#selfdrive/modeld",
-    "#cereal/messaging",
+    "#selfdrive/sensord",
+    "#selfdrive/ui",
     "#cereal",
+    "#cereal/messaging",
+    "#cereal/visionipc",
     "#opendbc/can",
   ],
 
   CC='clang',
   CXX='clang++',
-  LINKFLAGS=ldflags_asan,
+  LINKFLAGS=ldflags,
 
   RPATH=rpath,
 
   CFLAGS=["-std=gnu11"] + cflags,
-  CXXFLAGS=["-std=c++14"] + cxxflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
   LIBPATH=libpath + [
     "#cereal",
-    "#selfdrive/common",
     "#phonelibs",
-  ]
+    "#opendbc/can",
+    "#selfdrive/boardd",
+    "#selfdrive/common",
+  ],
+  CYTHONCFILESUFFIX=".cpp",
+  COMPILATIONDB_USE_ABSPATH=True,
+  tools=["default", "cython", "compilation_db"],
 )
 
+if GetOption('compile_db'):
+  env.CompilationDatabase('compile_commands.json')
+
 if os.environ.get('SCONS_CACHE'):
   cache_dir = '/tmp/scons_cache'
+  if TICI:
+    cache_dir = '/data/scons_cache'
 
-  if os.getenv('CI'):
-    branch = os.getenv('GIT_BRANCH')
+  if QCOM_REPLAY:
+    cache_dir = '/tmp/scons_cache_qcom_replay'
 
-    if QCOM_REPLAY:
-      cache_dir = '/tmp/scons_cache_qcom_replay'
-    elif branch is not None and branch != 'master':
-      cache_dir_branch = '/tmp/scons_cache_' + branch
-      if not os.path.isdir(cache_dir_branch) and os.path.isdir(cache_dir):
-        shutil.copytree(cache_dir, cache_dir_branch)
-      cache_dir = cache_dir_branch
   CacheDir(cache_dir)
 
 node_interval = 5
@@ -206,9 +245,93 @@ def abspath(x):
     # rpath works elsewhere
     return x[0].path.rsplit("/", 1)[1][:-3]
 
+# Cython build enviroment
+py_include = sysconfig.get_paths()['include']
+envCython = env.Clone()
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"]
+
+envCython["LIBS"] = []
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+elif arch == "aarch64":
+  envCython["LINKFLAGS"] = ["-shared"]
+  envCython["LIBS"] = [os.path.basename(py_include)]
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+Export('envCython')
+
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets"]
+if arch != "aarch64":
+  qt_modules += ["DBus"]
+
+qt_libs = []
+if arch == "Darwin":
+  qt_env['QTDIR'] = "/usr/local/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+elif arch == "aarch64":
+  qt_env['QTDIR'] = "/system/comma/usr"
+  qt_dirs = [
+    f"/system/comma/usr/include/qt",
+  ]
+  qt_dirs += [f"/system/comma/usr/include/qt/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  qt_libs += ['EGL', 'GLESv3', 'c++_shared']
+else:
+  qt_env['QTDIR'] = "/usr"
+  qt_dirs = [
+    f"/usr/include/{real_arch}-linux-gnu/qt5",
+    f"/usr/include/{real_arch}-linux-gnu/qt5/QtGui/5.12.8/QtGui",
+  ]
+  qt_dirs += [f"/usr/include/{real_arch}-linux-gnu/qt5/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+
+qt_env.Tool('qt')
+qt_env['CPPPATH'] += qt_dirs + ["#selfdrive/ui/qt/"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_QUICK_LIB",
+  "-DQT_QUICKWIDGETS_LIB",
+  "-DQT_QML_LIB",
+  "-DQT_CORE_LIB"
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui']
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+Export('qt_env')
+
+
 # still needed for apks
 zmq = 'zmq'
-Export('env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY')
+Export('env', 'arch', 'real_arch', 'zmq', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY')
 
 # cereal and messaging are shared with the system
 SConscript(['cereal/SConscript'])
@@ -218,26 +341,33 @@ if SHARED:
 else:
   cereal = [File('#cereal/libcereal.a')]
   messaging = [File('#cereal/libmessaging.a')]
+  visionipc = [File('#cereal/libvisionipc.a')]
+
 Export('cereal', 'messaging')
 
 SConscript(['selfdrive/common/SConscript'])
-Import('_common', '_visionipc', '_gpucommon', '_gpu_libs')
+Import('_common', '_gpucommon', '_gpu_libs')
 
 if SHARED:
-  common, visionipc, gpucommon = abspath(common), abspath(visionipc), abspath(gpucommon)
+  common, gpucommon = abspath(common), abspath(gpucommon)
 else:
   common = [_common, 'json11']
-  visionipc = _visionipc
   gpucommon = [_gpucommon] + _gpu_libs
 
-Export('common', 'visionipc', 'gpucommon')
+Export('common', 'gpucommon', 'visionipc')
 
+
+# Build openpilot
+
+SConscript(['cereal/SConscript'])
+SConscript(['panda/board/SConscript'])
 SConscript(['opendbc/can/SConscript'])
 
+SConscript(['phonelibs/SConscript'])
+
 SConscript(['common/SConscript'])
 SConscript(['common/kalman/SConscript'])
 SConscript(['common/transformations/SConscript'])
-SConscript(['phonelibs/SConscript'])
 
 SConscript(['selfdrive/camerad/SConscript'])
 SConscript(['selfdrive/modeld/SConscript'])
@@ -249,16 +379,21 @@ SConscript(['selfdrive/controls/lib/longitudinal_mpc_model/SConscript'])
 
 SConscript(['selfdrive/boardd/SConscript'])
 SConscript(['selfdrive/proclogd/SConscript'])
+SConscript(['selfdrive/clocksd/SConscript'])
 
-SConscript(['selfdrive/ui/SConscript'])
 SConscript(['selfdrive/loggerd/SConscript'])
 
 SConscript(['selfdrive/locationd/SConscript'])
 SConscript(['selfdrive/locationd/models/SConscript'])
+SConscript(['selfdrive/sensord/SConscript'])
+SConscript(['selfdrive/ui/SConscript'])
 
-if arch == "aarch64":
+if arch != "Darwin":
   SConscript(['selfdrive/logcatd/SConscript'])
-  SConscript(['selfdrive/sensord/SConscript'])
-  SConscript(['selfdrive/clocksd/SConscript'])
-else:
-  SConscript(['tools/lib/index_log/SConscript'])
+
+if real_arch == "x86_64":
+  SConscript(['tools/nui/SConscript'])
+
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

cereal/.gitignore

@@ -1,14 +1,21 @@
 gen
 node_modules
 package-lock.json
+*.tmp
 *.pyc
 __pycache__
 .*.swp
 .*.swo
-libcereal*.a
+*.os
+*.o
+*.a
+
+test_runner
+
 libmessaging.*
 libmessaging_shared.*
 services.h
 .sconsign.dblite
 libcereal_shared.*
 .mypy_cache/
+catch2/

cereal/SConscript

@@ -1,35 +1,33 @@
-Import('env', 'arch', 'zmq')
+Import('env', 'envCython', 'arch', 'zmq', 'QCOM_REPLAY')
 
+import shutil
+
+cereal_dir = Dir('.')
 gen_dir = Dir('gen')
 messaging_dir = Dir('messaging')
 
-# TODO: remove src-prefix and cereal from command string. can we set working directory?
+# Build cereal
+
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp']
 env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
-env.Command(
-  ['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'],
-  ['car.capnp', 'log.capnp'],
-  'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/')
-import shutil
+env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
+            schema_files,
+            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
+
 if shutil.which('capnpc-java'):
-  env.Command(
-    ['gen/java/Car.java', 'gen/java/Log.java'],
-    ['car.capnp', 'log.capnp'],
-    'capnpc $SOURCES --src-prefix=cereal -o java:' + gen_dir.path + '/java/')
+  env.Command(['gen/java/Car.java', 'gen/java/Log.java'],
+              schema_files,
+              f"capnpc $SOURCES --src-prefix={cereal_dir.path} -o java:{gen_dir.path}/java/")
 
 # TODO: remove non shared cereal and messaging
-cereal_objects = env.SharedObject([
-    'gen/cpp/car.capnp.c++',
-    'gen/cpp/log.capnp.c++',
-  ])
+cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files])
 
 env.Library('cereal', cereal_objects)
 env.SharedLibrary('cereal_shared', cereal_objects)
 
-cereal_dir = Dir('.')
-services_h = env.Command(
-  ['services.h'],
-  ['service_list.yaml', 'services.py'],
-  'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+# Build messaging
+
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
 
 messaging_objects = env.SharedObject([
   'messaging/messaging.cc',
@@ -52,14 +50,33 @@ if arch == "aarch64":
 env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
 Depends('messaging/bridge.cc', services_h)
 
-# different target?
-#env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=[messaging_lib, 'zmq'])
+envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging_lib, "zmq"])
 
 
-env.Command(['messaging/messaging_pyx.so'],
-  [messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'],
-  "cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace")
+# Build Vision IPC
+vipc_sources = [
+  'visionipc/ipc.cc',
+  'visionipc/visionipc_server.cc',
+  'visionipc/visionipc_client.cc',
+  'visionipc/visionbuf.cc',
+]
+
+if arch in ["aarch64", "larch64"] and (not QCOM_REPLAY):
+  vipc_sources += ['visionipc/visionbuf_ion.cc']
+else:
+  vipc_sources += ['visionipc/visionbuf_cl.cc']
+
+vipc_objects = env.SharedObject(vipc_sources)
+vipc = env.Library('visionipc', vipc_objects)
+
+
+libs = envCython["LIBS"]+["OpenCL", "zmq", vipc, messaging_lib]
+if arch == "Darwin":
+  del libs[libs.index('OpenCL')]
+  envCython['FRAMEWORKS'] += ['OpenCL']
+envCython.Program('visionipc/visionipc_pyx.so', 'visionipc/visionipc_pyx.pyx', LIBS=libs)
 
 
 if GetOption('test'):
   env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib])
+  env.Program('visionipc/test_runner', ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'], LIBS=[vipc, messaging_lib, 'zmq', 'pthread', 'OpenCL'])

cereal/car.capnp

@@ -11,6 +11,8 @@ $Java.outerClassname("Car");
 
 struct CarEvent @0x9b1657f34caf3ad3 {
   name @0 :EventName;
+
+  # event types
   enable @1 :Bool;
   noEntry @2 :Bool;
   warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
@@ -21,11 +23,9 @@ struct CarEvent @0x9b1657f34caf3ad3 {
   permanent @8 :Bool; # alerts presented regardless of openpilot state
 
   enum EventName @0xbaa8c5d505f727de {
-    # TODO: copy from error list
     canError @0;
     steerUnavailable @1;
     brakeUnavailable @2;
-    gasUnavailable @3;
     wrongGear @4;
     doorOpen @5;
     seatbeltNotLatched @6;
@@ -37,8 +37,6 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     buttonEnable @12;
     pedalPressed @13;
     cruiseDisabled @14;
-    radarCanError @15;
-    dataNeededDEPRECATED @16;
     speedTooLow @17;
     outOfSpace @18;
     overheat @19;
@@ -49,29 +47,22 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     pcmDisable @24;
     noTarget @25;
     radarFault @26;
-    modelCommIssueDEPRECATED @27;
     brakeHold @28;
     parkBrake @29;
     manualRestart @30;
     lowSpeedLockout @31;
     plannerError @32;
-    ipasOverrideDEPRECATED @33;
     debugAlert @34;
     steerTempUnavailableMute @35;
     resumeRequired @36;
     preDriverDistracted @37;
     promptDriverDistracted @38;
     driverDistracted @39;
-    geofenceDEPRECATED @40;
-    driverMonitorOnDEPRECATED @41;
-    driverMonitorOffDEPRECATED @42;
     preDriverUnresponsive @43;
     promptDriverUnresponsive @44;
     driverUnresponsive @45;
     belowSteerSpeed @46;
-    calibrationProgressDEPRECATED @47;
     lowBattery @48;
-    invalidGiraffeHondaDEPRECATED @49;
     vehicleModelInvalid @50;
     controlsFailed @51;
     sensorDataInvalid @52;
@@ -82,14 +73,11 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     preLaneChangeLeft @57;
     preLaneChangeRight @58;
     laneChange @59;
-    invalidGiraffeToyota @60;
-    internetConnectivityNeeded @61;
     communityFeatureDisallowed @62;
     lowMemory @63;
     stockAeb @64;
     ldw @65;
     carUnrecognized @66;
-    radarCommIssue @67;
     driverMonitorLowAcc @68;
     invalidLkasSetting @69;
     speedTooHigh @70;
@@ -103,15 +91,36 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     startupMaster @78;
     fcw @79;
     steerSaturated @80;
-    whitePandaUnsupported @81;
-    startupWhitePanda @82;
-    canErrorPersistentDEPRECATED @83;
     belowEngageSpeed @84;
     noGps @85;
-    focusRecoverActive @86;
     wrongCruiseMode @87;
-    neosUpdateRequired @88;
     modeldLagging @89;
+    deviceFalling @90;
+    fanMalfunction @91;
+    cameraMalfunction @92;
+    gpsMalfunction @94;
+    processNotRunning @95;
+    dashcamMode @96;
+
+    radarCanErrorDEPRECATED @15;
+    radarCommIssueDEPRECATED @67;
+    gasUnavailableDEPRECATED @3;
+    dataNeededDEPRECATED @16;
+    modelCommIssueDEPRECATED @27;
+    ipasOverrideDEPRECATED @33;
+    geofenceDEPRECATED @40;
+    driverMonitorOnDEPRECATED @41;
+    driverMonitorOffDEPRECATED @42;
+    calibrationProgressDEPRECATED @47;
+    invalidGiraffeHondaDEPRECATED @49;
+    invalidGiraffeToyotaDEPRECATED @60;
+    internetConnectivityNeededDEPRECATED @61;
+    whitePandaUnsupportedDEPRECATED @81;
+    commIssueWarningDEPRECATED @83;
+    focusRecoverActiveDEPRECATED @86;
+    neosUpdateRequiredDEPRECATED @88;
+    modelLagWarningDEPRECATED @93;
+    startupOneplusDEPRECATED @82;
   }
 }
 
@@ -119,7 +128,6 @@ struct CarEvent @0x9b1657f34caf3ad3 {
 # all speeds in m/s
 
 struct CarState {
-  errorsDEPRECATED @0 :List(CarEvent.EventName);
   events @13 :List(CarEvent);
 
   # car speed
@@ -140,8 +148,8 @@ struct CarState {
   brakeLights @19 :Bool;
 
   # steering wheel
-  steeringAngle @7 :Float32;       # deg
-  steeringRate @15 :Float32;       # deg/s
+  steeringAngleDeg @7 :Float32;
+  steeringRateDeg @15 :Float32;
   steeringTorque @8 :Float32;      # TODO: standardize units
   steeringTorqueEps @27 :Float32;  # TODO: standardize units
   steeringPressed @9 :Bool;        # if the user is using the steering wheel
@@ -229,6 +237,8 @@ struct CarState {
       gapAdjustCruise @11;
     }
   }
+
+  errorsDEPRECATED @0 :List(CarEvent.EventName);
 }
 
 # ******* radar state @ 20hz *******
@@ -272,10 +282,6 @@ struct CarControl {
   enabled @0 :Bool;
   active @7 :Bool;
 
-  gasDEPRECATED @1 :Float32;
-  brakeDEPRECATED @2 :Float32;
-  steeringTorqueDEPRECATED @3 :Float32;
-
   actuators @6 :Actuators;
 
   cruiseControl @4 :CruiseControl;
@@ -287,7 +293,7 @@ struct CarControl {
     brake @1: Float32;
     # range from -1.0 - 1.0
     steer @2: Float32;
-    steerAngle @3: Float32;
+    steeringAngleDeg @3: Float32;
   }
 
   struct CruiseControl {
@@ -323,8 +329,6 @@ struct CarControl {
     }
 
     enum AudibleAlert {
-      # these are the choices from the Honda
-      # map as good as you can for your car
       none @0;
       chimeEngage @1;
       chimeDisengage @2;
@@ -336,6 +340,10 @@ struct CarControl {
       chimeWarning2Repeat @8;
     }
   }
+
+  gasDEPRECATED @1 :Float32;
+  brakeDEPRECATED @2 :Float32;
+  steeringTorqueDEPRECATED @3 :Float32;
 }
 
 # ****** car param ******
@@ -352,6 +360,7 @@ struct CarParams {
 
   minEnableSpeed @7 :Float32;
   minSteerSpeed @8 :Float32;
+  maxSteeringAngleDeg @54 :Float32;
   safetyModel @9 :SafetyModel;
   safetyModelPassive @42 :SafetyModel = silent;
   safetyParam @10 :Int16;
@@ -364,11 +373,11 @@ struct CarParams {
   brakeMaxV @16 :List(Float32);
 
   # things about the car in the manual
-  mass @17 :Float32;             # [kg] running weight
-  wheelbase @18 :Float32;        # [m] distance from rear to front axle
-  centerToFront @19 :Float32;   # [m] GC distance to front axle
-  steerRatio @20 :Float32;       # [] ratio between front wheels and steering wheel angles
-  steerRatioRear @21 :Float32;  # [] rear steering ratio wrt front steering (usually 0)
+  mass @17 :Float32;            # [kg] curb weight: all fluids no cargo
+  wheelbase @18 :Float32;       # [m] distance from rear axle to front axle
+  centerToFront @19 :Float32;   # [m] distance from center of mass to front axle
+  steerRatio @20 :Float32;      # [] ratio of steering wheel angle to front wheel angle
+  steerRatioRear @21 :Float32;  # [] ratio of steering wheel angle to rear wheel angle (usually 0)
 
   # things we can derive
   rotationalInertia @22 :Float32;    # [kg*m2] body rotational inertia
@@ -393,11 +402,13 @@ struct CarParams {
   steerRateCost @33 :Float32; # Lateral MPC cost on steering rate
   steerControlType @34 :SteerControlType;
   radarOffCan @35 :Bool; # True when radar objects aren't visible on CAN
+  minSpeedCan @51 :Float32; # Minimum vehicle speed from CAN (below this value drops to 0)
+  stoppingBrakeRate @52 :Float32; # brake_travel/s while trying to stop
+  startingBrakeRate @53 :Float32; # brake_travel/s while releasing on restart
 
   steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds
   openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control?
   carVin @38 :Text; # VIN number queried during fingerprinting
-  isPandaBlack @39: Bool;
   dashcamOnly @41: Bool;
   transmissionType @43 :TransmissionType;
   carFw @44 :List(CarFw);
@@ -429,10 +440,19 @@ struct CarParams {
   }
 
   struct LateralINDITuning {
-    outerLoopGain @0 :Float32;
-    innerLoopGain @1 :Float32;
-    timeConstant @2 :Float32;
-    actuatorEffectiveness @3 :Float32;
+    outerLoopGainBP @4 :List(Float32);
+    outerLoopGainV @5 :List(Float32);
+    innerLoopGainBP @6 :List(Float32);
+    innerLoopGainV @7 :List(Float32);
+    timeConstantBP @8 :List(Float32);
+    timeConstantV @9 :List(Float32);
+    actuatorEffectivenessBP @10 :List(Float32);
+    actuatorEffectivenessV @11 :List(Float32);
+
+    outerLoopGainDEPRECATED @0 :Float32;
+    innerLoopGainDEPRECATED @1 :Float32;
+    timeConstantDEPRECATED @2 :Float32;
+    actuatorEffectivenessDEPRECATED @3 :Float32;
   }
 
   struct LateralLQRTuning {
@@ -474,6 +494,7 @@ struct CarParams {
     volkswagenPq @21;
     subaruLegacy @22;  # pre-Global platform
     hyundaiLegacy @23;
+    hyundaiCommunity @24;
   }
 
   enum SteerControlType {
@@ -529,4 +550,6 @@ struct CarParams {
     fwdCamera @0;  # Standard/default integration at LKAS camera
     gateway @1;    # Integration at vehicle's CAN gateway
   }
+
+  isPandaBlackDEPRECATED @39: Bool;
 }

cereal/legacy.capnp

@@ -0,0 +1,865 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+using Java = import "./include/java.capnp";
+$Java.package("ai.comma.openpilot.cereal");
+$Java.outerClassname("Legacy");
+
+@0x80ef1ec4889c2a63;
+
+# legacy.capnp: a home for deprecated structs
+
+struct LogRotate @0x9811e1f38f62f2d1 {
+  segmentNum @0 :Int32;
+  path @1 :Text;
+}
+
+struct LiveUI @0xc08240f996aefced {
+  rearViewCam @0 :Bool;
+  alertText1 @1 :Text;
+  alertText2 @2 :Text;
+  awarenessStatus @3 :Float32;
+}
+
+struct UiLayoutState @0x88dcce08ad29dda0 {
+  activeApp @0 :App;
+  sidebarCollapsed @1 :Bool;
+  mapEnabled @2 :Bool;
+  mockEngaged @3 :Bool;
+
+  enum App @0x9917470acf94d285 {
+    home @0;
+    music @1;
+    nav @2;
+    settings @3;
+    none @4;
+  }
+}
+
+struct OrbslamCorrection @0x8afd33dc9b35e1aa {
+  correctionMonoTime @0 :UInt64;
+  prePositionECEF @1 :List(Float64);
+  postPositionECEF @2 :List(Float64);
+  prePoseQuatECEF @3 :List(Float32);
+  postPoseQuatECEF @4 :List(Float32);
+  numInliers @5 :UInt32;
+}
+
+struct EthernetPacket @0xa99a9d5b33cf5859 {
+  pkt @0 :Data;
+  ts @1 :Float32;
+}
+
+struct CellInfo @0xcff7566681c277ce {
+  timestamp @0 :UInt64;
+  repr @1 :Text; # android toString() for now
+}
+
+struct WifiScan @0xd4df5a192382ba0b {
+  bssid @0 :Text;
+  ssid @1 :Text;
+  capabilities @2 :Text;
+  frequency @3 :Int32;
+  level @4 :Int32;
+  timestamp @5 :Int64;
+
+  centerFreq0 @6 :Int32;
+  centerFreq1 @7 :Int32;
+  channelWidth @8 :ChannelWidth;
+  operatorFriendlyName @9 :Text;
+  venueName @10 :Text;
+  is80211mcResponder @11 :Bool;
+  passpoint @12 :Bool;
+
+  distanceCm @13 :Int32;
+  distanceSdCm @14 :Int32;
+
+  enum ChannelWidth @0xcb6a279f015f6b51 {
+    w20Mhz @0;
+    w40Mhz @1;
+    w80Mhz @2;
+    w160Mhz @3;
+    w80Plus80Mhz @4;
+  }
+}
+
+struct LiveEventData @0x94b7baa90c5c321e {
+  name @0 :Text;
+  value @1 :Int32;
+}
+
+struct ModelData @0xb8aad62cffef28a9 {
+  frameId @0 :UInt32;
+  frameAge @12 :UInt32;
+  frameDropPerc @13 :Float32;
+  timestampEof @9 :UInt64;
+  modelExecutionTime @14 :Float32;
+  gpuExecutionTime @16 :Float32;
+  rawPred @15 :Data;
+
+  path @1 :PathData;
+  leftLane @2 :PathData;
+  rightLane @3 :PathData;
+  lead @4 :LeadData;
+  freePath @6 :List(Float32);
+
+  settings @5 :ModelSettings;
+  leadFuture @7 :LeadData;
+  speed @8 :List(Float32);
+  meta @10 :MetaData;
+  longitudinal @11 :LongitudinalData;
+
+  struct PathData @0x8817eeea389e9f08 {
+    points @0 :List(Float32);
+    prob @1 :Float32;
+    std @2 :Float32;
+    stds @3 :List(Float32);
+    poly @4 :List(Float32);
+    validLen @5 :Float32;
+  }
+
+  struct LeadData @0xd1c9bef96d26fa91 {
+    dist @0 :Float32;
+    prob @1 :Float32;
+    std @2 :Float32;
+    relVel @3 :Float32;
+    relVelStd @4 :Float32;
+    relY @5 :Float32;
+    relYStd @6 :Float32;
+    relA @7 :Float32;
+    relAStd @8 :Float32;
+  }
+
+  struct ModelSettings @0xa26e3710efd3e914 {
+    bigBoxX @0 :UInt16;
+    bigBoxY @1 :UInt16;
+    bigBoxWidth @2 :UInt16;
+    bigBoxHeight @3 :UInt16;
+    boxProjection @4 :List(Float32);
+    yuvCorrection @5 :List(Float32);
+    inputTransform @6 :List(Float32);
+  }
+
+  struct MetaData @0x9744f25fb60f2bf8 {
+    engagedProb @0 :Float32;
+    desirePrediction @1 :List(Float32);
+    brakeDisengageProb @2 :Float32;
+    gasDisengageProb @3 :Float32;
+    steerOverrideProb @4 :Float32;
+    desireState @5 :List(Float32);
+  }
+
+  struct LongitudinalData @0xf98f999c6a071122 {
+    distances @2 :List(Float32);
+    speeds @0 :List(Float32);
+    accelerations @1 :List(Float32);
+  }
+}
+
+struct ECEFPoint @0xc25bbbd524983447 {
+  x @0 :Float64;
+  y @1 :Float64;
+  z @2 :Float64;
+}
+
+struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 {
+  x @0 :Float32;
+  y @1 :Float32;
+  z @2 :Float32;
+}
+
+struct GPSPlannerPoints @0xab54c59699f8f9f3 {
+  curPosDEPRECATED @0 :ECEFPointDEPRECATED;
+  pointsDEPRECATED @1 :List(ECEFPointDEPRECATED);
+  curPos @6 :ECEFPoint;
+  points @7 :List(ECEFPoint);
+  valid @2 :Bool;
+  trackName @3 :Text;
+  speedLimit @4 :Float32;
+  accelTarget @5 :Float32;
+}
+
+struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b {
+  valid @0 :Bool;
+  poly @1 :List(Float32);
+  trackName @2 :Text;
+  speed @3 :Float32;
+  acceleration @4 :Float32;
+  pointsDEPRECATED @5 :List(ECEFPointDEPRECATED);
+  points @6 :List(ECEFPoint);
+  xLookahead @7 :Float32;
+}
+
+struct UiNavigationEvent @0x90c8426c3eaddd3b {
+  type @0: Type;
+  status @1: Status;
+  distanceTo @2: Float32;
+  endRoadPointDEPRECATED @3: ECEFPointDEPRECATED;
+  endRoadPoint @4: ECEFPoint;
+
+  enum Type @0xe8db07dcf8fcea05 {
+    none @0;
+    laneChangeLeft @1;
+    laneChangeRight @2;
+    mergeLeft @3;
+    mergeRight @4;
+    turnLeft @5;
+    turnRight @6;
+  }
+
+  enum Status @0xb9aa88c75ef99a1f {
+    none @0;
+    passive @1;
+    approaching @2;
+    active @3;
+  }
+}
+
+struct LiveLocationData @0xb99b2bc7a57e8128 {
+  status @0 :UInt8;
+
+  # 3D fix
+  lat @1 :Float64;
+  lon @2 :Float64;
+  alt @3 :Float32;     # m
+
+  # speed
+  speed @4 :Float32;   # m/s
+
+  # NED velocity components
+  vNED @5 :List(Float32);
+
+  # roll, pitch, heading (x,y,z)
+  roll @6 :Float32;     # WRT to center of earth?
+  pitch @7 :Float32;    # WRT to center of earth?
+  heading @8 :Float32;  # WRT to north?
+
+  # what are these?
+  wanderAngle @9 :Float32;
+  trackAngle @10 :Float32;
+
+  # car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png
+
+  # gyro, in car frame, deg/s
+  gyro @11 :List(Float32);
+
+  # accel, in car frame, m/s^2
+  accel @12 :List(Float32);
+
+  accuracy @13 :Accuracy;
+
+  source @14 :SensorSource;
+  # if we are fixing a location in the past
+  fixMonoTime @15 :UInt64;
+
+  gpsWeek @16 :Int32;
+  timeOfWeek @17 :Float64;
+
+  positionECEF @18 :List(Float64);
+  poseQuatECEF @19 :List(Float32);
+  pitchCalibration @20 :Float32;
+  yawCalibration @21 :Float32;
+  imuFrame @22 :List(Float32);
+
+  struct Accuracy @0x943dc4625473b03f {
+    pNEDError @0 :List(Float32);
+    vNEDError @1 :List(Float32);
+    rollError @2 :Float32;
+    pitchError @3 :Float32;
+    headingError @4 :Float32;
+    ellipsoidSemiMajorError @5 :Float32;
+    ellipsoidSemiMinorError @6 :Float32;
+    ellipsoidOrientationError @7 :Float32;
+  }
+
+  enum SensorSource @0xc871d3cc252af657 {
+    applanix @0;
+    kalman @1;
+    orbslam @2;
+    timing @3;
+    dummy @4;
+  }
+}
+
+struct OrbOdometry @0xd7700859ed1f5b76 {
+  # timing first
+  startMonoTime @0 :UInt64;
+  endMonoTime @1 :UInt64;
+
+  # fundamental matrix and error
+  f @2: List(Float64);
+  err @3: Float64;
+
+  # number of inlier points
+  inliers @4: Int32;
+
+  # for debug only
+  # indexed by endMonoTime features
+  # value is startMonoTime feature match
+  # -1 if no match
+  matches @5: List(Int16);
+}
+
+struct OrbFeatures @0xcd60164a8a0159ef {
+  timestampEof @0 :UInt64;
+  # transposed arrays of normalized image coordinates
+  # len(xs) == len(ys) == len(descriptors) * 32
+  xs @1 :List(Float32);
+  ys @2 :List(Float32);
+  descriptors @3 :Data;
+  octaves @4 :List(Int8);
+
+  # match index to last OrbFeatures
+  # -1 if no match
+  timestampLastEof @5 :UInt64;
+  matches @6: List(Int16);
+}
+
+struct OrbFeaturesSummary @0xd500d30c5803fa4f {
+  timestampEof @0 :UInt64;
+  timestampLastEof @1 :UInt64;
+
+  featureCount @2 :UInt16;
+  matchCount @3 :UInt16;
+  computeNs @4 :UInt64;
+}
+
+struct OrbKeyFrame @0xc8233c0345e27e24 {
+  # this is a globally unique id for the KeyFrame
+  id @0: UInt64;
+
+  # this is the location of the KeyFrame
+  pos @1: ECEFPoint;
+
+  # these are the features in the world
+  # len(dpos) == len(descriptors) * 32
+  dpos @2 :List(ECEFPoint);
+  descriptors @3 :Data;
+}
+
+struct KalmanOdometry @0x92e21bb7ea38793a {
+  trans @0 :List(Float32); # m/s in device frame
+  rot @1 :List(Float32); # rad/s in device frame
+  transStd @2 :List(Float32); # std m/s in device frame
+  rotStd @3 :List(Float32); # std rad/s in device frame
+}
+
+struct OrbObservation @0x9b326d4e436afec7 {
+  observationMonoTime @0 :UInt64;
+  normalizedCoordinates @1 :List(Float32);
+  locationECEF @2 :List(Float64);
+  matchDistance @3: UInt32;
+}
+
+struct CalibrationFeatures @0x8fdfadb254ea867a {
+  frameId @0 :UInt32;
+
+  p0 @1 :List(Float32);
+  p1 @2 :List(Float32);
+  status @3 :List(Int8);
+}
+
+struct NavStatus @0xbd8822120928120c {
+  isNavigating @0 :Bool;
+  currentAddress @1 :Address;
+
+  struct Address @0xce7cd672cacc7814 {
+    title @0 :Text;
+    lat @1 :Float64;
+    lng @2 :Float64;
+    house @3 :Text;
+    address @4 :Text;
+    street @5 :Text;
+    city @6 :Text;
+    state @7 :Text;
+    country @8 :Text;
+  }
+}
+
+struct NavUpdate @0xdb98be6565516acb {
+  isNavigating @0 :Bool;
+  curSegment @1 :Int32;
+  segments @2 :List(Segment);
+
+  struct LatLng @0x9eaef9187cadbb9b {
+    lat @0 :Float64;
+    lng @1 :Float64;
+  }
+
+  struct Segment @0xa5b39b4fc4d7da3f {
+    from @0 :LatLng;
+    to @1 :LatLng;
+    updateTime @2 :Int32;
+    distance @3 :Int32;
+    crossTime @4 :Int32;
+    exitNo @5 :Int32;
+    instruction @6 :Instruction;
+
+    parts @7 :List(LatLng);
+
+    enum Instruction @0xc5417a637451246f {
+      turnLeft @0;
+      turnRight @1;
+      keepLeft @2;
+      keepRight @3;
+      straight @4;
+      roundaboutExitNumber @5;
+      roundaboutExit @6;
+      roundaboutTurnLeft @7;
+      unkn8 @8;
+      roundaboutStraight @9;
+      unkn10 @10;
+      roundaboutTurnRight @11;
+      unkn12 @12;
+      roundaboutUturn @13;
+      unkn14 @14;
+      arrive @15;
+      exitLeft @16;
+      exitRight @17;
+      unkn18 @18;
+      uturn @19;
+      # ...
+    }
+  }
+}
+
+struct TrafficEvent @0xacfa74a094e62626 {
+  type @0 :Type;
+  distance @1 :Float32;
+  action @2 :Action;
+  resuming @3 :Bool;
+
+  enum Type @0xd85d75253435bf4b {
+    stopSign @0;
+    lightRed @1;
+    lightYellow @2;
+    lightGreen @3;
+    stopLight @4;
+  }
+
+  enum Action @0xa6f6ce72165ccb49 {
+    none @0;
+    yield @1;
+    stop @2;
+    resumeReady @3;
+  }
+
+}
+
+
+struct AndroidGnss @0xdfdf30d03fc485bd {
+  union {
+    measurements @0 :Measurements;
+    navigationMessage @1 :NavigationMessage;
+  }
+
+  struct Measurements @0xa20710d4f428d6cd {
+    clock @0 :Clock;
+    measurements @1 :List(Measurement);
+
+    struct Clock @0xa0e27b453a38f450 {
+      timeNanos @0 :Int64;
+      hardwareClockDiscontinuityCount @1 :Int32;
+
+      hasTimeUncertaintyNanos @2 :Bool;
+      timeUncertaintyNanos @3 :Float64;
+
+      hasLeapSecond @4 :Bool;
+      leapSecond @5 :Int32;
+
+      hasFullBiasNanos @6 :Bool;
+      fullBiasNanos @7 :Int64;
+
+      hasBiasNanos @8 :Bool;
+      biasNanos @9 :Float64;
+
+      hasBiasUncertaintyNanos @10 :Bool;
+      biasUncertaintyNanos @11 :Float64;
+
+      hasDriftNanosPerSecond @12 :Bool;
+      driftNanosPerSecond @13 :Float64;
+
+      hasDriftUncertaintyNanosPerSecond @14 :Bool;
+      driftUncertaintyNanosPerSecond @15 :Float64;
+    }
+
+    struct Measurement @0xd949bf717d77614d {
+      svId @0 :Int32;
+      constellation @1 :Constellation;
+
+      timeOffsetNanos @2 :Float64;
+      state @3 :Int32;
+      receivedSvTimeNanos @4 :Int64;
+      receivedSvTimeUncertaintyNanos @5 :Int64;
+      cn0DbHz @6 :Float64;
+      pseudorangeRateMetersPerSecond @7 :Float64;
+      pseudorangeRateUncertaintyMetersPerSecond @8 :Float64;
+      accumulatedDeltaRangeState @9 :Int32;
+      accumulatedDeltaRangeMeters @10 :Float64;
+      accumulatedDeltaRangeUncertaintyMeters @11 :Float64;
+
+      hasCarrierFrequencyHz @12 :Bool;
+      carrierFrequencyHz @13 :Float32;
+      hasCarrierCycles @14 :Bool;
+      carrierCycles @15 :Int64;
+      hasCarrierPhase @16 :Bool;
+      carrierPhase @17 :Float64;
+      hasCarrierPhaseUncertainty @18 :Bool;
+      carrierPhaseUncertainty @19 :Float64;
+      hasSnrInDb @20 :Bool;
+      snrInDb @21 :Float64;
+
+      multipathIndicator @22 :MultipathIndicator;
+
+      enum Constellation @0x9ef1f3ff0deb5ffb {
+        unknown @0;
+        gps @1;
+        sbas @2;
+        glonass @3;
+        qzss @4;
+        beidou @5;
+        galileo @6;
+      }
+
+      enum State @0xcbb9490adce12d72 {
+        unknown @0;
+        codeLock @1;
+        bitSync @2;
+        subframeSync @3;
+        towDecoded @4;
+        msecAmbiguous @5;
+        symbolSync @6;
+        gloStringSync @7;
+        gloTodDecoded @8;
+        bdsD2BitSync @9;
+        bdsD2SubframeSync @10;
+        galE1bcCodeLock @11;
+        galE1c2ndCodeLock @12;
+        galE1bPageSync @13;
+        sbasSync @14;
+      }
+
+      enum MultipathIndicator @0xc04e7b6231d4caa8 {
+        unknown @0;
+        detected @1;
+        notDetected @2;
+      }
+    }
+  }
+
+  struct NavigationMessage @0xe2517b083095fd4e {
+    type @0 :Int32;
+    svId @1 :Int32;
+    messageId @2 :Int32;
+    submessageId @3 :Int32;
+    data @4 :Data;
+    status @5 :Status;
+
+    enum Status @0xec1ff7996b35366f {
+      unknown @0;
+      parityPassed @1;
+      parityRebuilt @2;
+    }
+  }
+}
+
+struct QcomGnss @0xde94674b07ae51c1 {
+  logTs @0 :UInt64;
+  union {
+    measurementReport @1 :MeasurementReport;
+    clockReport @2 :ClockReport;
+    drMeasurementReport @3 :DrMeasurementReport;
+    drSvPoly @4 :DrSvPolyReport;
+    rawLog @5 :Data;
+  }
+
+  enum MeasurementSource @0xd71a12b6faada7ee {
+    gps @0;
+    glonass @1;
+    beidou @2;
+  }
+
+  enum SVObservationState @0xe81e829a0d6c83e9 {
+    idle @0;
+    search @1;
+    searchVerify @2;
+    bitEdge @3;
+    trackVerify @4;
+    track @5;
+    restart @6;
+    dpo @7;
+    glo10msBe @8;
+    glo10msAt @9;
+  }
+
+  struct MeasurementStatus @0xe501010e1bcae83b {
+    subMillisecondIsValid @0 :Bool;
+    subBitTimeIsKnown @1 :Bool;
+    satelliteTimeIsKnown @2 :Bool;
+    bitEdgeConfirmedFromSignal @3 :Bool;
+    measuredVelocity @4 :Bool;
+    fineOrCoarseVelocity @5 :Bool;
+    lockPointValid @6 :Bool;
+    lockPointPositive @7 :Bool;
+    lastUpdateFromDifference @8 :Bool;
+    lastUpdateFromVelocityDifference @9 :Bool;
+    strongIndicationOfCrossCorelation @10 :Bool;
+    tentativeMeasurement @11 :Bool;
+    measurementNotUsable @12 :Bool;
+    sirCheckIsNeeded @13 :Bool;
+    probationMode @14 :Bool;
+
+    glonassMeanderBitEdgeValid @15 :Bool;
+    glonassTimeMarkValid @16 :Bool;
+
+    gpsRoundRobinRxDiversity @17 :Bool;
+    gpsRxDiversity @18 :Bool;
+    gpsLowBandwidthRxDiversityCombined @19 :Bool;
+    gpsHighBandwidthNu4 @20 :Bool;
+    gpsHighBandwidthNu8 @21 :Bool;
+    gpsHighBandwidthUniform @22 :Bool;
+    multipathIndicator @23 :Bool;
+
+    imdJammingIndicator @24 :Bool;
+    lteB13TxJammingIndicator @25 :Bool;
+    freshMeasurementIndicator @26 :Bool;
+
+    multipathEstimateIsValid @27 :Bool;
+    directionIsValid @28 :Bool;
+  }
+
+  struct MeasurementReport @0xf580d7d86b7b8692 {
+    source @0 :MeasurementSource;
+
+    fCount @1 :UInt32;
+
+    gpsWeek @2 :UInt16;
+    glonassCycleNumber @3 :UInt8;
+    glonassNumberOfDays @4 :UInt16;
+
+    milliseconds @5 :UInt32;
+    timeBias @6 :Float32;
+    clockTimeUncertainty @7 :Float32;
+    clockFrequencyBias @8 :Float32;
+    clockFrequencyUncertainty @9 :Float32;
+
+    sv @10 :List(SV);
+
+    struct SV @0xf10c595ae7bb2c27 {
+      svId @0 :UInt8;
+      observationState @2 :SVObservationState;
+      observations @3 :UInt8;
+      goodObservations @4 :UInt8;
+      gpsParityErrorCount @5 :UInt16;
+      glonassFrequencyIndex @1 :Int8;
+      glonassHemmingErrorCount @6 :UInt8;
+      filterStages @7 :UInt8;
+      carrierNoise @8 :UInt16;
+      latency @9 :Int16;
+      predetectInterval @10 :UInt8;
+      postdetections @11 :UInt16;
+
+      unfilteredMeasurementIntegral @12 :UInt32;
+      unfilteredMeasurementFraction @13 :Float32;
+      unfilteredTimeUncertainty @14 :Float32;
+      unfilteredSpeed @15 :Float32;
+      unfilteredSpeedUncertainty @16 :Float32;
+      measurementStatus @17 :MeasurementStatus;
+      multipathEstimate @18 :UInt32;
+      azimuth @19 :Float32;
+      elevation @20 :Float32;
+      carrierPhaseCyclesIntegral @21 :Int32;
+      carrierPhaseCyclesFraction @22 :UInt16;
+      fineSpeed @23 :Float32;
+      fineSpeedUncertainty @24 :Float32;
+      cycleSlipCount @25 :UInt8;
+    }
+
+  }
+
+  struct ClockReport @0xca965e4add8f4f0b {
+    hasFCount @0 :Bool;
+    fCount @1 :UInt32;
+
+    hasGpsWeek @2 :Bool;
+    gpsWeek @3 :UInt16;
+    hasGpsMilliseconds @4 :Bool;
+    gpsMilliseconds @5 :UInt32;
+    gpsTimeBias @6 :Float32;
+    gpsClockTimeUncertainty @7 :Float32;
+    gpsClockSource @8 :UInt8;
+
+    hasGlonassYear @9 :Bool;
+    glonassYear @10 :UInt8;
+    hasGlonassDay @11 :Bool;
+    glonassDay @12 :UInt16;
+    hasGlonassMilliseconds @13 :Bool;
+    glonassMilliseconds @14 :UInt32;
+    glonassTimeBias @15 :Float32;
+    glonassClockTimeUncertainty @16 :Float32;
+    glonassClockSource @17 :UInt8;
+
+    bdsWeek @18 :UInt16;
+    bdsMilliseconds @19 :UInt32;
+    bdsTimeBias @20 :Float32;
+    bdsClockTimeUncertainty @21 :Float32;
+    bdsClockSource @22 :UInt8;
+
+    galWeek @23 :UInt16;
+    galMilliseconds @24 :UInt32;
+    galTimeBias @25 :Float32;
+    galClockTimeUncertainty @26 :Float32;
+    galClockSource @27 :UInt8;
+
+    clockFrequencyBias @28 :Float32;
+    clockFrequencyUncertainty @29 :Float32;
+    frequencySource @30 :UInt8;
+    gpsLeapSeconds @31 :UInt8;
+    gpsLeapSecondsUncertainty @32 :UInt8;
+    gpsLeapSecondsSource @33 :UInt8;
+
+    gpsToGlonassTimeBiasMilliseconds @34 :Float32;
+    gpsToGlonassTimeBiasMillisecondsUncertainty @35 :Float32;
+    gpsToBdsTimeBiasMilliseconds @36 :Float32;
+    gpsToBdsTimeBiasMillisecondsUncertainty @37 :Float32;
+    bdsToGloTimeBiasMilliseconds @38 :Float32;
+    bdsToGloTimeBiasMillisecondsUncertainty @39 :Float32;
+    gpsToGalTimeBiasMilliseconds @40 :Float32;
+    gpsToGalTimeBiasMillisecondsUncertainty @41 :Float32;
+    galToGloTimeBiasMilliseconds @42 :Float32;
+    galToGloTimeBiasMillisecondsUncertainty @43 :Float32;
+    galToBdsTimeBiasMilliseconds @44 :Float32;
+    galToBdsTimeBiasMillisecondsUncertainty @45 :Float32;
+
+    hasRtcTime @46 :Bool;
+    systemRtcTime @47 :UInt32;
+    fCountOffset @48 :UInt32;
+    lpmRtcCount @49 :UInt32;
+    clockResets @50 :UInt32;
+  }
+
+  struct DrMeasurementReport @0x8053c39445c6c75c {
+
+    reason @0 :UInt8;
+    seqNum @1 :UInt8;
+    seqMax @2 :UInt8;
+    rfLoss @3 :UInt16;
+
+    systemRtcValid @4 :Bool;
+    fCount @5 :UInt32;
+    clockResets @6 :UInt32;
+    systemRtcTime @7 :UInt64;
+
+    gpsLeapSeconds @8 :UInt8;
+    gpsLeapSecondsUncertainty @9 :UInt8;
+    gpsToGlonassTimeBiasMilliseconds @10 :Float32;
+    gpsToGlonassTimeBiasMillisecondsUncertainty @11 :Float32;
+
+    gpsWeek @12 :UInt16;
+    gpsMilliseconds @13 :UInt32;
+    gpsTimeBiasMs @14 :UInt32;
+    gpsClockTimeUncertaintyMs @15 :UInt32;
+    gpsClockSource @16 :UInt8;
+
+    glonassClockSource @17 :UInt8;
+    glonassYear @18 :UInt8;
+    glonassDay @19 :UInt16;
+    glonassMilliseconds @20 :UInt32;
+    glonassTimeBias @21 :Float32;
+    glonassClockTimeUncertainty @22 :Float32;
+
+    clockFrequencyBias @23 :Float32;
+    clockFrequencyUncertainty @24 :Float32;
+    frequencySource @25 :UInt8;
+
+    source @26 :MeasurementSource;
+
+    sv @27 :List(SV);
+
+    struct SV @0xf08b81df8cbf459c {
+      svId @0 :UInt8;
+      glonassFrequencyIndex @1 :Int8;
+      observationState @2 :SVObservationState;
+      observations @3 :UInt8;
+      goodObservations @4 :UInt8;
+      filterStages @5 :UInt8;
+      predetectInterval @6 :UInt8;
+      cycleSlipCount @7 :UInt8;
+      postdetections @8 :UInt16;
+
+      measurementStatus @9 :MeasurementStatus;
+
+      carrierNoise @10 :UInt16;
+      rfLoss @11 :UInt16;
+      latency @12 :Int16;
+
+      filteredMeasurementFraction @13 :Float32;
+      filteredMeasurementIntegral @14 :UInt32;
+      filteredTimeUncertainty @15 :Float32;
+      filteredSpeed @16 :Float32;
+      filteredSpeedUncertainty @17 :Float32;
+
+      unfilteredMeasurementFraction @18 :Float32;
+      unfilteredMeasurementIntegral @19 :UInt32;
+      unfilteredTimeUncertainty @20 :Float32;
+      unfilteredSpeed @21 :Float32;
+      unfilteredSpeedUncertainty @22 :Float32;
+
+      multipathEstimate @23 :UInt32;
+      azimuth @24 :Float32;
+      elevation @25 :Float32;
+      dopplerAcceleration @26 :Float32;
+      fineSpeed @27 :Float32;
+      fineSpeedUncertainty @28 :Float32;
+
+      carrierPhase @29 :Float64;
+      fCount @30 :UInt32;
+
+      parityErrorCount @31 :UInt16;
+      goodParity @32 :Bool;
+    }
+  }
+
+  struct DrSvPolyReport @0xb1fb80811a673270 {
+    svId @0 :UInt16;
+    frequencyIndex @1 :Int8;
+
+    hasPosition @2 :Bool;
+    hasIono @3 :Bool;
+    hasTropo @4 :Bool;
+    hasElevation @5 :Bool;
+    polyFromXtra @6 :Bool;
+    hasSbasIono @7 :Bool;
+
+    iode @8 :UInt16;
+    t0 @9 :Float64;
+    xyz0 @10 :List(Float64);
+    xyzN @11 :List(Float64);
+    other @12 :List(Float32);
+
+    positionUncertainty @13 :Float32;
+    ionoDelay @14 :Float32;
+    ionoDot @15 :Float32;
+    sbasIonoDelay @16 :Float32;
+    sbasIonoDot @17 :Float32;
+    tropoDelay @18 :Float32;
+    elevation @19 :Float32;
+    elevationDot @20 :Float32;
+    elevationUncertainty @21 :Float32;
+    velocityCoeff @22 :List(Float64);
+  }
+}
+
+struct LidarPts @0xe3d6685d4e9d8f7a {
+  r @0 :List(UInt16);        # uint16   m*500.0
+  theta @1 :List(UInt16);    # uint16 deg*100.0
+  reflect @2 :List(UInt8);   # uint8      0-255
+
+  # For storing out of file.
+  idx @3 :UInt64;
+
+  # For storing in file
+  pkt @4 :Data;
+}
+
+

cereal/messaging/__init__.py

@@ -3,6 +3,8 @@ from .messaging_pyx import Context, Poller, SubSocket, PubSocket  # pylint: disa
 from .messaging_pyx import MultiplePublishersError, MessagingError  # pylint: disable=no-name-in-module, import-error
 import capnp
 
+from typing import Optional, List, Union
+
 from cereal import log
 from cereal.services import service_list
 
@@ -19,7 +21,7 @@ except ImportError:
 
 context = Context()
 
-def new_message(service=None, size=None):
+def new_message(service: Optional[str] = None, size: Optional[int] = None) -> capnp.lib.capnp._DynamicStructBuilder:
   dat = log.Event.new_message()
   dat.logMonoTime = int(sec_since_boot() * 1e9)
   dat.valid = True
@@ -30,15 +32,15 @@ def new_message(service=None, size=None):
       dat.init(service, size)
   return dat
 
-def pub_sock(endpoint):
+def pub_sock(endpoint: str) -> PubSocket:
   sock = PubSocket()
   sock.connect(context, endpoint)
   return sock
 
-def sub_sock(endpoint, poller=None, addr="127.0.0.1", conflate=False, timeout=None):
+def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
+             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
   sock = SubSocket()
-  addr = addr.encode('utf8')
-  sock.connect(context, endpoint, addr, conflate)
+  sock.connect(context, endpoint, addr.encode('utf8'), conflate)
 
   if timeout is not None:
     sock.setTimeout(timeout)
@@ -48,9 +50,9 @@ def sub_sock(endpoint, poller=None, addr="127.0.0.1", conflate=False, timeout=No
   return sock
 
 
-def drain_sock_raw(sock, wait_for_one=False):
+def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]:
   """Receive all message currently available on the queue"""
-  ret = []
+  ret: List[bytes] = []
   while 1:
     if wait_for_one and len(ret) == 0:
       dat = sock.receive()
@@ -64,9 +66,9 @@ def drain_sock_raw(sock, wait_for_one=False):
 
   return ret
 
-def drain_sock(sock, wait_for_one=False):
+def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
   """Receive all message currently available on the queue"""
-  ret = []
+  ret: List[capnp.lib.capnp._DynamicStructReader] = []
   while 1:
     if wait_for_one and len(ret) == 0:
       dat = sock.receive()
@@ -83,7 +85,7 @@ def drain_sock(sock, wait_for_one=False):
 
 
 # TODO: print when we drop packets?
-def recv_sock(sock, wait=False):
+def recv_sock(sock: SubSocket, wait: bool = False) -> Union[None, capnp.lib.capnp._DynamicStructReader]:
   """Same as drain sock, but only returns latest message. Consider using conflate instead."""
   dat = None
 
@@ -103,35 +105,28 @@ def recv_sock(sock, wait=False):
 
   return dat
 
-def recv_one(sock):
+def recv_one(sock: SubSocket) -> Union[None, capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive()
   if dat is not None:
     dat = log.Event.from_bytes(dat)
   return dat
 
-def recv_one_or_none(sock):
+def recv_one_or_none(sock: SubSocket) -> Union[None, capnp.lib.capnp._DynamicStructReader]:
   dat = sock.receive(non_blocking=True)
   if dat is not None:
     dat = log.Event.from_bytes(dat)
   return dat
 
-def recv_one_retry(sock):
+def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
   """Keep receiving until we get a message"""
   while True:
     dat = sock.receive()
     if dat is not None:
       return log.Event.from_bytes(dat)
 
-# TODO: This does not belong in messaging
-def get_one_can(logcan):
-  while True:
-    can = recv_one_retry(logcan)
-    if len(can.can) > 0:
-      return can
-
 class SubMaster():
-  def __init__(self, services, ignore_alive=None, addr="127.0.0.1"):
-    self.poller = Poller()
+  def __init__(self, services: List[str], poll: Optional[List[str]] = None,
+               ignore_alive: Optional[List[str]] = None, addr:str ="127.0.0.1"):
     self.frame = -1
     self.updated = {s: False for s in services}
     self.rcv_time = {s: 0. for s in services}
@@ -140,8 +135,12 @@ class SubMaster():
     self.sock = {}
     self.freq = {}
     self.data = {}
-    self.logMonoTime = {}
     self.valid = {}
+    self.logMonoTime = {}
+
+    self.poller = Poller()
+    self.non_polled_services = [s for s in services if poll is not None and
+                                len(poll) and s not in poll]
 
     if ignore_alive is not None:
       self.ignore_alive = ignore_alive
@@ -150,30 +149,33 @@ class SubMaster():
 
     for s in services:
       if addr is not None:
-        self.sock[s] = sub_sock(s, poller=self.poller, addr=addr, conflate=True)
+        p = self.poller if s not in self.non_polled_services else None
+        self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
       self.freq[s] = service_list[s].frequency
 
       try:
         data = new_message(s)
       except capnp.lib.capnp.KjException:  # pylint: disable=c-extension-no-member
-        # lists
-        data = new_message(s, 0)
+        data = new_message(s, 0) # lists
 
       self.data[s] = getattr(data, s)
       self.logMonoTime[s] = 0
       self.valid[s] = data.valid
 
-  def __getitem__(self, s):
+  def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
     return self.data[s]
 
-  def update(self, timeout=1000):
+  def update(self, timeout: int = 1000) -> None:
     msgs = []
     for sock in self.poller.poll(timeout):
       msgs.append(recv_one_or_none(sock))
+
+    # non-blocking receive for non-polled sockets
+    for s in self.non_polled_services:
+      msgs.append(recv_one_or_none(self.sock[s]))
     self.update_msgs(sec_since_boot(), msgs)
 
-  def update_msgs(self, cur_time, msgs):
-    # TODO: add optional input that specify the service to wait for
+  def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1
     self.updated = dict.fromkeys(self.updated, False)
     for msg in msgs:
@@ -196,30 +198,28 @@ class SubMaster():
       else:
         self.alive[s] = True
 
-  def all_alive(self, service_list=None):
+  def all_alive(self, service_list=None) -> bool:
     if service_list is None:  # check all
       service_list = self.alive.keys()
     return all(self.alive[s] for s in service_list if s not in self.ignore_alive)
 
-  def all_valid(self, service_list=None):
+  def all_valid(self, service_list=None) -> bool:
     if service_list is None:  # check all
       service_list = self.valid.keys()
     return all(self.valid[s] for s in service_list)
 
-  def all_alive_and_valid(self, service_list=None):
+  def all_alive_and_valid(self, service_list=None) -> bool:
     if service_list is None:  # check all
       service_list = self.alive.keys()
     return self.all_alive(service_list=service_list) and self.all_valid(service_list=service_list)
 
-
 class PubMaster():
-  def __init__(self, services):
+  def __init__(self, services: List[str]):
     self.sock = {}
     for s in services:
       self.sock[s] = pub_sock(s)
 
-  def send(self, s, dat):
-    # accept either bytes or capnp builder
+  def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None:
     if not isinstance(dat, bytes):
       dat = dat.to_bytes()
     self.sock[s].send(dat)

cereal/messaging/impl_msgq.cc

@@ -5,9 +5,10 @@
 #include <csignal>
 #include <cerrno>
 
-
+#include "services.h"
 #include "impl_msgq.hpp"
 
+
 volatile sig_atomic_t msgq_do_exit = 0;
 
 void sig_handler(int signal) {
@@ -15,6 +16,25 @@ void sig_handler(int signal) {
   msgq_do_exit = 1;
 }
 
+static bool service_exists(std::string path){
+  for (const auto& it : services) {
+    if (it.name == path) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static size_t get_size(std::string endpoint){
+  size_t sz = DEFAULT_SEGMENT_SIZE;
+
+  if (endpoint == "roadCameraState" || endpoint == "driverCameraState" || endpoint == "wideRoadCameraState"){
+    sz *= 10;
+  }
+
+  return sz;
+}
+
 
 MSGQContext::MSGQContext() {
 }
@@ -49,13 +69,16 @@ MSGQMessage::~MSGQMessage() {
   this->close();
 }
 
-
-int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate){
+int MSGQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
   assert(context);
   assert(address == "127.0.0.1");
 
+  if (check_endpoint && !service_exists(std::string(endpoint))){
+    std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl;
+  }
+
   q = new msgq_queue_t;
-  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  int r = msgq_new_queue(q, endpoint.c_str(), get_size(endpoint));
   if (r != 0){
     return r;
   }
@@ -139,11 +162,15 @@ MSGQSubSocket::~MSGQSubSocket(){
   }
 }
 
-int MSGQPubSocket::connect(Context *context, std::string endpoint){
+int MSGQPubSocket::connect(Context *context, std::string endpoint, bool check_endpoint){
   assert(context);
 
+  if (check_endpoint && !service_exists(std::string(endpoint))){
+    std::cout << "Warning, " << std::string(endpoint) << " is not in service list." << std::endl;
+  }
+
   q = new msgq_queue_t;
-  int r = msgq_new_queue(q, endpoint.c_str(), DEFAULT_SEGMENT_SIZE);
+  int r = msgq_new_queue(q, endpoint.c_str(), get_size(endpoint));
   if (r != 0){
     return r;
   }

cereal/messaging/impl_msgq.hpp

@@ -34,7 +34,7 @@ private:
   msgq_queue_t * q = NULL;
   int timeout;
 public:
-  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false);
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true);
   void setTimeout(int timeout);
   void * getRawSocket() {return (void*)q;}
   Message *receive(bool non_blocking=false);
@@ -45,7 +45,7 @@ class MSGQPubSocket : public PubSocket {
 private:
   msgq_queue_t * q = NULL;
 public:
-  int connect(Context *context, std::string endpoint);
+  int connect(Context *context, std::string endpoint, bool check_endpoint=true);
   int sendMessage(Message *message);
   int send(char *data, size_t size);
   ~MSGQPubSocket();

cereal/messaging/impl_zmq.cc

@@ -54,7 +54,7 @@ ZMQMessage::~ZMQMessage() {
 }
 
 
-int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate){
+int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
   sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
   if (sock == NULL){
     return -1;
@@ -71,7 +71,11 @@ int ZMQSubSocket::connect(Context *context, std::string endpoint, std::string ad
   zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
 
   full_endpoint = "tcp://" + address + ":";
-  full_endpoint += std::to_string(get_port(endpoint));
+  if (check_endpoint){
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
 
   return zmq_connect(sock, full_endpoint.c_str());
 }
@@ -103,14 +107,18 @@ ZMQSubSocket::~ZMQSubSocket(){
   zmq_close(sock);
 }
 
-int ZMQPubSocket::connect(Context *context, std::string endpoint){
+int ZMQPubSocket::connect(Context *context, std::string endpoint, bool check_endpoint){
   sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
   if (sock == NULL){
     return -1;
   }
 
   full_endpoint = "tcp://*:";
-  full_endpoint += std::to_string(get_port(endpoint));
+  if (check_endpoint){
+    full_endpoint += std::to_string(get_port(endpoint));
+  } else {
+    full_endpoint += endpoint;
+  }
 
   return zmq_bind(sock, full_endpoint.c_str());
 }

cereal/messaging/impl_zmq.hpp

@@ -32,7 +32,7 @@ private:
   void * sock;
   std::string full_endpoint;
 public:
-  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false);
+  int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true);
   void setTimeout(int timeout);
   void * getRawSocket() {return sock;}
   Message *receive(bool non_blocking=false);
@@ -44,7 +44,7 @@ private:
   void * sock;
   std::string full_endpoint;
 public:
-  int connect(Context *context, std::string endpoint);
+  int connect(Context *context, std::string endpoint, bool check_endpoint=true);
   int sendMessage(Message *message);
   int send(char *data, size_t size);
   ~ZMQPubSocket();

cereal/messaging/messaging.cc

@@ -8,9 +8,13 @@ const bool MUST_USE_ZMQ = true;
 const bool MUST_USE_ZMQ = false;
 #endif
 
+bool messaging_use_zmq(){
+  return std::getenv("ZMQ") || MUST_USE_ZMQ;
+}
+
 Context * Context::create(){
   Context * c;
-  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+  if (messaging_use_zmq()){
     c = new ZMQContext();
   } else {
     c = new MSGQContext();
@@ -20,7 +24,7 @@ Context * Context::create(){
 
 SubSocket * SubSocket::create(){
   SubSocket * s;
-  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+  if (messaging_use_zmq()){
     s = new ZMQSubSocket();
   } else {
     s = new MSGQSubSocket();
@@ -28,33 +32,9 @@ SubSocket * SubSocket::create(){
   return s;
 }
 
-SubSocket * SubSocket::create(Context * context, std::string endpoint){
+SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address, bool conflate, bool check_endpoint){
   SubSocket *s = SubSocket::create();
-  int r = s->connect(context, endpoint, "127.0.0.1");
-
-  if (r == 0) {
-    return s;
-  } else {
-    delete s;
-    return NULL;
-  }
-}
-
-SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address){
-  SubSocket *s = SubSocket::create();
-  int r = s->connect(context, endpoint, address);
-
-  if (r == 0) {
-    return s;
-  } else {
-    delete s;
-    return NULL;
-  }
-}
-
-SubSocket * SubSocket::create(Context * context, std::string endpoint, std::string address, bool conflate){
-  SubSocket *s = SubSocket::create();
-  int r = s->connect(context, endpoint, address, conflate);
+  int r = s->connect(context, endpoint, address, conflate, check_endpoint);
 
   if (r == 0) {
     return s;
@@ -66,7 +46,7 @@ SubSocket * SubSocket::create(Context * context, std::string endpoint, std::stri
 
 PubSocket * PubSocket::create(){
   PubSocket * s;
-  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+  if (messaging_use_zmq()){
     s = new ZMQPubSocket();
   } else {
     s = new MSGQPubSocket();
@@ -74,9 +54,9 @@ PubSocket * PubSocket::create(){
   return s;
 }
 
-PubSocket * PubSocket::create(Context * context, std::string endpoint){
+PubSocket * PubSocket::create(Context * context, std::string endpoint, bool check_endpoint){
   PubSocket *s = PubSocket::create();
-  int r = s->connect(context, endpoint);
+  int r = s->connect(context, endpoint, check_endpoint);
 
   if (r == 0) {
     return s;
@@ -88,7 +68,7 @@ PubSocket * PubSocket::create(Context * context, std::string endpoint){
 
 Poller * Poller::create(){
   Poller * p;
-  if (std::getenv("ZMQ") || MUST_USE_ZMQ){
+  if (messaging_use_zmq()){
     p = new ZMQPoller();
   } else {
     p = new MSGQPoller();

cereal/messaging/messaging.hpp

@@ -6,8 +6,14 @@
 #include <capnp/serialize.h>
 #include "../gen/cpp/log.capnp.h"
 
+#ifdef __APPLE__
+#define CLOCK_BOOTTIME CLOCK_MONOTONIC
+#endif
+
 #define MSG_MULTIPLE_PUBLISHERS 100
 
+bool messaging_use_zmq();
+
 class Context {
 public:
   virtual void * getRawContext() = 0;
@@ -28,24 +34,23 @@ public:
 
 class SubSocket {
 public:
-  virtual int connect(Context *context, std::string endpoint, std::string address, bool conflate=false) = 0;
+  virtual int connect(Context *context, std::string endpoint, std::string address, bool conflate=false, bool check_endpoint=true) = 0;
   virtual void setTimeout(int timeout) = 0;
   virtual Message *receive(bool non_blocking=false) = 0;
   virtual void * getRawSocket() = 0;
   static SubSocket * create();
-  static SubSocket * create(Context * context, std::string endpoint);
-  static SubSocket * create(Context * context, std::string endpoint, std::string address);
-  static SubSocket * create(Context * context, std::string endpoint, std::string address, bool conflate);
+  static SubSocket * create(Context * context, std::string endpoint, std::string address="127.0.0.1", bool conflate=false, bool check_endpoint=true);
   virtual ~SubSocket(){};
 };
 
 class PubSocket {
 public:
-  virtual int connect(Context *context, std::string endpoint) = 0;
+  virtual int connect(Context *context, std::string endpoint, bool check_endpoint=true) = 0;
   virtual int sendMessage(Message *message) = 0;
   virtual int send(char *data, size_t size) = 0;
   static PubSocket * create();
-  static PubSocket * create(Context * context, std::string endpoint);
+  static PubSocket * create(Context * context, std::string endpoint, bool check_endpoint=true);
+  static PubSocket * create(Context * context, std::string endpoint, int port, bool check_endpoint=true);
   virtual ~PubSocket(){};
 };
 
@@ -59,34 +64,77 @@ public:
 };
 
 class SubMaster {
- public:
+public:
   SubMaster(const std::initializer_list<const char *> &service_list,
             const char *address = nullptr, const std::initializer_list<const char *> &ignore_alive = {});
   int update(int timeout = 1000);
   inline bool allAlive(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, false, true); }
   inline bool allValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, false); }
   inline bool allAliveAndValid(const std::initializer_list<const char *> &service_list = {}) { return all_(service_list, true, true); }
-  bool updated(const char *name) const;
   void drain();
-  cereal::Event::Reader &operator[](const char *name);
   ~SubMaster();
 
- private:
+  uint64_t frame = 0;
+  bool updated(const char *name) const;
+  uint64_t rcv_frame(const char *name) const;
+  cereal::Event::Reader &operator[](const char *name);
+
+private:
   bool all_(const std::initializer_list<const char *> &service_list, bool valid, bool alive);
   Poller *poller_ = nullptr;
-  uint64_t frame_ = 0;
   struct SubMessage;
   std::map<SubSocket *, SubMessage *> messages_;
   std::map<std::string, SubMessage *> services_;
 };
 
+class MessageBuilder : public capnp::MallocMessageBuilder {
+public:
+  MessageBuilder() = default;
+
+  cereal::Event::Builder initEvent(bool valid = true) {
+    cereal::Event::Builder event = initRoot<cereal::Event>();
+    struct timespec t;
+    clock_gettime(CLOCK_BOOTTIME, &t);
+    uint64_t current_time = t.tv_sec * 1000000000ULL + t.tv_nsec;
+    event.setLogMonoTime(current_time);
+    event.setValid(valid);
+    return event;
+  }
+
+  kj::ArrayPtr<capnp::byte> toBytes() {
+    heapArray_ = capnp::messageToFlatArray(*this);
+    return heapArray_.asBytes();
+  }
+
+private:
+  kj::Array<capnp::word> heapArray_;
+};
+
 class PubMaster {
- public:
+public:
   PubMaster(const std::initializer_list<const char *> &service_list);
   inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); }
-  int send(const char *name, capnp::MessageBuilder &msg);
+  int send(const char *name, MessageBuilder &msg);
   ~PubMaster();
 
- private:
+private:
   std::map<std::string, PubSocket *> sockets_;
 };
+
+class AlignedBuffer {
+public:
+  kj::ArrayPtr<const capnp::word> align(const char *data, const size_t size) {
+    words_size = size / sizeof(capnp::word) + 1;
+    if (aligned_buf.size() < words_size) {
+      aligned_buf = kj::heapArray<capnp::word>(words_size < 512 ? 512 : words_size);
+    }
+    memcpy(aligned_buf.begin(), data, size);
+    return aligned_buf.slice(0, words_size);
+  }
+  inline kj::ArrayPtr<const capnp::word> align(Message *m) {
+    return align(m->getData(), m->getSize());
+  }
+private:
+  kj::Array<capnp::word> aligned_buf;
+  size_t words_size;
+};

cereal/messaging/messaging_pyx.pyx

@@ -7,11 +7,11 @@ from libcpp cimport bool
 from libc cimport errno
 
 
-from messaging cimport Context as cppContext
-from messaging cimport SubSocket as cppSubSocket
-from messaging cimport PubSocket as cppPubSocket
-from messaging cimport Poller as cppPoller
-from messaging cimport Message as cppMessage
+from .messaging cimport Context as cppContext
+from .messaging cimport SubSocket as cppSubSocket
+from .messaging cimport PubSocket as cppPubSocket
+from .messaging cimport Poller as cppPoller
+from .messaging cimport Message as cppMessage
 
 
 class MessagingError(Exception):
@@ -59,12 +59,12 @@ cdef class Poller:
     cdef int t = timeout
 
     with nogil:
-        result = self.poller.poll(t)
+      result = self.poller.poll(t)
 
     for s in result:
-        socket = SubSocket()
-        socket.setPtr(s)
-        sockets.append(socket)
+      socket = SubSocket()
+      socket.setPtr(s)
+      sockets.append(socket)
 
     return sockets
 
@@ -140,9 +140,9 @@ cdef class PubSocket:
       else:
         raise MessagingError
 
-  def send(self, string data):
+  def send(self, bytes data):
     length = len(data)
-    r = self.socket.send(<char*>data.c_str(), length)
+    r = self.socket.send(<char*>data, length)
 
     if r != length:
       if errno.errno == errno.EADDRINUSE:

cereal/messaging/messaging_pyx_setup.py

@@ -1,57 +0,0 @@
-import os
-import subprocess
-import sysconfig
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-
-from Cython.Build import cythonize
-from Cython.Distutils import build_ext
-
-
-def get_ext_filename_without_platform_suffix(filename):
-  name, ext = os.path.splitext(filename)
-  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
-
-  if ext_suffix == ext:
-    return filename
-
-  ext_suffix = ext_suffix.replace(ext, '')
-  idx = name.find(ext_suffix)
-
-  if idx == -1:
-    return filename
-  else:
-    return name[:idx] + ext
-
-
-class BuildExtWithoutPlatformSuffix(build_ext):
-  def get_ext_filename(self, ext_name):
-    filename = super().get_ext_filename(ext_name)
-    return get_ext_filename_without_platform_suffix(filename)
-
-
-sourcefiles = ['messaging_pyx.pyx']
-extra_compile_args = ["-std=c++14"]
-libraries = ['zmq']
-ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()  # pylint: disable=unexpected-keyword-arg
-
-if ARCH == "aarch64" and os.path.isdir("/system"):
-  # android
-  extra_compile_args += ["-Wno-deprecated-register"]
-  libraries += ['gnustl_shared']
-
-setup(name='CAN parser',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "messaging_pyx",
-          language="c++",
-          sources=sourcefiles,
-          extra_compile_args=extra_compile_args,
-          libraries=libraries,
-          extra_objects=[
-            os.path.join(os.path.dirname(os.path.realpath(__file__)), '../', 'libmessaging.a'),
-          ]
-        )
-      ),
-      nthreads=4,
-)

cereal/messaging/msgq.cc

@@ -82,10 +82,8 @@ void msgq_wait_for_subscriber(msgq_queue_t *q){
 }
 
 
-
 int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){
   assert(size < 0xFFFFFFFF); // Buffer must be smaller than 2^32 bytes
-
   std::signal(SIGUSR2, sigusr2_handler);
 
   const char * prefix = "/dev/shm/";
@@ -102,15 +100,16 @@ int msgq_new_queue(msgq_queue_t * q, const char * path, size_t size){
   delete[] full_path;
 
   int rc = ftruncate(fd, size + sizeof(msgq_header_t));
-  if (rc < 0)
+  if (rc < 0){
+    close(fd);
     return -1;
-
+  }
   char * mem = (char*)mmap(NULL, size + sizeof(msgq_header_t), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
   close(fd);
 
-  if (mem == NULL)
+  if (mem == NULL){
     return -1;
-
+  }
   q->mmap_p = mem;
 
   msgq_header_t *header = (msgq_header_t *)mem;
@@ -418,8 +417,6 @@ int msgq_msg_recv(msgq_msg_t * msg, msgq_queue_t * q){
 
 
 int msgq_poll(msgq_pollitem_t * items, size_t nitems, int timeout){
-  assert(timeout >= 0);
-
   int num = 0;
 
   // Check if messages ready

cereal/messaging/socketmaster.cc

@@ -2,20 +2,20 @@
 #include <time.h>
 #include "messaging.hpp"
 #include "services.h"
-#ifdef __APPLE__
-#define CLOCK_BOOTTIME CLOCK_MONOTONIC
-#endif
+
 static inline uint64_t nanos_since_boot() {
   struct timespec t;
   clock_gettime(CLOCK_BOOTTIME, &t);
   return t.tv_sec * 1000000000ULL + t.tv_nsec;
 }
+
 static const service *get_service(const char *name) {
   for (const auto &it : services) {
     if (strcmp(it.name, name) == 0) return &it;
   }
   return nullptr;
 }
+
 static inline bool inList(const std::initializer_list<const char *> &list, const char *value) {
   for (auto &v : list) {
     if (strcmp(value, v) == 0) return true;
@@ -24,7 +24,7 @@ static inline bool inList(const std::initializer_list<const char *> &list, const
 }
 
 class MessageContext {
- public:
+public:
   MessageContext() { ctx_ = Context::create(); }
   ~MessageContext() { delete ctx_; }
   Context *ctx_;
@@ -39,7 +39,7 @@ struct SubMaster::SubMessage {
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
   capnp::FlatArrayMessageReader *msg_reader = nullptr;
-  kj::Array<capnp::word> buf;
+  AlignedBuffer aligned_buf;
   cereal::Event::Reader event;
 };
 
@@ -53,18 +53,17 @@ SubMaster::SubMaster(const std::initializer_list<const char *> &service_list, co
     assert(socket != 0);
     poller_->registerSocket(socket);
     SubMessage *m = new SubMessage{
-        .socket = socket,
-        .freq = serv->frequency,
-        .ignore_alive = inList(ignore_alive, name),
-        .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
-        .buf = kj::heapArray<capnp::word>(1024)};
+      .socket = socket,
+      .freq = serv->frequency,
+      .ignore_alive = inList(ignore_alive, name),
+      .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader))};
     messages_[socket] = m;
     services_[name] = m;
   }
 }
 
 int SubMaster::update(int timeout) {
-  if (++frame_ == UINT64_MAX) frame_ = 1;
+  if (++frame == UINT64_MAX) frame = 1;
   for (auto &kv : messages_) kv.second->updated = false;
 
   int updated = 0;
@@ -75,21 +74,15 @@ int SubMaster::update(int timeout) {
     if (msg == nullptr) continue;
 
     SubMessage *m = messages_.at(s);
-    const size_t size = (msg->getSize() / sizeof(capnp::word)) + 1;
-    if (m->buf.size() < size) {
-      m->buf = kj::heapArray<capnp::word>(size);
-    }
-    memcpy(m->buf.begin(), msg->getData(), msg->getSize());
-    delete msg;
-
     if (m->msg_reader) {
       m->msg_reader->~FlatArrayMessageReader();
     }
-    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(kj::ArrayPtr<capnp::word>(m->buf.begin(), size));
+    m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg));
+    delete msg;
     m->event = m->msg_reader->getRoot<cereal::Event>();
     m->updated = true;
     m->rcv_time = current_time;
-    m->rcv_frame = frame_;
+    m->rcv_frame = frame;
     m->valid = m->event.getValid();
 
     ++updated;
@@ -126,8 +119,17 @@ void SubMaster::drain() {
   }
 }
 
-bool SubMaster::updated(const char *name) const { return services_.at(name)->updated; }
-cereal::Event::Reader &SubMaster::operator[](const char *name) { return services_.at(name)->event; };
+bool SubMaster::updated(const char *name) const {
+  return services_.at(name)->updated;
+}
+
+uint64_t SubMaster::rcv_frame(const char *name) const {
+  return services_.at(name)->rcv_frame;
+}
+
+cereal::Event::Reader &SubMaster::operator[](const char *name) {
+  return services_.at(name)->event;
+};
 
 SubMaster::~SubMaster() {
   delete poller_;
@@ -151,9 +153,8 @@ PubMaster::PubMaster(const std::initializer_list<const char *> &service_list) {
   }
 }
 
-int PubMaster::send(const char *name, capnp::MessageBuilder &msg) {
-  auto words = capnp::messageToFlatArray(msg);
-  auto bytes = words.asBytes();
+int PubMaster::send(const char *name, MessageBuilder &msg) {
+  auto bytes = msg.toBytes();
   return send(name, bytes.begin(), bytes.size());
 }
 

cereal/service_list.yaml

@@ -1,177 +0,0 @@
-# TODO: these port numbers are hardcoded in c, fix this
-
-# LogRotate: 8001 is a PUSH PULL socket between loggerd and visiond
-
-# all ZMQ pub sub: port, should_log, frequency, (qlog_decimation)
-
-# frame syncing packet
-frame: [8002, true, 20., 1]
-# accel, gyro, and compass
-sensorEvents: [8003, true, 100., 100]
-# GPS data, also global timestamp
-gpsNMEA: [8004, true, 9.]  # 9 msgs each sec
-# CPU+MEM+GPU+BAT temps
-thermal: [8005, true, 2., 1]
-# List(CanData), list of can messages
-can: [8006, true, 100.]
-controlsState: [8007, true, 100., 100]
-#liveEvent: [8008, true, 0.]
-model: [8009, true, 20., 5]
-features: [8010, true, 0.]
-health: [8011, true, 2., 1]
-radarState: [8012, true, 20., 5]
-#liveUI: [8014, true, 0.]
-encodeIdx: [8015, true, 20.]
-liveTracks: [8016, true, 20.]
-sendcan: [8017, true, 100.]
-logMessage: [8018, true, 0.]
-liveCalibration: [8019, true, 4., 4]
-androidLog: [8020, true, 0.]
-carState: [8021, true, 100., 10]
-# 8022 is reserved for sshd
-carControl: [8023, true, 100., 10]
-plan: [8024, true, 20., 2]
-liveLocation: [8025, true, 0., 1]
-gpsLocation: [8026, true, 1., 1]
-ethernetData: [8027, true, 0.]
-navUpdate: [8028, true, 0.]
-qcomGnss: [8029, true, 0.]
-lidarPts: [8030, true, 0.]
-procLog: [8031, true, 0.5]
-gpsLocationExternal: [8032, true, 10., 1]
-ubloxGnss: [8033, true, 10.]
-clocks: [8034, true, 1., 1]
-liveMpc: [8035, false, 20.]
-liveLongitudinalMpc: [8036, false, 20.]
-navStatus: [8038, true, 0.]
-gpsLocationTrimble: [8039, true, 0.]
-trimbleGnss: [8041, true, 0.]
-ubloxRaw: [8042, true, 20.]
-gpsPlannerPoints: [8043, true, 0.]
-gpsPlannerPlan: [8044, true, 0.]
-applanixRaw: [8046, true, 0.]
-orbLocation: [8047, true, 0.]
-trafficEvents: [8048, true, 0.]
-liveLocationTiming: [8049, true, 0.]
-orbslamCorrection: [8050, true, 0.]
-liveLocationCorrected: [8051, true, 0.]
-orbObservation: [8052, true, 0.]
-applanixLocation: [8053, true, 0.]
-liveLocationKalman: [8054, true, 20., 2]
-uiNavigationEvent: [8055, true, 0.]
-orbOdometry: [8057, true, 0.]
-orbFeatures: [8058, false, 0.]
-orbKeyFrame: [8059, true, 0.]
-uiLayoutState: [8060, true, 0.]
-frontEncodeIdx: [8061, true, 5.]
-orbFeaturesSummary: [8062, true, 0.]
-driverState: [8063, true, 5., 1]
-liveParameters: [8064, true, 20., 2]
-liveMapData: [8065, true, 0.]
-cameraOdometry: [8066, true, 20., 5]
-pathPlan: [8067, true, 20., 2]
-kalmanOdometry: [8068, true, 0.]
-thumbnail: [8069, true, 0.2, 1]
-carEvents: [8070, true, 1., 1]
-carParams: [8071, true, 0.02, 1]
-frontFrame: [8072, true, 10.]
-dMonitoringState: [8073, true, 5., 1]
-offroadLayout: [8074, false, 0.]
-
-testModel: [8040, false, 0.]
-testLiveLocation: [8045, false, 0.]
-testJoystick: [8056, false, 0.]
-
-# 8080 is reserved for slave testing daemon
-# 8762 is reserved for logserver
-
-# manager -- base process to manage starting and stopping of all others
-#   subscribes: thermal
-
-# **** processes that communicate with the outside world ****
-
-# thermald -- decides when to start and stop onroad
-#   subscribes: health, location
-#   publishes: thermal
-
-# boardd -- communicates with the car
-#   subscribes: sendcan
-#   publishes: can, health, ubloxRaw
-
-# sensord -- publishes IMU and Magnetometer
-#   publishes: sensorEvents
-
-# gpsd -- publishes EON's gps
-#   publishes: gpsNMEA
-
-# camerad -- publishes camera frames
-#   publishes: frame, frontFrame, thumbnail
-#   subscribes: driverState
-
-# dmonitoringmodeld -- runs face detection on camera frames
-#   publishes: driverState
-
-# **** stateful data transformers ****
-
-# modeld -- runs & publishes the model
-#   publishes: model, cameraOdometry
-#   subscribes: liveCalibration, pathPlan
-
-# plannerd -- decides where to drive the car
-#   subscribes: carState, model, radarState, controlsState, liveParameters
-#   publishes:  plan, pathPlan, liveMpc, liveLongitudinalMpc
-
-# controlsd -- drives the car by sending CAN messages to panda
-#   subscribes: can, thermal, health, plan, pathPlan, dMonitoringState, liveCalibration, model
-#   publishes:  carState, carControl, sendcan, controlsState, carEvents, carParams
-
-# dmonitoringd -- processes driver monitoring data and publishes driver awareness
-#   subscribes: driverState, liveCalibration, carState, model, gpsLocation
-#   publishes: dMonitoringState
-
-# radard -- processes the radar and vision data
-#   subscribes: can, controlsState, model, liveParameters
-#   publishes:  radarState, liveTracks
-
-# params_learner -- learns vehicle params by observing the vehicle dynamics
-#   subscribes: controlsState, sensorEvents, cameraOdometry
-#   publishes: liveParameters
-
-# calibrationd -- reads posenet and applies a temporal filter on the frame region to look at
-#   subscribes: cameraOdometry
-#   publishes: liveCalibration
-
-# ubloxd -- read raw ublox data and converts them in readable format
-#   subscribes: ubloxRaw
-#   publishes: ubloxGnss
-
-# **** LOGGING SERVICE ****
-
-# loggerd
-#   subscribes: EVERYTHING
-
-# **** NON VITAL SERVICES ****
-
-# ui
-#   subscribes: thermal, model, controlsState, uiLayout, liveCalibration, radarState, liveMpc, plusFrame, liveMapData
-
-# uploader
-#   communicates through file system with loggerd
-
-# deleter
-#   communicates through file system with loggerd and uploader
-
-# logmessaged -- central logging service, can log to cloud
-#   publishes:  logMessage
-
-# logcatd -- fetches logcat info from android
-#   publishes:  androidLog
-
-# proclogd -- fetches process information
-#   publishes: procLog
-
-# tombstoned -- reports native crashes
-
-# athenad -- on request, open a sub socket and return the value
-
-# updated -- waits for network access and tries to update every hour

cereal/services.py

@@ -1,34 +1,83 @@
 #!/usr/bin/env python3
 import os
-import yaml
+from typing import Optional
 
+EON = os.path.isfile('/EON')
 
-class Service():
-  def __init__(self, port, should_log, frequency, decimation=None):
+class Service:
+  def __init__(self, port: int, should_log: bool, frequency: float, decimation: Optional[int] = None):
     self.port = port
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
 
+service_list = {
+  "roadCameraState": Service(8002, True, 20., 1),
+  "sensorEvents": Service(8003, True, 100., 100),
+  "gpsNMEA": Service(8004, True, 9.),
+  "deviceState": Service(8005, True, 2., 1),
+  "can": Service(8006, True, 100.),
+  "controlsState": Service(8007, True, 100., 100),
+  "features": Service(8010, True, 0.),
+  "pandaState": Service(8011, True, 2., 1),
+  "radarState": Service(8012, True, 20., 5),
+  "roadEncodeIdx": Service(8015, True, 20., 1),
+  "liveTracks": Service(8016, True, 20.),
+  "sendcan": Service(8017, True, 100.),
+  "logMessage": Service(8018, True, 0.),
+  "liveCalibration": Service(8019, True, 4., 4),
+  "androidLog": Service(8020, True, 0., 1),
+  "carState": Service(8021, True, 100., 10),
+  "carControl": Service(8023, True, 100., 10),
+  "longitudinalPlan": Service(8024, True, 20., 2),
+  "liveLocation": Service(8025, True, 0., 1),
+  "procLog": Service(8031, True, 0.5),
+  "gpsLocationExternal": Service(8032, True, 10., 1),
+  "ubloxGnss": Service(8033, True, 10.),
+  "clocks": Service(8034, True, 1., 1),
+  "liveMpc": Service(8035, False, 20.),
+  "liveLongitudinalMpc": Service(8036, False, 20.),
+  "ubloxRaw": Service(8042, True, 20.),
+  "liveLocationKalman": Service(8054, True, 20., 2),
+  "uiLayoutState": Service(8060, True, 0.),
+  "liveParameters": Service(8064, True, 20., 2),
+  "cameraOdometry": Service(8066, True, 20., 5),
+  "lateralPlan": Service(8067, True, 20., 2),
+  "thumbnail": Service(8069, True, 0.2, 1),
+  "carEvents": Service(8070, True, 1., 1),
+  "carParams": Service(8071, True, 0.02, 1),
+  "driverCameraState": Service(8072, True, 10. if EON else 20., 1),
+  "driverEncodeIdx": Service(8061, True, 10. if EON else 20., 1),
+  "driverState": Service(8063, True, 10. if EON else 20., 1),
+  "driverMonitoringState": Service(8073, True, 10. if EON else 20., 1),
+  "offroadLayout": Service(8074, False, 0.),
+  "wideRoadEncodeIdx": Service(8075, True, 20., 1),
+  "wideRoadCameraState": Service(8076, True, 20., 1),
+  "modelV2": Service(8077, True, 20., 20),
+  "managerState": Service(8078, True, 2., 1),
 
-service_list_path = os.path.join(os.path.dirname(__file__), "service_list.yaml")
+  "testModel": Service(8040, False, 0.),
+  "testLiveLocation": Service(8045, False, 0.),
+  "testJoystick": Service(8056, False, 0.),
+}
 
-service_list = {}
-with open(service_list_path, "r") as f:
-  for k, v in yaml.safe_load(f).items():
-    decimation = None
-    if len(v) == 4:
-      decimation = v[3]
 
-    service_list[k] = Service(v[0], v[1], v[2], decimation)
+def build_header():
+  h = ""
+  h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
+  h += "#ifndef __SERVICES_H\n"
+  h += "#define __SERVICES_H\n"
+  h += "struct service { char name[0x100]; int port; bool should_log; int frequency; int decimation; };\n"
+  h += "static struct service services[] = {\n"
+  for k, v in service_list.items():
+    should_log = "true" if v.should_log else "false"
+    decimation = -1 if v.decimation is None else v.decimation
+    h += '  { .name = "%s", .port = %d, .should_log = %s, .frequency = %d, .decimation = %d },\n' % \
+         (k, v.port, should_log, v.frequency, decimation)
+  h += "};\n"
+  h += "#endif\n"
+  return h
+
 
 if __name__ == "__main__":
-  print("/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT service_list.yaml */")
-  print("#ifndef __SERVICES_H")
-  print("#define __SERVICES_H")
-  print("struct service { char name[0x100]; int port; bool should_log; int frequency; int decimation; };")
-  print("static struct service services[] = {")
-  for k, v in service_list.items():
-    print('  { .name = "%s", .port = %d, .should_log = %s, .frequency = %d, .decimation = %d },' % (k, v.port, "true" if v.should_log else "false", v.frequency, -1 if v.decimation is None else v.decimation))
-  print("};")
-  print("#endif")
+  print(build_header())

cereal/visionipc/.gitignore

@@ -0,0 +1,2 @@
+visionipc_pyx.cpp
+*.so

cereal/visionipc/ipc.cc

@@ -1,6 +1,5 @@
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdbool.h>
 #include <string.h>
 #include <unistd.h>
 #include <assert.h>
@@ -10,16 +9,19 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 
+#ifdef __APPLE__
+#define getsocket() socket(AF_UNIX, SOCK_STREAM, 0)
+#else
+#define getsocket() socket(AF_UNIX, SOCK_SEQPACKET, 0)
+#endif
+
 #include "ipc.h"
 
 int ipc_connect(const char* socket_path) {
   int err;
 
-#ifdef __APPLE__
-  int sock = socket(AF_UNIX, SOCK_STREAM, 0);
-#else
-  int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
-#endif
+  int sock = getsocket();
+
   if (sock < 0) return -1;
   struct sockaddr_un addr = {
     .sun_family = AF_UNIX,
@@ -39,11 +41,8 @@ int ipc_bind(const char* socket_path) {
 
   unlink(socket_path);
 
-#ifdef __APPLE__
-  int sock = socket(AF_UNIX, SOCK_STREAM, 0);
-#else
-  int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
-#endif
+  int sock = getsocket();
+
   struct sockaddr_un addr = {
     .sun_family = AF_UNIX,
   };
@@ -87,7 +86,6 @@ int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fd
       cmsg->cmsg_type = SCM_RIGHTS;
       cmsg->cmsg_len = CMSG_LEN(sizeof(int) * num_fds);
       memcpy(CMSG_DATA(cmsg), fds, sizeof(int) * num_fds);
-      // printf("send clen %d -> %d\n", num_fds, cmsg->cmsg_len);
     }
     return sendmsg(fd, &msg, 0);
   } else {
@@ -102,8 +100,6 @@ int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fd
       recv_fds = (cmsg->cmsg_len - CMSG_LEN(0));
       assert(recv_fds > 0 && (recv_fds % sizeof(int)) == 0);
       recv_fds /= sizeof(int);
-      // printf("recv clen %d -> %d\n", cmsg->cmsg_len, recv_fds);
-      // assert(cmsg->cmsg_len == CMSG_LEN(sizeof(int) * num_fds));
 
       assert(fds && recv_fds <= num_fds);
       memcpy(fds, CMSG_DATA(cmsg), sizeof(int) * recv_fds);

cereal/visionipc/ipc.h

@@ -1,19 +1,7 @@
-#ifndef IPC_H
-#define IPC_H
-
-#include <stdbool.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
+#pragma once
+#include <cstddef>
 
 int ipc_connect(const char* socket_path);
 int ipc_bind(const char* socket_path);
 int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
                           int *out_num_fds);
-
-#ifdef __cplusplus
-}  // extern "C"
-#endif
-
-#endif

cereal/visionipc/test_runner.cc

@@ -0,0 +1,2 @@
+#define CATCH_CONFIG_MAIN
+#include "catch2/catch.hpp"

cereal/visionipc/visionbuf.cc

@@ -0,0 +1,40 @@
+#include "visionbuf.h"
+
+#define ALIGN(x, align) (((x) + (align)-1) & ~((align)-1))
+
+#ifdef QCOM
+// from libadreno_utils.so
+extern "C" void compute_aligned_width_and_height(int width,
+                                                 int height,
+                                                 int bpp,
+                                                 int tile_mode,
+                                                 int raster_mode,
+                                                 int padding_threshold,
+                                                 int *aligned_w,
+                                                 int *aligned_h);
+#endif
+
+void visionbuf_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h) {
+#if defined(QCOM) && !defined(QCOM_REPLAY)
+  compute_aligned_width_and_height(ALIGN(width, 32), ALIGN(height, 32), 3, 0, 0, 512, aligned_w, aligned_h);
+#else
+  *aligned_w = width; *aligned_h = height;
+#endif
+}
+
+void VisionBuf::init_rgb(size_t width, size_t height, size_t stride) {
+  this->rgb = true;
+  this->width = width;
+  this->height = height;
+  this->stride = stride;
+}
+
+void VisionBuf::init_yuv(size_t width, size_t height){
+  this->rgb = false;
+  this->width = width;
+  this->height = height;
+
+  this->y = (uint8_t *)this->addr;
+  this->u = this->y + (width * height);
+  this->v = this->u + (width / 2 * height / 2);
+}

cereal/visionipc/visionbuf.h

@@ -0,0 +1,63 @@
+#pragma once
+#include "visionipc.h"
+
+#define CL_USE_DEPRECATED_OPENCL_1_2_APIS
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#define VISIONBUF_SYNC_FROM_DEVICE 0
+#define VISIONBUF_SYNC_TO_DEVICE 1
+
+enum VisionStreamType {
+  VISION_STREAM_RGB_BACK,
+  VISION_STREAM_RGB_FRONT,
+  VISION_STREAM_RGB_WIDE,
+  VISION_STREAM_YUV_BACK,
+  VISION_STREAM_YUV_FRONT,
+  VISION_STREAM_YUV_WIDE,
+  VISION_STREAM_MAX,
+};
+
+class VisionBuf {
+ public:
+  size_t len = 0;
+  size_t mmap_len = 0;
+  void * addr = nullptr;
+  int fd = 0;
+
+  bool rgb = false;
+  size_t width = 0;
+  size_t height = 0;
+  size_t stride = 0;
+
+  // YUV
+  uint8_t * y = nullptr;
+  uint8_t * u = nullptr;
+  uint8_t * v = nullptr;
+
+  // Visionipc
+  uint64_t server_id = 0;
+  size_t idx = 0;
+  VisionStreamType type;
+
+  // OpenCL
+  cl_mem buf_cl = nullptr;
+  cl_command_queue copy_q = nullptr;
+
+  // ion
+  int handle = 0;
+  bool owner = false;
+
+  void allocate(size_t len);
+  void import();
+  void init_cl(cl_device_id device_id, cl_context ctx);
+  void init_rgb(size_t width, size_t height, size_t stride);
+  void init_yuv(size_t width, size_t height);
+  void sync(int dir);
+  void free();
+};
+
+void visionbuf_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h);

cereal/visionipc/visionbuf_cl.cc

@@ -0,0 +1,86 @@
+#include "visionbuf.h"
+
+#include <atomic>
+#include <stdio.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+
+std::atomic<int> offset = 0;
+
+static void *malloc_with_fd(size_t len, int *fd) {
+  char full_path[0x100];
+
+#ifdef __APPLE__
+  snprintf(full_path, sizeof(full_path)-1, "/tmp/visionbuf_%d_%d", getpid(), offset++);
+#else
+  snprintf(full_path, sizeof(full_path)-1, "/dev/shm/visionbuf_%d_%d", getpid(), offset++);
+#endif
+
+  *fd = open(full_path, O_RDWR | O_CREAT, 0777);
+  assert(*fd >= 0);
+
+  unlink(full_path);
+
+  ftruncate(*fd, len);
+  void *addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
+  assert(addr != MAP_FAILED);
+
+  return addr;
+}
+
+void VisionBuf::allocate(size_t len) {
+  int fd;
+  void *addr = malloc_with_fd(len, &fd);
+
+  this->len = len;
+  this->mmap_len = len;
+  this->addr = addr;
+  this->fd = fd;
+}
+
+void VisionBuf::init_cl(cl_device_id device_id, cl_context ctx){
+  int err;
+
+  this->copy_q = clCreateCommandQueue(ctx, device_id, 0, &err);
+  assert(err == 0);
+
+  this->buf_cl = clCreateBuffer(ctx, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, this->len, this->addr, &err);
+  assert(err == 0);
+}
+
+
+void VisionBuf::import(){
+  assert(this->fd >= 0);
+  this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
+  assert(this->addr != MAP_FAILED);
+}
+
+
+void VisionBuf::sync(int dir) {
+  int err = 0;
+  if (!this->buf_cl) return;
+
+  if (dir == VISIONBUF_SYNC_FROM_DEVICE) {
+    err = clEnqueueReadBuffer(this->copy_q, this->buf_cl, CL_FALSE, 0, this->len, this->addr, 0, NULL, NULL);
+  } else {
+    err = clEnqueueWriteBuffer(this->copy_q, this->buf_cl, CL_FALSE, 0, this->len, this->addr, 0, NULL, NULL);
+  }
+  assert(err == 0);
+  clFinish(this->copy_q);
+}
+
+void VisionBuf::free() {
+  if (this->buf_cl){
+    int err = clReleaseMemObject(this->buf_cl);
+    assert(err == 0);
+
+    clReleaseCommandQueue(this->copy_q);
+  }
+
+  munmap(this->addr, this->len);
+  close(this->fd);
+}

cereal/visionipc/visionbuf_ion.cc

@@ -36,7 +36,7 @@ static void ion_init() {
   }
 }
 
-VisionBuf visionbuf_allocate(size_t len) {
+void VisionBuf::allocate(size_t len) {
   int err;
 
   ion_init();
@@ -62,54 +62,58 @@ VisionBuf visionbuf_allocate(size_t len) {
 
   memset(addr, 0, ion_alloc.len);
 
-  return (VisionBuf){
-    .len = len,
-    .mmap_len = ion_alloc.len,
-    .addr = addr,
-    .handle = ion_alloc.handle,
-    .fd = ion_fd_data.fd,
-  };
+  this->owner = true;
+  this->len = len;
+  this->mmap_len = ion_alloc.len;
+  this->addr = addr;
+  this->handle = ion_alloc.handle;
+  this->fd = ion_fd_data.fd;
 }
 
-VisionBuf visionbuf_allocate_cl(size_t len, cl_device_id device_id, cl_context ctx, cl_mem *out_mem) {
-  VisionBuf r = visionbuf_allocate(len);
-  *out_mem = visionbuf_to_cl(&r, device_id, ctx);
-  r.buf_cl = *out_mem;
-  return r;
+void VisionBuf::import(){
+  int err;
+  assert(this->fd >= 0);
+
+  ion_init();
+
+  // Get handle
+  struct ion_fd_data fd_data = {0};
+  fd_data.fd = this->fd;
+  err = ioctl(ion_fd, ION_IOC_IMPORT, &fd_data);
+  assert(err == 0);
+
+  this->owner = false;
+  this->handle = fd_data.handle;
+  this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
+  assert(this->addr != MAP_FAILED);
 }
 
-cl_mem visionbuf_to_cl(const VisionBuf* buf, cl_device_id device_id, cl_context ctx) {
-  int err = 0;
+void VisionBuf::init_cl(cl_device_id device_id, cl_context ctx) {
+  int err;
 
-  assert(((uintptr_t)buf->addr % DEVICE_PAGE_SIZE_CL) == 0);
+  assert(((uintptr_t)this->addr % DEVICE_PAGE_SIZE_CL) == 0);
 
   cl_mem_ion_host_ptr ion_cl = {0};
   ion_cl.ext_host_ptr.allocation_type = CL_MEM_ION_HOST_PTR_QCOM;
   ion_cl.ext_host_ptr.host_cache_policy = CL_MEM_HOST_UNCACHED_QCOM;
-  ion_cl.ion_filedesc = buf->fd;
-  ion_cl.ion_hostptr = buf->addr;
+  ion_cl.ion_filedesc = this->fd;
+  ion_cl.ion_hostptr = this->addr;
 
-  cl_mem mem = clCreateBuffer(ctx,
+  this->buf_cl = clCreateBuffer(ctx,
                               CL_MEM_USE_HOST_PTR | CL_MEM_EXT_HOST_PTR_QCOM,
-                              buf->len, &ion_cl, &err);
+                              this->len, &ion_cl, &err);
   assert(err == 0);
-
-  return mem;
 }
 
-void visionbuf_sync(const VisionBuf* buf, int dir) {
+
+void VisionBuf::sync(int dir) {
   int err;
 
-  struct ion_fd_data fd_data = {0};
-  fd_data.fd = buf->fd;
-  err = ioctl(ion_fd, ION_IOC_IMPORT, &fd_data);
-  assert(err == 0);
-
   struct ion_flush_data flush_data = {0};
-  flush_data.handle = fd_data.handle;
-  flush_data.vaddr = buf->addr;
+  flush_data.handle = this->handle;
+  flush_data.vaddr = this->addr;
   flush_data.offset = 0;
-  flush_data.length = buf->len;
+  flush_data.length = this->len;
 
   // ION_IOC_INV_CACHES ~= DMA_FROM_DEVICE
   // ION_IOC_CLEAN_CACHES ~= DMA_TO_DEVICE
@@ -117,34 +121,28 @@ void visionbuf_sync(const VisionBuf* buf, int dir) {
 
   struct ion_custom_data custom_data = {0};
 
-  switch (dir) {
-  case VISIONBUF_SYNC_FROM_DEVICE:
-    custom_data.cmd = ION_IOC_INV_CACHES;
-    break;
-  case VISIONBUF_SYNC_TO_DEVICE:
-    custom_data.cmd = ION_IOC_CLEAN_CACHES;
-    break;
-  default:
-    assert(0);
-  }
+   assert(dir == VISIONBUF_SYNC_FROM_DEVICE || dir == VISIONBUF_SYNC_TO_DEVICE);
+   custom_data.cmd = (dir == VISIONBUF_SYNC_FROM_DEVICE) ?
+     ION_IOC_INV_CACHES : ION_IOC_CLEAN_CACHES;
 
   custom_data.arg = (unsigned long)&flush_data;
   err = ioctl(ion_fd, ION_IOC_CUSTOM, &custom_data);
   assert(err == 0);
-
-  struct ion_handle_data handle_data = {0};
-  handle_data.handle = fd_data.handle;
-  err = ioctl(ion_fd, ION_IOC_FREE, &handle_data);
-  assert(err == 0);
 }
 
-void visionbuf_free(const VisionBuf* buf) {
-  clReleaseMemObject(buf->buf_cl);
-  munmap(buf->addr, buf->mmap_len);
-  close(buf->fd);
-  struct ion_handle_data handle_data = {
-    .handle = buf->handle,
-  };
-  int ret = ioctl(ion_fd, ION_IOC_FREE, &handle_data);
-  assert(ret == 0);
+void VisionBuf::free() {
+  if (this->buf_cl){
+    int err = clReleaseMemObject(this->buf_cl);
+    assert(err == 0);
+  }
+
+  munmap(this->addr, this->mmap_len);
+  close(this->fd);
+
+  // Free the ION buffer if we also shared it
+  if (this->owner){
+    struct ion_handle_data handle_data = {.handle = this->handle};
+    int ret = ioctl(ion_fd, ION_IOC_FREE, &handle_data);
+    assert(ret == 0);
+  }
 }

cereal/visionipc/visionipc.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+
+constexpr int VISIONIPC_MAX_FDS = 64;
+
+struct VisionIpcBufExtra {
+  uint32_t frame_id;
+  uint64_t timestamp_sof;
+  uint64_t timestamp_eof;
+};
+
+struct VisionIpcPacket {
+  uint64_t server_id;
+  size_t idx;
+  struct VisionIpcBufExtra extra;
+};

cereal/visionipc/visionipc.pxd

@@ -0,0 +1,29 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool
+
+cdef extern from "visionbuf.h":
+  cdef enum VisionStreamType:
+    pass
+
+  cdef cppclass VisionBuf:
+    void * addr
+    size_t len
+
+cdef extern from "visionipc.h":
+  struct VisionIpcBufExtra:
+    uint32_t frame_id
+    uint64_t timestamp_sof
+    uint64_t timestamp_eof
+
+cdef extern from "visionipc_server.h":
+  cdef cppclass VisionIpcServer:
+    VisionIpcServer(string, void*, void*)
+    void create_buffers(VisionStreamType, size_t, bool, size_t, size_t)
+    VisionBuf * get_buffer(VisionStreamType)
+    void send(VisionBuf *, VisionIpcBufExtra *, bool)
+    void start_listener()

cereal/visionipc/visionipc_client.cc

@@ -0,0 +1,119 @@
+#include <chrono>
+#include <cassert>
+#include <iostream>
+#include <thread>
+
+#include "ipc.h"
+#include "visionipc_client.h"
+#include "visionipc_server.h"
+
+VisionIpcClient::VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id, cl_context ctx) : name(name), type(type), device_id(device_id), ctx(ctx) {
+  msg_ctx = Context::create();
+  sock = SubSocket::create(msg_ctx, get_endpoint_name(name, type), "127.0.0.1", conflate, false);
+
+  poller = Poller::create();
+  poller->registerSocket(sock);
+}
+
+// Connect is not thread safe. Do not use the buffers while calling connect
+bool VisionIpcClient::connect(bool blocking){
+  connected = false;
+
+  // Cleanup old buffers on reconnect
+  for (size_t i = 0; i < num_buffers; i++){
+    buffers[i].free();
+  }
+  num_buffers = 0;
+
+  // Connect to server socket and ask for all FDs of type
+  std::string path = "/tmp/visionipc_" + name;
+
+  int socket_fd = -1;
+  while (socket_fd < 0) {
+    socket_fd = ipc_connect(path.c_str());
+
+    if (socket_fd < 0) {
+      if (blocking){
+        std::cout << "VisionIpcClient connecting" << std::endl;
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+      } else {
+        return false;
+      }
+    }
+  }
+
+  // Send stream type to server to request FDs
+  int r = ipc_sendrecv_with_fds(true, socket_fd, &type, sizeof(type), nullptr, 0, nullptr);
+  assert(r == sizeof(type));
+
+  // Get FDs
+  int fds[VISIONIPC_MAX_FDS];
+  VisionBuf bufs[VISIONIPC_MAX_FDS];
+  r = ipc_sendrecv_with_fds(false, socket_fd, &bufs, sizeof(bufs), fds, VISIONIPC_MAX_FDS, &num_buffers);
+
+  assert(num_buffers > 0);
+  assert(r == sizeof(VisionBuf) * num_buffers);
+
+  // Import buffers
+  for (size_t i = 0; i < num_buffers; i++){
+    buffers[i] = bufs[i];
+    buffers[i].fd = fds[i];
+    buffers[i].import();
+    if (buffers[i].rgb) {
+      buffers[i].init_rgb(buffers[i].width, buffers[i].height, buffers[i].stride);
+    } else {
+      buffers[i].init_yuv(buffers[i].width, buffers[i].height);
+    }
+
+    if (device_id) buffers[i].init_cl(device_id, ctx);
+  }
+
+  connected = true;
+  return true;
+}
+
+VisionBuf * VisionIpcClient::recv(VisionIpcBufExtra * extra, const int timeout_ms){
+  auto p = poller->poll(timeout_ms);
+
+  if (!p.size()){
+    return nullptr;
+  }
+
+  Message * r = sock->receive(true);
+  if (r == nullptr){
+    return nullptr;
+  }
+
+  // Get buffer
+  assert(r->getSize() == sizeof(VisionIpcPacket));
+  VisionIpcPacket *packet = (VisionIpcPacket*)r->getData();
+
+  assert(packet->idx < num_buffers);
+  VisionBuf * buf = &buffers[packet->idx];
+
+  if (buf->server_id != packet->server_id){
+    connected = false;
+    delete r;
+    return nullptr;
+  }
+
+  if (extra) {
+    *extra = packet->extra;
+  }
+
+  buf->sync(VISIONBUF_SYNC_TO_DEVICE);
+  delete r;
+  return buf;
+}
+
+
+
+VisionIpcClient::~VisionIpcClient(){
+  for (size_t i = 0; i < num_buffers; i++){
+    buffers[i].free();
+  }
+
+  delete sock;
+  delete poller;
+  delete msg_ctx;
+}

cereal/visionipc/visionipc_client.h

@@ -0,0 +1,32 @@
+#pragma once
+#include <vector>
+#include <string>
+#include <unistd.h>
+
+#include "messaging.hpp"
+#include "visionipc.h"
+#include "visionbuf.h"
+
+class VisionIpcClient {
+private:
+  std::string name;
+  Context * msg_ctx;
+  SubSocket * sock;
+  Poller * poller;
+
+  VisionStreamType type;
+
+  cl_device_id device_id = nullptr;
+  cl_context ctx = nullptr;
+
+  void init_msgq(bool conflate);
+
+public:
+  bool connected = false;
+  int num_buffers = 0;
+  VisionBuf buffers[VISIONIPC_MAX_FDS];
+  VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
+  ~VisionIpcClient();
+  VisionBuf * recv(VisionIpcBufExtra * extra=nullptr, const int timeout_ms=100);
+  bool connect(bool blocking=true);
+};

cereal/visionipc/visionipc_pyx.pyx

@@ -0,0 +1,49 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+from libcpp.string cimport string
+from libcpp cimport bool
+from libc.string cimport memcpy
+from libc.stdint cimport uint32_t, uint64_t
+
+from .visionipc cimport VisionIpcServer as cppVisionIpcServer
+from .visionipc cimport VisionBuf as cppVisionBuf
+from .visionipc cimport VisionIpcBufExtra
+
+cpdef enum VisionStreamType:
+  VISION_STREAM_RGB_BACK
+  VISION_STREAM_RGB_FRONT
+  VISION_STREAM_RGB_WIDE
+  VISION_STREAM_YUV_BACK
+  VISION_STREAM_YUV_FRONT
+  VISION_STREAM_YUV_WIDE
+
+cdef class VisionIpcServer:
+  cdef cppVisionIpcServer * server
+
+  def __init__(self, string name):
+    self.server = new cppVisionIpcServer(name, NULL, NULL)
+
+  def create_buffers(self,  VisionStreamType tp, size_t num_buffers, bool rgb, size_t width, size_t height):
+    self.server.create_buffers(tp, num_buffers, rgb, width, height)
+
+  def send(self, VisionStreamType tp, bytes data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0):
+    cdef cppVisionBuf * buf = self.server.get_buffer(tp)
+
+    # Populate buffer
+    assert buf.len == len(data)
+    memcpy(buf.addr, <char*>data, len(data))
+
+    cdef VisionIpcBufExtra extra
+    extra.frame_id = frame_id
+    extra.timestamp_sof = timestamp_sof
+    extra.timestamp_eof = timestamp_eof
+
+    self.server.send(buf, &extra, False)
+
+  def start_listener(self):
+    self.server.start_listener()
+
+  def __dealloc__(self):
+    del self.server

cereal/visionipc/visionipc_server.cc

@@ -0,0 +1,179 @@
+#include <iostream>
+#include <chrono>
+#include <cassert>
+#include <random>
+
+#include <poll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "messaging.hpp"
+#include "ipc.h"
+#include "visionipc_server.h"
+
+std::string get_endpoint_name(std::string name, VisionStreamType type){
+  if (messaging_use_zmq()){
+    assert(name == "camerad");
+    return std::to_string(9000 + static_cast<int>(type));
+  } else {
+    return "visionipc_" + name + "_" + std::to_string(type);
+  }
+}
+
+VisionIpcServer::VisionIpcServer(std::string name, cl_device_id device_id, cl_context ctx) : name(name), device_id(device_id), ctx(ctx) {
+  msg_ctx = Context::create();
+
+  std::random_device rd("/dev/urandom");
+  std::uniform_int_distribution<uint64_t> distribution(0,std::numeric_limits<uint64_t>::max());
+  server_id = distribution(rd);
+}
+
+void VisionIpcServer::create_buffers(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height){
+  // TODO: assert that this type is not created yet
+  assert(num_buffers < VISIONIPC_MAX_FDS);
+  int aligned_w = 0, aligned_h = 0;
+
+  size_t size = 0;
+  size_t stride = 0; // Only used for RGB
+
+  if (rgb) {
+    visionbuf_compute_aligned_width_and_height(width, height, &aligned_w, &aligned_h);
+    size = (size_t)aligned_w * (size_t)aligned_h * 3;
+    stride = aligned_w * 3;
+  } else {
+    size = width * height * 3 / 2;
+  }
+
+
+  // Create map + alloc requested buffers
+  for (size_t i = 0; i < num_buffers; i++){
+    VisionBuf* buf = new VisionBuf();
+    buf->allocate(size);
+    buf->idx = i;
+    buf->type = type;
+
+    if (device_id) buf->init_cl(device_id, ctx);
+
+    rgb ? buf->init_rgb(width, height, stride) : buf->init_yuv(width, height);
+
+    buffers[type].push_back(buf);
+  }
+
+  cur_idx[type] = 0;
+
+  // Create msgq publisher for each of the `name` + type combos
+  // TODO: compute port number directly if using zmq
+  sockets[type] = PubSocket::create(msg_ctx, get_endpoint_name(name, type), false);
+}
+
+
+void VisionIpcServer::start_listener(){
+  listener_thread = std::thread(&VisionIpcServer::listener, this);
+}
+
+
+void VisionIpcServer::listener(){
+  std::cout << "Starting listener for: " << name << std::endl;
+
+  std::string path = "/tmp/visionipc_" + name;
+  int sock = ipc_bind(path.c_str());
+  assert(sock >= 0);
+
+  while (!should_exit){
+    // Wait for incoming connection
+    struct pollfd polls[1] = {{0}};
+    polls[0].fd = sock;
+    polls[0].events = POLLIN;
+
+    int ret = poll(polls, 1, 100);
+    if (ret < 0) {
+      if (errno == EINTR || errno == EAGAIN) continue;
+      std::cout << "poll failed, stopping listener" << std::endl;
+      break;
+    }
+
+    if (should_exit) break;
+    if (!polls[0].revents) {
+      continue;
+    }
+
+    // Handle incoming request
+    int fd = accept(sock, NULL, NULL);
+    assert(fd >= 0);
+
+    VisionStreamType type = VisionStreamType::VISION_STREAM_MAX;
+    int r = ipc_sendrecv_with_fds(false, fd, &type, sizeof(type), nullptr, 0, nullptr);
+    assert(r == sizeof(type));
+    if (buffers.count(type) <= 0) {
+      std::cout << "got request for invalid buffer type: " << type << std::endl;
+      close(fd);
+      continue;
+    }
+
+    int fds[VISIONIPC_MAX_FDS];
+    int num_fds = buffers[type].size();
+    VisionBuf bufs[VISIONIPC_MAX_FDS];
+
+    for (int i = 0; i < num_fds; i++){
+      fds[i] = buffers[type][i]->fd;
+      bufs[i] = *buffers[type][i];
+
+      // Remove some private openCL/ion metadata
+      bufs[i].buf_cl = 0;
+      bufs[i].copy_q = 0;
+      bufs[i].handle = 0;
+      bufs[i].owner = false;
+
+      bufs[i].server_id = server_id;
+    }
+
+    r = ipc_sendrecv_with_fds(true, fd, &bufs, sizeof(VisionBuf) * num_fds, fds, num_fds, nullptr);
+
+    close(fd);
+  }
+
+  std::cout << "Stopping listener for: " << name << std::endl;
+  close(sock);
+}
+
+
+
+VisionBuf * VisionIpcServer::get_buffer(VisionStreamType type){
+  // Do we want to keep track if the buffer has been sent out yet and warn user?
+  assert(buffers.count(type));
+  auto b = buffers[type];
+  return b[cur_idx[type]++ % b.size()];
+}
+
+void VisionIpcServer::send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync){
+  if (sync) buf->sync(VISIONBUF_SYNC_FROM_DEVICE);
+  assert(buffers.count(buf->type));
+  assert(buf->idx < buffers[buf->type].size());
+
+  // Send over correct msgq socket
+  VisionIpcPacket packet = {0};
+  packet.server_id = server_id;
+  packet.idx = buf->idx;
+  packet.extra = *extra;
+
+  sockets[buf->type]->send((char*)&packet, sizeof(packet));
+}
+
+VisionIpcServer::~VisionIpcServer(){
+  should_exit = true;
+  listener_thread.join();
+
+  // VisionBuf cleanup
+  for( auto const& [type, buf] : buffers ) {
+    for (VisionBuf* b : buf){
+      b->free();
+      delete b;
+    }
+  }
+
+  // Messaging cleanup
+  for( auto const& [type, sock] : sockets ) {
+    delete sock;
+  }
+  delete msg_ctx;
+}

cereal/visionipc/visionipc_server.h

@@ -0,0 +1,42 @@
+#pragma once
+#include <vector>
+#include <string>
+#include <thread>
+#include <atomic>
+#include <map>
+
+#include "messaging.hpp"
+#include "visionipc.h"
+#include "visionbuf.h"
+
+std::string get_endpoint_name(std::string name, VisionStreamType type);
+
+class VisionIpcServer {
+ private:
+  cl_device_id device_id = nullptr;
+  cl_context ctx = nullptr;
+  uint64_t server_id;
+
+  std::atomic<bool> should_exit = false;
+  std::string name;
+  std::thread listener_thread;
+
+  std::map<VisionStreamType, std::atomic<size_t> > cur_idx;
+  std::map<VisionStreamType, std::vector<VisionBuf*> > buffers;
+  std::map<VisionStreamType, std::map<VisionBuf*, size_t> > idxs;
+
+  Context * msg_ctx;
+  std::map<VisionStreamType, PubSocket*> sockets;
+
+  void listener(void);
+
+ public:
+  VisionIpcServer(std::string name, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
+  ~VisionIpcServer();
+
+  VisionBuf * get_buffer(VisionStreamType type);
+
+  void create_buffers(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height);
+  void send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync=true);
+  void start_listener();
+};

cereal/visionipc/visionipc_tests.cc

@@ -0,0 +1,135 @@
+#include <thread>
+#include <chrono>
+
+#include "catch2/catch.hpp"
+#include "visionipc_server.h"
+#include "visionipc_client.h"
+
+static void zmq_sleep(int milliseconds=1000){
+  if (messaging_use_zmq()){
+    std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
+  }
+}
+
+TEST_CASE("Connecting"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_YUV_BACK, 1, false, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_YUV_BACK, false);
+  REQUIRE(client.connect());
+
+  REQUIRE(client.connected);
+}
+
+TEST_CASE("Check buffers"){
+  size_t width = 100, height = 200, num_buffers = 5;
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_YUV_BACK, num_buffers, false, width, height);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_YUV_BACK, false);
+  REQUIRE(client.connect());
+
+  REQUIRE(client.buffers[0].width == width);
+  REQUIRE(client.buffers[0].height == height);
+  REQUIRE(client.buffers[0].len);
+  REQUIRE(client.num_buffers == num_buffers);
+}
+
+TEST_CASE("Check yuv/rgb"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_YUV_BACK, 1, false, 100, 100);
+  server.create_buffers(VISION_STREAM_RGB_BACK, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client_yuv = VisionIpcClient("camerad", VISION_STREAM_YUV_BACK, false);
+  VisionIpcClient client_rgb = VisionIpcClient("camerad", VISION_STREAM_RGB_BACK, false);
+  client_yuv.connect();
+  client_rgb.connect();
+
+  REQUIRE(client_rgb.buffers[0].rgb == true);
+  REQUIRE(client_yuv.buffers[0].rgb == false);
+}
+
+TEST_CASE("Send single buffer"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_YUV_BACK, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_YUV_BACK, false);
+  REQUIRE(client.connect());
+  zmq_sleep();
+
+  VisionBuf * buf = server.get_buffer(VISION_STREAM_YUV_BACK);
+  REQUIRE(buf != nullptr);
+
+  *((uint64_t*)buf->addr) = 1234;
+
+  VisionIpcBufExtra extra = {0};
+  extra.frame_id = 1337;
+
+  server.send(buf, &extra);
+
+  VisionIpcBufExtra extra_recv = {0};
+  VisionBuf * recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(*(uint64_t*)recv_buf->addr == 1234);
+  REQUIRE(extra_recv.frame_id == extra.frame_id);
+}
+
+
+TEST_CASE("Test no conflate"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_YUV_BACK, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_YUV_BACK, false);
+  REQUIRE(client.connect());
+  zmq_sleep();
+
+  VisionBuf * buf = server.get_buffer(VISION_STREAM_YUV_BACK);
+  REQUIRE(buf != nullptr);
+
+  VisionIpcBufExtra extra = {0};
+  extra.frame_id = 1;
+  server.send(buf, &extra);
+  extra.frame_id = 2;
+  server.send(buf, &extra);
+
+  VisionIpcBufExtra extra_recv = {0};
+  VisionBuf * recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(extra_recv.frame_id == 1);
+
+  recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(extra_recv.frame_id == 2);
+}
+
+TEST_CASE("Test conflate"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_YUV_BACK, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_YUV_BACK, true);
+  REQUIRE(client.connect());
+  zmq_sleep();
+
+  VisionBuf * buf = server.get_buffer(VISION_STREAM_YUV_BACK);
+  REQUIRE(buf != nullptr);
+
+  VisionIpcBufExtra extra = {0};
+  extra.frame_id = 1;
+  server.send(buf, &extra);
+  extra.frame_id = 2;
+  server.send(buf, &extra);
+
+  VisionIpcBufExtra extra_recv = {0};
+  VisionBuf * recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(extra_recv.frame_id == 2);
+
+  recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf == nullptr);
+}

common/SConscript

@@ -1,6 +1,4 @@
-Import('env')
+Import('envCython')
 
-# parser
-env.Command(['common_pyx.so'],
-  ['common_pyx_setup.py', 'clock.pyx'],
-  "cd common && python3 common_pyx_setup.py build_ext --inplace")
+envCython.Program('clock.so', 'clock.pyx')
+envCython.Program('params_pyx.so', 'params_pyx.pyx')

common/android.py

@@ -1,286 +0,0 @@
-import os
-import binascii
-import itertools
-import re
-import struct
-import subprocess
-import random
-from cereal import log
-
-NetworkType = log.ThermalData.NetworkType
-NetworkStrength = log.ThermalData.NetworkStrength
-
-ANDROID = os.path.isfile('/EON')
-
-def get_sound_card_online():
-  return (os.path.isfile('/proc/asound/card0/state') and
-          open('/proc/asound/card0/state').read().strip() == 'ONLINE')
-
-def getprop(key):
-  if not ANDROID:
-    return ""
-  return subprocess.check_output(["getprop", key], encoding='utf8').strip()
-
-def get_imei(slot):
-  slot = str(slot)
-  if slot not in ("0", "1"):
-    raise ValueError("SIM slot must be 0 or 1")
-
-  ret = parse_service_call_string(service_call(["iphonesubinfo", "3" , "i32", str(slot)]))
-  if not ret:
-    # allow non android to be identified differently
-    ret = "%015d" % random.randint(0, 1 << 32)
-  return ret
-
-def get_serial():
-  ret = getprop("ro.serialno")
-  if ret == "":
-    ret = "cccccccc"
-  return ret
-
-def get_subscriber_info():
-  ret = parse_service_call_string(service_call(["iphonesubinfo", "7"]))
-  if ret is None or len(ret) < 8:
-    return ""
-  return ret
-
-def reboot(reason=None):
-  if reason is None:
-    reason_args = ["null"]
-  else:
-    reason_args = ["s16", reason]
-
-  subprocess.check_output([
-    "service", "call", "power", "16",  # IPowerManager.reboot
-    "i32", "0",  # no confirmation,
-    *reason_args,
-    "i32", "1"  # wait
-  ])
-
-def service_call(call):
-  if not ANDROID:
-    return None
-
-  ret = subprocess.check_output(["service", "call", *call], encoding='utf8').strip()
-  if 'Parcel' not in ret:
-    return None
-
-  return parse_service_call_bytes(ret)
-
-def parse_service_call_unpack(r, fmt):
-  try:
-    return struct.unpack(fmt, r)[0]
-  except Exception:
-    return None
-
-def parse_service_call_string(r):
-  try:
-    r = r[8:]  # Cut off length field
-    r = r.decode('utf_16_be')
-
-    # All pairs of two characters seem to be swapped. Not sure why
-    result = ""
-    for a, b, in itertools.zip_longest(r[::2], r[1::2], fillvalue='\x00'):
-        result += b + a
-
-    result = result.replace('\x00', '')
-
-    return result
-  except Exception:
-    return None
-
-def parse_service_call_bytes(ret):
-  try:
-    r = b""
-    for hex_part in re.findall(r'[ (]([0-9a-f]{8})', ret):
-      r += binascii.unhexlify(hex_part)
-    return r
-  except Exception:
-    return None
-
-def get_network_type():
-  if not ANDROID:
-    return NetworkType.none
-
-  wifi_check = parse_service_call_string(service_call(["connectivity", "2"]))
-  if wifi_check is None:
-    return NetworkType.none
-  elif 'WIFI' in wifi_check:
-    return NetworkType.wifi
-  else:
-    cell_check = parse_service_call_unpack(service_call(['phone', '59']), ">q")
-    # from TelephonyManager.java
-    cell_networks = {
-      0: NetworkType.none,
-      1: NetworkType.cell2G,
-      2: NetworkType.cell2G,
-      3: NetworkType.cell3G,
-      4: NetworkType.cell2G,
-      5: NetworkType.cell3G,
-      6: NetworkType.cell3G,
-      7: NetworkType.cell3G,
-      8: NetworkType.cell3G,
-      9: NetworkType.cell3G,
-      10: NetworkType.cell3G,
-      11: NetworkType.cell2G,
-      12: NetworkType.cell3G,
-      13: NetworkType.cell4G,
-      14: NetworkType.cell4G,
-      15: NetworkType.cell3G,
-      16: NetworkType.cell2G,
-      17: NetworkType.cell3G,
-      18: NetworkType.cell4G,
-      19: NetworkType.cell4G
-    }
-    return cell_networks.get(cell_check, NetworkType.none)
-
-def get_network_strength(network_type):
-  network_strength = NetworkStrength.unknown
-
-  # from SignalStrength.java
-  def get_lte_level(rsrp, rssnr):
-    INT_MAX = 2147483647
-    if rsrp == INT_MAX:
-      lvl_rsrp = NetworkStrength.unknown
-    elif rsrp >= -95:
-      lvl_rsrp = NetworkStrength.great
-    elif rsrp >= -105:
-      lvl_rsrp = NetworkStrength.good
-    elif rsrp >= -115:
-      lvl_rsrp = NetworkStrength.moderate
-    else:
-      lvl_rsrp = NetworkStrength.poor
-    if rssnr == INT_MAX:
-      lvl_rssnr = NetworkStrength.unknown
-    elif rssnr >= 45:
-      lvl_rssnr = NetworkStrength.great
-    elif rssnr >= 10:
-      lvl_rssnr = NetworkStrength.good
-    elif rssnr >= -30:
-      lvl_rssnr = NetworkStrength.moderate
-    else:
-      lvl_rssnr = NetworkStrength.poor
-    return max(lvl_rsrp, lvl_rssnr)
-
-  def get_tdscdma_level(tdscmadbm):
-    lvl = NetworkStrength.unknown
-    if tdscmadbm > -25:
-      lvl = NetworkStrength.unknown
-    elif tdscmadbm >= -49:
-      lvl = NetworkStrength.great
-    elif tdscmadbm >= -73:
-      lvl = NetworkStrength.good
-    elif tdscmadbm >= -97:
-      lvl = NetworkStrength.moderate
-    elif tdscmadbm >= -110:
-      lvl = NetworkStrength.poor
-    return lvl
-
-  def get_gsm_level(asu):
-    if asu <= 2 or asu == 99:
-      lvl = NetworkStrength.unknown
-    elif asu >= 12:
-      lvl = NetworkStrength.great
-    elif asu >= 8:
-      lvl = NetworkStrength.good
-    elif asu >= 5:
-      lvl = NetworkStrength.moderate
-    else:
-      lvl = NetworkStrength.poor
-    return lvl
-
-  def get_evdo_level(evdodbm, evdosnr):
-    lvl_evdodbm = NetworkStrength.unknown
-    lvl_evdosnr = NetworkStrength.unknown
-    if evdodbm >= -65:
-      lvl_evdodbm = NetworkStrength.great
-    elif evdodbm >= -75:
-      lvl_evdodbm = NetworkStrength.good
-    elif evdodbm >= -90:
-      lvl_evdodbm = NetworkStrength.moderate
-    elif evdodbm >= -105:
-      lvl_evdodbm = NetworkStrength.poor
-    if evdosnr >= 7:
-      lvl_evdosnr = NetworkStrength.great
-    elif evdosnr >= 5:
-      lvl_evdosnr = NetworkStrength.good
-    elif evdosnr >= 3:
-      lvl_evdosnr = NetworkStrength.moderate
-    elif evdosnr >= 1:
-      lvl_evdosnr = NetworkStrength.poor
-    return max(lvl_evdodbm, lvl_evdosnr)
-
-  def get_cdma_level(cdmadbm, cdmaecio):
-    lvl_cdmadbm = NetworkStrength.unknown
-    lvl_cdmaecio = NetworkStrength.unknown
-    if cdmadbm >= -75:
-      lvl_cdmadbm = NetworkStrength.great
-    elif cdmadbm >= -85:
-      lvl_cdmadbm = NetworkStrength.good
-    elif cdmadbm >= -95:
-      lvl_cdmadbm = NetworkStrength.moderate
-    elif cdmadbm >= -100:
-      lvl_cdmadbm = NetworkStrength.poor
-    if cdmaecio >= -90:
-      lvl_cdmaecio = NetworkStrength.great
-    elif cdmaecio >= -110:
-      lvl_cdmaecio = NetworkStrength.good
-    elif cdmaecio >= -130:
-      lvl_cdmaecio = NetworkStrength.moderate
-    elif cdmaecio >= -150:
-      lvl_cdmaecio = NetworkStrength.poor
-    return max(lvl_cdmadbm, lvl_cdmaecio)
-
-  if network_type == NetworkType.none:
-    return network_strength
-  if network_type == NetworkType.wifi:
-    out = subprocess.check_output('dumpsys connectivity', shell=True).decode('utf-8')
-    network_strength = NetworkStrength.unknown
-    for line in out.split('\n'):
-      signal_str = "SignalStrength: "
-      if signal_str in line:
-        lvl_idx_start = line.find(signal_str) + len(signal_str)
-        lvl_idx_end = line.find(']', lvl_idx_start)
-        lvl = int(line[lvl_idx_start : lvl_idx_end])
-        if lvl >= -50:
-          network_strength = NetworkStrength.great
-        elif lvl >= -60:
-          network_strength = NetworkStrength.good
-        elif lvl >= -70:
-          network_strength = NetworkStrength.moderate
-        else:
-          network_strength = NetworkStrength.poor
-    return network_strength
-  else:
-    # check cell strength
-    out = subprocess.check_output('dumpsys telephony.registry', shell=True).decode('utf-8')
-    for line in out.split('\n'):
-      if "mSignalStrength" in line:
-        arr = line.split(' ')
-        ns = 0
-        if ("gsm" in arr[14]):
-          rsrp = int(arr[9])
-          rssnr = int(arr[11])
-          ns = get_lte_level(rsrp, rssnr)
-          if ns == NetworkStrength.unknown:
-            tdscmadbm = int(arr[13])
-            ns = get_tdscdma_level(tdscmadbm)
-            if ns == NetworkStrength.unknown:
-              asu = int(arr[1])
-              ns = get_gsm_level(asu)
-        else:
-          cdmadbm = int(arr[3])
-          cdmaecio = int(arr[4])
-          evdodbm = int(arr[5])
-          evdosnr = int(arr[7])
-          lvl_cdma = get_cdma_level(cdmadbm, cdmaecio)
-          lvl_edmo = get_evdo_level(evdodbm, evdosnr)
-          if lvl_edmo == NetworkStrength.unknown:
-            ns = lvl_cdma
-          elif lvl_cdma == NetworkStrength.unknown:
-            ns = lvl_edmo
-          else:
-            ns = min(lvl_cdma, lvl_edmo)
-        network_strength = max(network_strength, ns)
-
-    return network_strength

common/api/__init__.py

@@ -27,7 +27,11 @@ class Api():
       'iat': now,
       'exp': now + timedelta(hours=1)
     }
-    return jwt.encode(payload, self.private_key, algorithm='RS256').decode('utf8')
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+    
 
 def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
   backend = "https://api.commadotai.com/"

common/apk.py

@@ -1,97 +0,0 @@
-import os
-import subprocess
-import glob
-import hashlib
-import shutil
-from common.basedir import BASEDIR
-from selfdrive.swaglog import cloudlog
-
-android_packages = ("ai.comma.plus.offroad",)
-
-def get_installed_apks():
-  dat = subprocess.check_output(["pm", "list", "packages", "-f"], encoding='utf8').strip().split("\n")
-  ret = {}
-  for x in dat:
-    if x.startswith("package:"):
-      v, k = x.split("package:")[1].split("=")
-      ret[k] = v
-  return ret
-
-def install_apk(path):
-  # can only install from world readable path
-  install_path = "/sdcard/%s" % os.path.basename(path)
-  shutil.copyfile(path, install_path)
-
-  ret = subprocess.call(["pm", "install", "-r", install_path])
-  os.remove(install_path)
-  return ret == 0
-
-def start_offroad():
-  set_package_permissions()
-  system("am start -n ai.comma.plus.offroad/.MainActivity")
-
-def set_package_permissions():
-  pm_grant("ai.comma.plus.offroad", "android.permission.ACCESS_FINE_LOCATION")
-  pm_grant("ai.comma.plus.offroad", "android.permission.READ_PHONE_STATE")
-  pm_grant("ai.comma.plus.offroad", "android.permission.READ_EXTERNAL_STORAGE")
-  appops_set("ai.comma.plus.offroad", "SU", "allow")
-  appops_set("ai.comma.plus.offroad", "WIFI_SCAN", "allow")
-
-def appops_set(package, op, mode):
-  system(f"LD_LIBRARY_PATH= appops set {package} {op} {mode}")
-
-def pm_grant(package, permission):
-  system(f"pm grant {package} {permission}")
-
-def system(cmd):
-  try:
-    cloudlog.info("running %s" % cmd)
-    subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
-  except subprocess.CalledProcessError as e:
-    cloudlog.event("running failed",
-      cmd=e.cmd,
-      output=e.output[-1024:],
-      returncode=e.returncode)
-
-# *** external functions ***
-
-def update_apks():
-  # install apks
-  installed = get_installed_apks()
-
-  install_apks = glob.glob(os.path.join(BASEDIR, "apk/*.apk"))
-  for apk in install_apks:
-    app = os.path.basename(apk)[:-4]
-    if app not in installed:
-      installed[app] = None
-
-  cloudlog.info("installed apks %s" % (str(installed), ))
-
-  for app in installed.keys():
-    apk_path = os.path.join(BASEDIR, "apk/"+app+".apk")
-    if not os.path.exists(apk_path):
-      continue
-
-    h1 = hashlib.sha1(open(apk_path, 'rb').read()).hexdigest()
-    h2 = None
-    if installed[app] is not None:
-      h2 = hashlib.sha1(open(installed[app], 'rb').read()).hexdigest()
-      cloudlog.info("comparing version of %s  %s vs %s" % (app, h1, h2))
-
-    if h2 is None or h1 != h2:
-      cloudlog.info("installing %s" % app)
-
-      success = install_apk(apk_path)
-      if not success:
-        cloudlog.info("needing to uninstall %s" % app)
-        system("pm uninstall %s" % app)
-        success = install_apk(apk_path)
-
-      assert success
-
-def pm_apply_packages(cmd):
-  for p in android_packages:
-    system("pm %s %s" % (cmd, p))
-
-if __name__ == "__main__":
-  update_apks()

common/basedir.py

@@ -1,10 +1,11 @@
 import os
+from pathlib import Path
+
+from selfdrive.hardware import PC
+
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
 
-from common.android import ANDROID
-if ANDROID:
-  PERSIST = "/persist"
-  PARAMS = "/data/params"
+if PC:
+  PERSIST = os.path.join(str(Path.home()), ".comma", "persist")
 else:
-  PERSIST = os.path.join(BASEDIR, "persist")
-  PARAMS = os.path.join(BASEDIR, "persist", "params")
+  PERSIST = "/persist"

common/clock.pyx

@@ -1,3 +1,5 @@
+# distutils: language = c++
+# cython: language_level = 3
 from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
 
 IF UNAME_SYSNAME == "Darwin":

common/common_pyx_setup.py

@@ -1,20 +0,0 @@
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-from Cython.Build import cythonize
-
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-
-sourcefiles = ['clock.pyx']
-extra_compile_args = ["-std=c++11"]
-
-setup(name='Common',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "common_pyx",
-          language="c++",
-          sources=sourcefiles,
-          extra_compile_args=extra_compile_args,
-        )
-      ),
-      nthreads=4,
-)

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/ffi_wrapper.py

@@ -44,7 +44,7 @@ def compile_code(name, c_code, c_header, directory, cflags="", libraries=None):
   ffibuilder = FFI()
   ffibuilder.set_source(name, c_code, source_extension='.cpp', libraries=libraries)
   ffibuilder.cdef(c_header)
-  os.environ['OPT'] = "-fwrapv -O2 -DNDEBUG -std=c++11"
+  os.environ['OPT'] = "-fwrapv -O2 -DNDEBUG -std=c++1z"
   os.environ['CFLAGS'] = cflags
   ffibuilder.compile(verbose=True, debug=False, tmpdir=directory)
 

common/gpio.py

@@ -0,0 +1,14 @@
+def gpio_init(pin, output):
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
+      f.write(b"out" if output else b"in")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} direction: {e}")
+
+
+def gpio_set(pin, high):
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
+      f.write(b"1" if high else b"0")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")

common/kalman/SConscript

@@ -1,6 +1,3 @@
-Import('env')
-
-env.Command(['simple_kalman_impl.so'],
-  ['simple_kalman_impl.pyx', 'simple_kalman_impl.pxd', 'simple_kalman_setup.py'],
-  "cd common/kalman && python3 simple_kalman_setup.py build_ext --inplace")
+Import('envCython')
 
+envCython.Program('simple_kalman_impl.so', 'simple_kalman_impl.pyx')

common/kalman/simple_kalman_impl.pxd

@@ -1,3 +1,5 @@
+# cython: language_level = 3
+
 cdef class KF1D:
   cdef public:
     double x0_0
@@ -13,4 +15,4 @@ cdef class KF1D:
     double A_K_0
     double A_K_1
     double A_K_2
-    double A_K_3
+    double A_K_3

common/kalman/simple_kalman_impl.pyx

@@ -1,3 +1,4 @@
+# distutils: language = c++
 # cython: language_level=3
 
 cdef class KF1D:

common/kalman/simple_kalman_old.py

@@ -8,7 +8,7 @@ class KF1D:
   def __init__(self, x0, A, C, K):
     self.x = x0
     self.A = A
-    self.C = C
+    self.C = np.atleast_2d(C)
     self.K = K
 
     self.A_K = self.A - np.dot(self.K, self.C)

common/kalman/simple_kalman_setup.py

@@ -1,9 +0,0 @@
-from distutils.core import Extension, setup
-
-from Cython.Build import cythonize
-
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
-
-setup(name='Simple Kalman Implementation',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(Extension("simple_kalman_impl", ["simple_kalman_impl.pyx"])))

common/kalman/tests/test_simple_kalman.py

@@ -21,10 +21,10 @@ class TestSimpleKalman(unittest.TestCase):
     K0_0 = 0.12287673
     K1_0 = 0.29666309
 
-    self.kf_old = KF1D_old(x0=np.matrix([[x0_0], [x1_0]]),
-                           A=np.matrix([[A0_0, A0_1], [A1_0, A1_1]]),
-                           C=np.matrix([C0_0, C0_1]),
-                           K=np.matrix([[K0_0], [K1_0]]))
+    self.kf_old = KF1D_old(x0=np.array([[x0_0], [x1_0]]),
+                           A=np.array([[A0_0, A0_1], [A1_0, A1_1]]),
+                           C=np.array([C0_0, C0_1]),
+                           K=np.array([[K0_0], [K1_0]]))
 
     self.kf = KF1D(x0=[[x0_0], [x1_0]],
                    A=[[A0_0, A0_1], [A1_0, A1_1]],
@@ -47,8 +47,8 @@ class TestSimpleKalman(unittest.TestCase):
       x = self.kf.update(v_wheel)
 
       # Compare the output x, verify that the error is less than 1e-4
-      self.assertAlmostEqual(x_old[0], x[0])
-      self.assertAlmostEqual(x_old[1], x[1])
+      np.testing.assert_almost_equal(x_old[0], x[0])
+      np.testing.assert_almost_equal(x_old[1], x[1])
 
   def test_new_is_faster(self):
     setup = """
@@ -69,10 +69,10 @@ C0_1 = 0.0
 K0_0 = 0.12287673
 K1_0 = 0.29666309
 
-kf_old = KF1D_old(x0=np.matrix([[x0_0], [x1_0]]),
-                  A=np.matrix([[A0_0, A0_1], [A1_0, A1_1]]),
-                  C=np.matrix([C0_0, C0_1]),
-                  K=np.matrix([[K0_0], [K1_0]]))
+kf_old = KF1D_old(x0=np.array([[x0_0], [x1_0]]),
+                  A=np.array([[A0_0, A0_1], [A1_0, A1_1]]),
+                  C=np.array([C0_0, C0_1]),
+                  K=np.array([[K0_0], [K1_0]]))
 
 kf = KF1D(x0=[[x0_0], [x1_0]],
           A=[[A0_0, A0_1], [A1_0, A1_1]],
@@ -82,3 +82,6 @@ kf = KF1D(x0=[[x0_0], [x1_0]],
     kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000)
     kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000)
     self.assertTrue(kf_speed < kf_old_speed / 4)
+
+if __name__ == "__main__":
+  unittest.main()

common/logging_extra.py

@@ -3,6 +3,7 @@ import os
 import sys
 import copy
 import json
+import uuid
 import socket
 import logging
 import traceback
@@ -62,8 +63,48 @@ class SwagFormatter(logging.Formatter):
     return record_dict
 
   def format(self, record):
+    if self.swaglogger is None:
+      raise Exception("must set swaglogger before calling format()")
     return json_robust_dumps(self.format_dict(record))
 
+class SwagLogFileFormatter(SwagFormatter):
+  def fix_kv(self, k, v):
+    # append type to names to preserve legacy naming in logs
+    # avoids overlapping key namespaces with different types
+    # e.g. log.info() creates 'msg' -> 'msg$s'
+    #      log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
+    #      because overlapping namespace 'msg' caused problems
+    if isinstance(v, (str, bytes)):
+      k += "$s"
+    elif isinstance(v, float):
+      k += "$f"
+    elif isinstance(v, bool):
+      k += "$b"
+    elif isinstance(v, int):
+      k += "$i"
+    elif isinstance(v, dict):
+      nv = {}
+      for ik, iv in v.items():
+        ik, iv = self.fix_kv(ik, iv)
+        nv[ik] = iv
+      v = nv
+    elif isinstance(v, list):
+      k += "$a"
+    return k, v
+
+  def format(self, record):
+    if isinstance(record, str):
+      v = json.loads(record)
+    else:
+      v = self.format_dict(record)
+
+    mk, mv = self.fix_kv('msg', v['msg'])
+    del v['msg']
+    v[mk] = mv
+    v['id'] = uuid.uuid4().hex
+
+    return json_robust_dumps(v)
+
 class SwagErrorFilter(logging.Filter):
   def filter(self, record):
     return record.levelno < logging.ERROR
@@ -117,6 +158,8 @@ class SwagLogger(logging.Logger):
     evt.update(kwargs)
     if 'error' in kwargs:
       self.error(evt)
+    if 'debug' in kwargs:
+      self.debug(evt)
     else:
       self.info(evt)
 

common/params.py

@@ -1,410 +1,23 @@
-#!/usr/bin/env python3
-"""ROS has a parameter server, we have files.
-
-The parameter store is a persistent key value store, implemented as a directory with a writer lock.
-On Android, we store params under params_dir = /data/params. The writer lock is a file
-"<params_dir>/.lock" taken using flock(), and data is stored in a directory symlinked to by
-"<params_dir>/d".
-
-Each key, value pair is stored as a file with named <key> with contents <value>, located in
-  <params_dir>/d/<key>
-
-Readers of a single key can just open("<params_dir>/d/<key>") and read the file contents.
-Readers who want a consistent snapshot of multiple keys should take the lock.
-
-Writers should take the lock before modifying anything. Writers should also leave the DB in a
-consistent state after a crash. The implementation below does this by copying all params to a temp
-directory <params_dir>/<tmp>, then atomically symlinking <params_dir>/<d> to <params_dir>/<tmp>
-before deleting the old <params_dir>/<d> directory.
-
-Writers that only modify a single key can simply take the lock, then swap the corresponding value
-file in place without messing with <params_dir>/d.
-"""
-import time
-import os
-import errno
-import shutil
-import fcntl
-import tempfile
-import threading
-from enum import Enum
-from common.basedir import PARAMS
-
-
-def mkdirs_exists_ok(path):
-  try:
-    os.makedirs(path)
-  except OSError:
-    if not os.path.isdir(path):
-      raise
-
-
-class TxType(Enum):
-  PERSISTENT = 1
-  CLEAR_ON_MANAGER_START = 2
-  CLEAR_ON_PANDA_DISCONNECT = 3
-
-
-class UnknownKeyName(Exception):
-  pass
-
-
-keys = {
-  "AccessToken": [TxType.CLEAR_ON_MANAGER_START],
-  "AthenadPid": [TxType.PERSISTENT],
-  "CalibrationParams": [TxType.PERSISTENT],
-  "CarParams": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "CarParamsCache": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "CarVin": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "CommunityFeaturesToggle": [TxType.PERSISTENT],
-  "CompletedTrainingVersion": [TxType.PERSISTENT],
-  "ControlsParams": [TxType.PERSISTENT],
-  "DisablePowerDown": [TxType.PERSISTENT],
-  "DisableUpdates": [TxType.PERSISTENT],
-  "DoUninstall": [TxType.CLEAR_ON_MANAGER_START],
-  "DongleId": [TxType.PERSISTENT],
-  "GitBranch": [TxType.PERSISTENT],
-  "GitCommit": [TxType.PERSISTENT],
-  "GitRemote": [TxType.PERSISTENT],
-  "GithubSshKeys": [TxType.PERSISTENT],
-  "HasAcceptedTerms": [TxType.PERSISTENT],
-  "HasCompletedSetup": [TxType.PERSISTENT],
-  "IsDriverViewEnabled": [TxType.CLEAR_ON_MANAGER_START],
-  "IsLdwEnabled": [TxType.PERSISTENT],
-  "IsGeofenceEnabled": [TxType.PERSISTENT],
-  "IsMetric": [TxType.PERSISTENT],
-  "IsOffroad": [TxType.CLEAR_ON_MANAGER_START],
-  "IsRHD": [TxType.PERSISTENT],
-  "IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
-  "IsUpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
-  "IsUploadRawEnabled": [TxType.PERSISTENT],
-  "LastAthenaPingTime": [TxType.PERSISTENT],
-  "LastUpdateTime": [TxType.PERSISTENT],
-  "LimitSetSpeed": [TxType.PERSISTENT],
-  "LimitSetSpeedNeural": [TxType.PERSISTENT],
-  "LiveParameters": [TxType.PERSISTENT],
-  "LongitudinalControl": [TxType.PERSISTENT],
-  "OpenpilotEnabledToggle": [TxType.PERSISTENT],
-  "LaneChangeEnabled": [TxType.PERSISTENT],
-  "PandaFirmware": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "PandaFirmwareHex": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "PandaDongleId": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "Passive": [TxType.PERSISTENT],
-  "RecordFront": [TxType.PERSISTENT],
-  "ReleaseNotes": [TxType.PERSISTENT],
-  "ShouldDoUpdate": [TxType.CLEAR_ON_MANAGER_START],
-  "SpeedLimitOffset": [TxType.PERSISTENT],
-  "SubscriberInfo": [TxType.PERSISTENT],
-  "TermsVersion": [TxType.PERSISTENT],
-  "TrainingVersion": [TxType.PERSISTENT],
-  "UpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
-  "UpdateFailedCount": [TxType.CLEAR_ON_MANAGER_START],
-  "Version": [TxType.PERSISTENT],
-  "Offroad_ChargeDisabled": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "Offroad_ConnectivityNeeded": [TxType.CLEAR_ON_MANAGER_START],
-  "Offroad_ConnectivityNeededPrompt": [TxType.CLEAR_ON_MANAGER_START],
-  "Offroad_TemperatureTooHigh": [TxType.CLEAR_ON_MANAGER_START],
-  "Offroad_PandaFirmwareMismatch": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
-  "Offroad_InvalidTime": [TxType.CLEAR_ON_MANAGER_START],
-  "Offroad_IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
-  "Offroad_NeosUpdate": [TxType.CLEAR_ON_MANAGER_START],
-}
-
-
-def fsync_dir(path):
-  fd = os.open(path, os.O_RDONLY)
-  try:
-    os.fsync(fd)
-  finally:
-    os.close(fd)
-
-
-class FileLock():
-  def __init__(self, path, create):
-    self._path = path
-    self._create = create
-    self._fd = None
-
-  def acquire(self):
-    self._fd = os.open(self._path, os.O_CREAT if self._create else 0)
-    fcntl.flock(self._fd, fcntl.LOCK_EX)
-
-  def release(self):
-    if self._fd is not None:
-      os.close(self._fd)
-      self._fd = None
-
-
-class DBAccessor():
-  def __init__(self, path):
-    self._path = path
-    self._vals = None
-
-  def keys(self):
-    self._check_entered()
-    return self._vals.keys()
-
-  def get(self, key):
-    self._check_entered()
-
-    if self._vals is None:
-      return None
-
-    try:
-      return self._vals[key]
-    except KeyError:
-      return None
-
-  def _get_lock(self, create):
-    lock = FileLock(os.path.join(self._path, ".lock"), create)
-    lock.acquire()
-    return lock
-
-  def _read_values_locked(self):
-    """Callers should hold a lock while calling this method."""
-    vals = {}
-    try:
-      data_path = self._data_path()
-      keys = os.listdir(data_path)
-      for key in keys:
-        with open(os.path.join(data_path, key), "rb") as f:
-          vals[key] = f.read()
-    except (OSError, IOError) as e:
-      # Either the DB hasn't been created yet, or somebody wrote a bug and left the DB in an
-      # inconsistent state. Either way, return empty.
-      if e.errno == errno.ENOENT:
-        return {}
-
-    return vals
-
-  def _data_path(self):
-    return os.path.join(self._path, "d")
-
-  def _check_entered(self):
-    if self._vals is None:
-      raise Exception("Must call __enter__ before using DB")
-
-
-class DBReader(DBAccessor):
-  def __enter__(self):
-    try:
-      lock = self._get_lock(False)
-    except OSError as e:
-      # Do not create lock if it does not exist.
-      if e.errno == errno.ENOENT:
-        self._vals = {}
-        return self
-
-    try:
-      # Read everything.
-      self._vals = self._read_values_locked()
-      return self
-    finally:
-      lock.release()
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    pass
-
-
-class DBWriter(DBAccessor):
-  def __init__(self, path):
-    super(DBWriter, self).__init__(path)
-    self._lock = None
-    self._prev_umask = None
-
-  def put(self, key, value):
-    self._vals[key] = value
-
-  def delete(self, key):
-    self._vals.pop(key, None)
-
-  def __enter__(self):
-    mkdirs_exists_ok(self._path)
-
-    # Make sure we can write and that permissions are correct.
-    self._prev_umask = os.umask(0)
-
-    try:
-      os.chmod(self._path, 0o777)
-      self._lock = self._get_lock(True)
-      self._vals = self._read_values_locked()
-    except Exception:
-      os.umask(self._prev_umask)
-      self._prev_umask = None
-      raise
-
-    return self
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    self._check_entered()
-
-    try:
-      # data_path refers to the externally used path to the params. It is a symlink.
-      # old_data_path is the path currently pointed to by data_path.
-      # tempdir_path is a path where the new params will go, which the new data path will point to.
-      # new_data_path is a temporary symlink that will atomically overwrite data_path.
-      #
-      # The current situation is:
-      #   data_path -> old_data_path
-      # We're going to write params data to tempdir_path
-      #   tempdir_path -> params data
-      # Then point new_data_path to tempdir_path
-      #   new_data_path -> tempdir_path
-      # Then atomically overwrite data_path with new_data_path
-      #   data_path -> tempdir_path
-      old_data_path = None
-      new_data_path = None
-      tempdir_path = tempfile.mkdtemp(prefix=".tmp", dir=self._path)
-
-      try:
-        # Write back all keys.
-        os.chmod(tempdir_path, 0o777)
-        for k, v in self._vals.items():
-          with open(os.path.join(tempdir_path, k), "wb") as f:
-            f.write(v)
-            f.flush()
-            os.fsync(f.fileno())
-        fsync_dir(tempdir_path)
-
-        data_path = self._data_path()
-        try:
-          old_data_path = os.path.join(self._path, os.readlink(data_path))
-        except (OSError, IOError):
-          # NOTE(mgraczyk): If other DB implementations have bugs, this could cause
-          #                 copies to be left behind, but we still want to overwrite.
-          pass
-
-        new_data_path = "{}.link".format(tempdir_path)
-        os.symlink(os.path.basename(tempdir_path), new_data_path)
-        os.rename(new_data_path, data_path)
-        fsync_dir(self._path)
-      finally:
-        # If the rename worked, we can delete the old data. Otherwise delete the new one.
-        success = new_data_path is not None and os.path.exists(data_path) and (
-          os.readlink(data_path) == os.path.basename(tempdir_path))
-
-        if success:
-          if old_data_path is not None:
-            shutil.rmtree(old_data_path)
-        else:
-          shutil.rmtree(tempdir_path)
-
-        # Regardless of what happened above, there should be no link at new_data_path.
-        if new_data_path is not None and os.path.islink(new_data_path):
-          os.remove(new_data_path)
-    finally:
-      os.umask(self._prev_umask)
-      self._prev_umask = None
-
-      # Always release the lock.
-      self._lock.release()
-      self._lock = None
-
-
-def read_db(params_path, key):
-  path = "%s/d/%s" % (params_path, key)
-  try:
-    with open(path, "rb") as f:
-      return f.read()
-  except IOError:
-    return None
-
-
-def write_db(params_path, key, value):
-  if isinstance(value, str):
-    value = value.encode('utf8')
-
-  prev_umask = os.umask(0)
-  lock = FileLock(params_path + "/.lock", True)
-  lock.acquire()
-
-  try:
-    tmp_path = tempfile.mktemp(prefix=".tmp", dir=params_path)
-    with open(tmp_path, "wb") as f:
-      f.write(value)
-      f.flush()
-      os.fsync(f.fileno())
-
-    path = "%s/d/%s" % (params_path, key)
-    os.rename(tmp_path, path)
-    fsync_dir(os.path.dirname(path))
-  finally:
-    os.umask(prev_umask)
-    lock.release()
-
-
-class Params():
-  def __init__(self, db=PARAMS):
-    self.db = db
-
-    # create the database if it doesn't exist...
-    if not os.path.exists(self.db + "/d"):
-      with self.transaction(write=True):
-        pass
-
-  def clear_all(self):
-    shutil.rmtree(self.db, ignore_errors=True)
-    with self.transaction(write=True):
-      pass
-
-  def transaction(self, write=False):
-    if write:
-      return DBWriter(self.db)
-    else:
-      return DBReader(self.db)
-
-  def _clear_keys_with_type(self, tx_type):
-    with self.transaction(write=True) as txn:
-      for key in keys:
-        if tx_type in keys[key]:
-          txn.delete(key)
-
-  def manager_start(self):
-    self._clear_keys_with_type(TxType.CLEAR_ON_MANAGER_START)
-
-  def panda_disconnect(self):
-    self._clear_keys_with_type(TxType.CLEAR_ON_PANDA_DISCONNECT)
-
-  def delete(self, key):
-    with self.transaction(write=True) as txn:
-      txn.delete(key)
-
-  def get(self, key, block=False, encoding=None):
-    if key not in keys:
-      raise UnknownKeyName(key)
-
-    while 1:
-      ret = read_db(self.db, key)
-      if not block or ret is not None:
-        break
-      # is polling really the best we can do?
-      time.sleep(0.05)
-
-    if ret is not None and encoding is not None:
-      ret = ret.decode(encoding)
-
-    return ret
-
-  def put(self, key, dat):
-    """
-    Warning: This function blocks until the param is written to disk!
-    In very rare cases this can take over a second, and your code will hang.
-
-    Use the put_nonblocking helper function in time sensitive code, but
-    in general try to avoid writing params as much as possible.
-    """
-
-    if key not in keys:
-      raise UnknownKeyName(key)
-
-    write_db(self.db, key, dat)
-
-
-def put_nonblocking(key, val):
-  def f(key, val):
-    params = Params()
-    params.put(key, val)
-
-  t = threading.Thread(target=f, args=(key, val))
-  t.start()
-  return t
+from common.params_pyx import Params, UnknownKeyName, put_nonblocking # pylint: disable=no-name-in-module, import-error
+assert Params
+assert UnknownKeyName
+assert put_nonblocking
+
+if __name__ == "__main__":
+  import sys
+  from common.params_pyx import keys # pylint: disable=no-name-in-module, import-error
+
+  params = Params()
+  if len(sys.argv) == 3:
+    name = sys.argv[1]
+    val = sys.argv[2]
+    assert name.encode("utf-8") in keys.keys(), f"unknown param: {name}"
+    print(f"SET: {name} = {val}")
+    params.put(name, val)
+  elif len(sys.argv) == 2:
+    name = sys.argv[1]
+    assert name.encode("utf-8") in keys.keys(), f"unknown param: {name}"
+    print(f"GET: {name} = {params.get(name)}")
+  else:
+    for k in keys.keys():
+      print(f"GET: {k} = {params.get(k)}")

common/params_pxd.pxd

@@ -0,0 +1,16 @@
+from libcpp.string cimport string
+from libcpp cimport bool
+
+cdef extern from "selfdrive/common/params.cc":
+  pass
+
+cdef extern from "selfdrive/common/util.cc":
+  pass
+
+cdef extern from "selfdrive/common/params.h":
+  cdef cppclass Params:
+    Params(bool)
+    Params(string)
+    string get(string, bool) nogil
+    int delete_db_value(string)
+    int write_db_value(string, string)

common/params_pyx.pyx

@@ -0,0 +1,174 @@
+# distutils: language = c++
+# cython: language_level = 3
+from libcpp cimport bool
+from libcpp.string cimport string
+from common.params_pxd cimport Params as c_Params
+
+import os
+import threading
+from common.basedir import BASEDIR
+
+cdef enum TxType:
+  PERSISTENT = 1
+  CLEAR_ON_MANAGER_START = 2
+  CLEAR_ON_PANDA_DISCONNECT = 3
+
+keys = {
+  b"AccessToken": [TxType.CLEAR_ON_MANAGER_START],
+  b"ApiCache_DriveStats": [TxType.PERSISTENT],
+  b"ApiCache_Device": [TxType.PERSISTENT],
+  b"ApiCache_Owner": [TxType.PERSISTENT],
+  b"AthenadPid": [TxType.PERSISTENT],
+  b"CalibrationParams": [TxType.PERSISTENT],
+  b"CarBatteryCapacity": [TxType.PERSISTENT],
+  b"CarParams": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"CarParamsCache": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"CarVin": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"CommunityFeaturesToggle": [TxType.PERSISTENT],
+  b"EndToEndToggle": [TxType.PERSISTENT],
+  b"CompletedTrainingVersion": [TxType.PERSISTENT],
+  b"DisablePowerDown": [TxType.PERSISTENT],
+  b"DisableUpdates": [TxType.PERSISTENT],
+  b"DoUninstall": [TxType.CLEAR_ON_MANAGER_START],
+  b"DongleId": [TxType.PERSISTENT],
+  b"GitDiff": [TxType.PERSISTENT],
+  b"GitBranch": [TxType.PERSISTENT],
+  b"GitCommit": [TxType.PERSISTENT],
+  b"GitRemote": [TxType.PERSISTENT],
+  b"GithubSshKeys": [TxType.PERSISTENT],
+  b"GithubUsername": [TxType.PERSISTENT],
+  b"HardwareSerial": [TxType.PERSISTENT],
+  b"HasAcceptedTerms": [TxType.PERSISTENT],
+  b"HasCompletedSetup": [TxType.PERSISTENT],
+  b"IsDriverViewEnabled": [TxType.CLEAR_ON_MANAGER_START],
+  b"IMEI": [TxType.PERSISTENT],
+  b"IsLdwEnabled": [TxType.PERSISTENT],
+  b"IsMetric": [TxType.PERSISTENT],
+  b"IsOffroad": [TxType.CLEAR_ON_MANAGER_START],
+  b"IsRHD": [TxType.PERSISTENT],
+  b"IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
+  b"IsUpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
+  b"IsUploadRawEnabled": [TxType.PERSISTENT],
+  b"LastAthenaPingTime": [TxType.PERSISTENT],
+  b"LastGPSPosition": [TxType.PERSISTENT],
+  b"LastUpdateException": [TxType.PERSISTENT],
+  b"LastUpdateTime": [TxType.PERSISTENT],
+  b"LiveParameters": [TxType.PERSISTENT],
+  b"OpenpilotEnabledToggle": [TxType.PERSISTENT],
+  b"PandaFirmware": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"PandaFirmwareHex": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"PandaDongleId": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"Passive": [TxType.PERSISTENT],
+  b"RecordFront": [TxType.PERSISTENT],
+  b"RecordFrontLock": [TxType.PERSISTENT],  # for the internal fleet
+  b"ReleaseNotes": [TxType.PERSISTENT],
+  b"ShouldDoUpdate": [TxType.CLEAR_ON_MANAGER_START],
+  b"SubscriberInfo": [TxType.PERSISTENT],
+  b"SshEnabled": [TxType.PERSISTENT],
+  b"TermsVersion": [TxType.PERSISTENT],
+  b"Timezone": [TxType.PERSISTENT],
+  b"TrainingVersion": [TxType.PERSISTENT],
+  b"UpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
+  b"UpdateFailedCount": [TxType.CLEAR_ON_MANAGER_START],
+  b"Version": [TxType.PERSISTENT],
+  b"VisionRadarToggle": [TxType.PERSISTENT],
+  b"Offroad_ChargeDisabled": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"Offroad_ConnectivityNeeded": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_ConnectivityNeededPrompt": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_TemperatureTooHigh": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_PandaFirmwareMismatch": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
+  b"Offroad_InvalidTime": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_NeosUpdate": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_UpdateFailed": [TxType.CLEAR_ON_MANAGER_START],
+  b"Offroad_HardwareUnsupported": [TxType.CLEAR_ON_MANAGER_START],
+  b"ForcePowerDown": [TxType.CLEAR_ON_MANAGER_START],
+}
+
+def ensure_bytes(v):
+  if isinstance(v, str):
+    return v.encode()
+  else:
+    return v
+
+
+class UnknownKeyName(Exception):
+  pass
+
+cdef class Params:
+  cdef c_Params* p
+
+  def __cinit__(self, d=None, bool persistent_params=False):
+    if d is None:
+      self.p = new c_Params(persistent_params)
+    else:
+      self.p = new c_Params(<string>d.encode())
+
+  def __dealloc__(self):
+    del self.p
+
+  def clear_all(self, tx_type=None):
+    for key in keys:
+      if tx_type is None or tx_type in keys[key]:
+        self.delete(key)
+
+  def manager_start(self):
+    self.clear_all(TxType.CLEAR_ON_MANAGER_START)
+
+  def panda_disconnect(self):
+    self.clear_all(TxType.CLEAR_ON_PANDA_DISCONNECT)
+
+  def get(self, key, block=False, encoding=None):
+    key = ensure_bytes(key)
+
+    if key not in keys:
+      raise UnknownKeyName(key)
+
+    cdef string k = key
+    cdef bool b = block
+
+    cdef string val
+    with nogil:
+      val = self.p.get(k, b)
+
+    if val == b"":
+      if block:
+        # If we got no value while running in blocked mode
+        # it means we got an interrupt while waiting
+        raise KeyboardInterrupt
+      else:
+        return None
+
+    if encoding is not None:
+      return val.decode(encoding)
+    else:
+      return val
+
+  def put(self, key, dat):
+    """
+    Warning: This function blocks until the param is written to disk!
+    In very rare cases this can take over a second, and your code will hang.
+    Use the put_nonblocking helper function in time sensitive code, but
+    in general try to avoid writing params as much as possible.
+    """
+    key = ensure_bytes(key)
+    dat = ensure_bytes(dat)
+
+    if key not in keys:
+      raise UnknownKeyName(key)
+
+    self.p.write_db_value(key, dat)
+
+  def delete(self, key):
+    key = ensure_bytes(key)
+    self.p.delete_db_value(key)
+
+
+def put_nonblocking(key, val, d=None):
+  def f(key, val):
+    params = Params(d)
+    params.put(key, val)
+
+  t = threading.Thread(target=f, args=(key, val))
+  t.start()
+  return t

common/profiler.py

@@ -36,10 +36,10 @@ class Profiler():
     if not self.enabled:
       return
     self.iter += 1
-    print("******* Profiling *******")
+    print("******* Profiling %d *******" % self.iter)
     for n, ms in sorted(self.cp.items(), key=lambda x: -x[1]):
       if n in self.cp_ignored:
-        print("%30s: %9.2f   percent: %3.0f   IGNORED" % (n, ms*1000.0, ms/self.tot*100))
+        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f   IGNORED" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
       else:
-        print("%30s: %9.2f   percent: %3.0f" % (n, ms*1000.0, ms/self.tot*100))
+        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
     print("Iter clock: %2.6f   TOTAL: %2.2f" % (self.tot/self.iter, self.tot))

common/realtime.py

@@ -1,53 +1,50 @@
 """Utilities for reading real time clocks and keeping soft real time constraints."""
+import gc
 import os
 import time
-import platform
-import subprocess
 import multiprocessing
-from cffi import FFI
 
-from common.android import ANDROID
-from common.common_pyx import sec_since_boot  # pylint: disable=no-name-in-module, import-error
+from common.clock import sec_since_boot  # pylint: disable=no-name-in-module, import-error
+from selfdrive.hardware import PC, TICI
 
 
 # time step for each process
 DT_CTRL = 0.01  # controlsd
 DT_MDL = 0.05  # model
-DT_DMON = 0.1  # driver monitoring
 DT_TRML = 0.5  # thermald and manager
 
+# driver monitoring
+if TICI:
+  DT_DMON = 0.05
+else:
+  DT_DMON = 0.1
 
-ffi = FFI()
-ffi.cdef("long syscall(long number, ...);")
-libc = ffi.dlopen(None)
 
-def _get_tid():
-  if platform.machine() == "x86_64":
-    NR_gettid = 186
-  elif platform.machine() == "aarch64":
-    NR_gettid = 178
-  else:
-    raise NotImplementedError
+class Priority:
+  # CORE 2
+  # - modeld = 55
+  # - camerad = 54
+  CTRL_LOW = 51 # plannerd & radard
 
-  return libc.syscall(NR_gettid)
+  # CORE 3
+  # - boardd = 55
+  CTRL_HIGH = 53
 
 
 def set_realtime_priority(level):
-  if os.getuid() != 0:
-    print("not setting priority, not root")
-    return
+  if not PC:
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
 
-  return subprocess.call(['chrt', '-f', '-p', str(level), str(_get_tid())])
 
 def set_core_affinity(core):
-  if os.getuid() != 0:
-    print("not setting affinity, not root")
-    return
+  if not PC:
+    os.sched_setaffinity(0, [core,])
 
-  if ANDROID:
-    return subprocess.call(['taskset', '-p', str(core), str(_get_tid())])
-  else:
-    return -1
+
+def config_realtime_process(core, priority):
+  gc.disable()
+  set_realtime_priority(priority)
+  set_core_affinity(core)
 
 
 class Ratekeeper():

common/spinner.py

@@ -8,7 +8,7 @@ class Spinner():
     try:
       self.spinner_proc = subprocess.Popen(["./spinner"],
                                            stdin=subprocess.PIPE,
-                                           cwd=os.path.join(BASEDIR, "selfdrive", "ui", "spinner"),
+                                           cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
                                            close_fds=True)
     except OSError:
       self.spinner_proc = None
@@ -16,7 +16,7 @@ class Spinner():
   def __enter__(self):
     return self
 
-  def update(self, spinner_text):
+  def update(self, spinner_text: str):
     if self.spinner_proc is not None:
       self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
       try:
@@ -24,6 +24,9 @@ class Spinner():
       except BrokenPipeError:
         pass
 
+  def update_progress(self, cur: int, total: int):
+    self.update(str(round(100 * cur / total)))
+
   def close(self):
     if self.spinner_proc is not None:
       try:
@@ -40,23 +43,6 @@ class Spinner():
     self.close()
 
 
-class FakeSpinner(Spinner):
-  def __init__(self):  # pylint: disable=super-init-not-called
-    pass
-
-  def __enter__(self):
-    return self
-
-  def update(self, _):
-    pass
-
-  def close(self):
-    pass
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    pass
-
-
 if __name__ == "__main__":
   import time
   with Spinner() as s:

common/text_window.py

@@ -5,12 +5,12 @@ import subprocess
 from common.basedir import BASEDIR
 
 
-class TextWindow():
-  def __init__(self, s):
+class TextWindow:
+  def __init__(self, text):
     try:
-      self.text_proc = subprocess.Popen(["./text", s],
+      self.text_proc = subprocess.Popen(["./text", text],
                                         stdin=subprocess.PIPE,
-                                        cwd=os.path.join(BASEDIR, "selfdrive", "ui", "text"),
+                                        cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
                                         close_fds=True)
     except OSError:
       self.text_proc = None
@@ -19,7 +19,6 @@ class TextWindow():
     if self.text_proc is not None:
       self.text_proc.poll()
       return self.text_proc.returncode
-
     return None
 
   def __enter__(self):
@@ -31,10 +30,11 @@ class TextWindow():
       self.text_proc = None
 
   def wait_for_exit(self):
-    while True:
-      if self.get_status() == 1:
-        return
-      time.sleep(0.1)
+    if self.text_proc is not None:
+      while True:
+        if self.get_status() == 1:
+          return
+        time.sleep(0.1)
 
   def __del__(self):
     self.close()
@@ -43,29 +43,6 @@ class TextWindow():
     self.close()
 
 
-class FakeTextWindow(TextWindow):
-  def __init__(self, s):  # pylint: disable=super-init-not-called
-    pass
-
-  def get_status(self):
-    return 1
-
-  def wait_for_exit(self):
-    return
-
-  def __enter__(self):
-    return self
-
-  def update(self, _):
-    pass
-
-  def close(self):
-    pass
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    pass
-
-
 if __name__ == "__main__":
   text = """Traceback (most recent call last):
   File "./controlsd.py", line 608, in <module>

common/transformations/SConscript

@@ -1,9 +1,6 @@
-Import('env')
+Import('env', 'envCython')
 
-d = Dir('.')
+transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc'])
+Export('transformations')
 
-env.Command(
-  ['transformations.so'],
-  ['transformations.pxd', 'transformations.pyx',
-   'coordinates.cc', 'orientation.cc', 'coordinates.hpp', 'orientation.hpp'],
-  'cd ' + d.path + ' && python3 setup.py build_ext --inplace')
+envCython.Program('transformations.so', 'transformations.pyx')

common/transformations/camera.py

@@ -1,29 +1,67 @@
 import numpy as np
-import common.transformations.orientation as orient
 
-FULL_FRAME_SIZE = (1164, 874)
-W, H = FULL_FRAME_SIZE[0], FULL_FRAME_SIZE[1]
-eon_focal_length = FOCAL = 910.0
+import common.transformations.orientation as orient
+from selfdrive.hardware import TICI
+
+## -- hardcoded hardware params --
+eon_f_focal_length = 910.0
+eon_d_focal_length = 860.0
+leon_d_focal_length = 650.0
+tici_f_focal_length = 2648.0
+tici_e_focal_length = tici_d_focal_length = 567.0 # probably wrong? magnification is not consistent across frame
+
+eon_f_frame_size = (1164, 874)
+eon_d_frame_size = (1152, 864)
+leon_d_frame_size = (816, 612)
+tici_f_frame_size = tici_e_frame_size = tici_d_frame_size = (1928, 1208)
 
 # aka 'K' aka camera_frame_from_view_frame
-eon_intrinsics = np.array([
-  [FOCAL,   0.,   W/2.],
-  [  0.,  FOCAL,  H/2.],
-  [  0.,    0.,     1.]])
-
+eon_fcam_intrinsics = np.array([
+  [eon_f_focal_length,  0.0,  float(eon_f_frame_size[0])/2],
+  [0.0,  eon_f_focal_length,  float(eon_f_frame_size[1])/2],
+  [0.0,  0.0,                                          1.0]])
+eon_intrinsics = eon_fcam_intrinsics # xx
 
 leon_dcam_intrinsics = np.array([
-  [650,   0,   816//2],
-  [  0,  650,  612//2],
-  [  0,    0,     1]])
+  [leon_d_focal_length,  0.0,  float(leon_d_frame_size[0])/2],
+  [0.0,  leon_d_focal_length,  float(leon_d_frame_size[1])/2],
+  [0.0,  0.0,                                            1.0]])
 
 eon_dcam_intrinsics = np.array([
-  [860,   0,   1152//2],
-  [  0,  860,  864//2],
-  [  0,    0,     1]])
+  [eon_d_focal_length,  0.0,  float(eon_d_frame_size[0])/2],
+  [0.0,  eon_d_focal_length,  float(eon_d_frame_size[1])/2],
+  [0.0,  0.0,                                          1.0]])
+
+tici_fcam_intrinsics = np.array([
+  [tici_f_focal_length,  0.0,  float(tici_f_frame_size[0])/2],
+  [0.0,  tici_f_focal_length,  float(tici_f_frame_size[1])/2],
+  [0.0,  0.0,                                            1.0]])
+
+tici_dcam_intrinsics = np.array([
+  [tici_d_focal_length,  0.0,  float(tici_d_frame_size[0])/2],
+  [0.0,  tici_d_focal_length,  float(tici_d_frame_size[1])/2],
+  [0.0,  0.0,                                            1.0]])
+
+tici_ecam_intrinsics = tici_dcam_intrinsics
 
 # aka 'K_inv' aka view_frame_from_camera_frame
-eon_intrinsics_inv = np.linalg.inv(eon_intrinsics)
+eon_fcam_intrinsics_inv = np.linalg.inv(eon_fcam_intrinsics)
+eon_intrinsics_inv = eon_fcam_intrinsics_inv # xx
+
+tici_fcam_intrinsics_inv = np.linalg.inv(tici_fcam_intrinsics)
+tici_ecam_intrinsics_inv = np.linalg.inv(tici_ecam_intrinsics)
+
+
+if not TICI:
+  FULL_FRAME_SIZE = eon_f_frame_size
+  FOCAL = eon_f_focal_length
+  fcam_intrinsics = eon_fcam_intrinsics
+else:
+  FULL_FRAME_SIZE = tici_f_frame_size
+  FOCAL = tici_f_focal_length
+  fcam_intrinsics = tici_fcam_intrinsics
+
+W, H = FULL_FRAME_SIZE[0], FULL_FRAME_SIZE[1]
 
 
 # device/mesh : x->forward, y-> right, z->down
@@ -52,6 +90,13 @@ def get_view_frame_from_road_frame(roll, pitch, yaw, height):
   return np.hstack((view_from_road, [[0], [height], [0]]))
 
 
+# aka 'extrinsic_matrix'
+def get_view_frame_from_calib_frame(roll, pitch, yaw, height):
+  device_from_calib= orient.rot_from_euler([roll, pitch, yaw])
+  view_from_calib = view_frame_from_device_frame.dot(device_from_calib)
+  return np.hstack((view_from_calib, [[0], [height], [0]]))
+
+
 def vp_from_ke(m):
   """
   Computes the vanishing point from the product of the intrinsic and extrinsic
@@ -62,9 +107,9 @@ def vp_from_ke(m):
   return (m[0, 0]/m[2, 0], m[1, 0]/m[2, 0])
 
 
-def vp_from_rpy(rpy):
+def vp_from_rpy(rpy, intrinsics=fcam_intrinsics):
   e = get_view_frame_from_road_frame(rpy[0], rpy[1], rpy[2], 1.22)
-  ke = np.dot(eon_intrinsics, e)
+  ke = np.dot(intrinsics, e)
   return vp_from_ke(ke)
 
 
@@ -74,7 +119,7 @@ def roll_from_ke(m):
                     -(m[0, 0] - m[0, 1] * m[2, 0] / m[2, 1]))
 
 
-def normalize(img_pts, intrinsics=eon_intrinsics):
+def normalize(img_pts, intrinsics=fcam_intrinsics):
   # normalizes image coordinates
   # accepts single pt or array of pts
   intrinsics_inv = np.linalg.inv(intrinsics)
@@ -87,7 +132,7 @@ def normalize(img_pts, intrinsics=eon_intrinsics):
   return img_pts_normalized[:, :2].reshape(input_shape)
 
 
-def denormalize(img_pts, intrinsics=eon_intrinsics):
+def denormalize(img_pts, intrinsics=fcam_intrinsics, width=W, height=H):
   # denormalizes image coordinates
   # accepts single pt or array of pts
   img_pts = np.array(img_pts)
@@ -95,9 +140,9 @@ def denormalize(img_pts, intrinsics=eon_intrinsics):
   img_pts = np.atleast_2d(img_pts)
   img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1))))
   img_pts_denormalized = img_pts.dot(intrinsics.T)
-  img_pts_denormalized[img_pts_denormalized[:, 0] > W] = np.nan
+  img_pts_denormalized[img_pts_denormalized[:, 0] > width] = np.nan
   img_pts_denormalized[img_pts_denormalized[:, 0] < 0] = np.nan
-  img_pts_denormalized[img_pts_denormalized[:, 1] > H] = np.nan
+  img_pts_denormalized[img_pts_denormalized[:, 1] > height] = np.nan
   img_pts_denormalized[img_pts_denormalized[:, 1] < 0] = np.nan
   return img_pts_denormalized[:, :2].reshape(input_shape)
 
@@ -130,18 +175,10 @@ def img_from_device(pt_device):
   return pt_img.reshape(input_shape)[:, :2]
 
 
-def get_camera_frame_from_calib_frame(camera_frame_from_road_frame):
+def get_camera_frame_from_calib_frame(camera_frame_from_road_frame, intrinsics=fcam_intrinsics):
   camera_frame_from_ground = camera_frame_from_road_frame[:, (0, 1, 3)]
-  calib_frame_from_ground = np.dot(eon_intrinsics,
+  calib_frame_from_ground = np.dot(intrinsics,
                                      get_view_frame_from_road_frame(0, 0, 0, 1.22))[:, (0, 1, 3)]
   ground_from_calib_frame = np.linalg.inv(calib_frame_from_ground)
   camera_frame_from_calib_frame = np.dot(camera_frame_from_ground, ground_from_calib_frame)
   return camera_frame_from_calib_frame
-
-
-def pretransform_from_calib(calib):
-  roll, pitch, yaw, height = calib
-  view_frame_from_road_frame = get_view_frame_from_road_frame(roll, pitch, yaw, height)
-  camera_frame_from_road_frame = np.dot(eon_intrinsics, view_frame_from_road_frame)
-  camera_frame_from_calib_frame = get_camera_frame_from_calib_frame(camera_frame_from_road_frame)
-  return np.linalg.inv(camera_frame_from_calib_frame)

common/transformations/coordinates.cc

@@ -10,9 +10,9 @@
 #define RAD2DEG(x) ((x) * 180.0 / M_PI)
 
 
-double a = 6378137;
-double b = 6356752.3142;
-double esq = 6.69437999014 * 0.001;
+double a = 6378137; // lgtm [cpp/short-global-name]
+double b = 6356752.3142; // lgtm [cpp/short-global-name]
+double esq = 6.69437999014 * 0.001; // lgtm [cpp/short-global-name]
 double e1sq = 6.73949674228 * 0.001;
 
 

common/transformations/model.py

@@ -1,33 +1,33 @@
 import numpy as np
 
-from common.transformations.camera import (FULL_FRAME_SIZE, eon_focal_length,
+from common.transformations.camera import (FULL_FRAME_SIZE,
+                                           FOCAL,
                                            get_view_frame_from_road_frame,
+                                           get_view_frame_from_calib_frame,
                                            vp_from_ke)
 
 # segnet
-
 SEGNET_SIZE = (512, 384)
 
-segnet_frame_from_camera_frame = np.array([
-  [float(SEGNET_SIZE[0])/FULL_FRAME_SIZE[0],    0.,          ],
-  [     0.,          float(SEGNET_SIZE[1])/FULL_FRAME_SIZE[1]]])
-
+def get_segnet_frame_from_camera_frame(segnet_size=SEGNET_SIZE, full_frame_size=FULL_FRAME_SIZE):
+  return np.array([[float(segnet_size[0]) / full_frame_size[0],  0.0],
+                   [0.0,  float(segnet_size[1]) / full_frame_size[1]]])
+segnet_frame_from_camera_frame = get_segnet_frame_from_camera_frame() # xx
 
 # model
-
 MODEL_INPUT_SIZE = (320, 160)
 MODEL_YUV_SIZE = (MODEL_INPUT_SIZE[0], MODEL_INPUT_SIZE[1] * 3 // 2)
-MODEL_CX = MODEL_INPUT_SIZE[0]/2.
+MODEL_CX = MODEL_INPUT_SIZE[0] / 2.
 MODEL_CY = 21.
 
-model_zoom = 1.25
+model_fl = 728.0
 model_height = 1.22
 
 # canonical model transform
-model_intrinsics = np.array(
-  [[ eon_focal_length / model_zoom,    0. ,  MODEL_CX],
-   [   0. ,  eon_focal_length / model_zoom,  MODEL_CY],
-   [   0. ,                            0. ,   1.]])
+model_intrinsics = np.array([
+  [model_fl,  0.0,  MODEL_CX],
+  [0.0,  model_fl,  MODEL_CY],
+  [0.0,  0.0,            1.0]])
 
 
 # MED model
@@ -35,34 +35,45 @@ MEDMODEL_INPUT_SIZE = (512, 256)
 MEDMODEL_YUV_SIZE = (MEDMODEL_INPUT_SIZE[0], MEDMODEL_INPUT_SIZE[1] * 3 // 2)
 MEDMODEL_CY = 47.6
 
-medmodel_zoom = 1.
-medmodel_intrinsics = np.array(
-  [[ eon_focal_length / medmodel_zoom,    0. ,  0.5 * MEDMODEL_INPUT_SIZE[0]],
-   [   0. ,  eon_focal_length / medmodel_zoom,  MEDMODEL_CY],
-   [   0. ,                            0. ,   1.]])
+medmodel_fl = 910.0
+medmodel_intrinsics = np.array([
+  [medmodel_fl,  0.0,  0.5 * MEDMODEL_INPUT_SIZE[0]],
+  [0.0,  medmodel_fl,                   MEDMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
+
 
 # CAL model
 CALMODEL_INPUT_SIZE = (512, 256)
 CALMODEL_YUV_SIZE = (CALMODEL_INPUT_SIZE[0], CALMODEL_INPUT_SIZE[1] * 3 // 2)
 CALMODEL_CY = 47.6
 
-calmodel_zoom = 1.5
-calmodel_intrinsics = np.array(
-  [[ eon_focal_length / calmodel_zoom,    0. ,  0.5 * CALMODEL_INPUT_SIZE[0]],
-   [   0. ,  eon_focal_length / calmodel_zoom,  CALMODEL_CY],
-   [   0. ,                            0. ,   1.]])
+calmodel_fl = 606.7
+calmodel_intrinsics = np.array([
+  [calmodel_fl,  0.0,  0.5 * CALMODEL_INPUT_SIZE[0]],
+  [0.0,  calmodel_fl,                   CALMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
 
 
 # BIG model
-
 BIGMODEL_INPUT_SIZE = (1024, 512)
 BIGMODEL_YUV_SIZE = (BIGMODEL_INPUT_SIZE[0], BIGMODEL_INPUT_SIZE[1] * 3 // 2)
 
-bigmodel_zoom = 1.
-bigmodel_intrinsics = np.array(
-  [[ eon_focal_length / bigmodel_zoom,    0. , 0.5 * BIGMODEL_INPUT_SIZE[0]],
-   [   0. ,  eon_focal_length / bigmodel_zoom,  256+MEDMODEL_CY],
-   [   0. ,                            0. ,   1.]])
+bigmodel_fl = 910.0
+bigmodel_intrinsics = np.array([
+  [bigmodel_fl,  0.0,  0.5 * BIGMODEL_INPUT_SIZE[0]],
+  [0.0,  bigmodel_fl,             256 + MEDMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
+
+
+# SBIG model (big model with the size of small model)
+SBIGMODEL_INPUT_SIZE = (512, 256)
+SBIGMODEL_YUV_SIZE = (SBIGMODEL_INPUT_SIZE[0], SBIGMODEL_INPUT_SIZE[1] * 3 // 2)
+
+sbigmodel_fl = 455.0
+sbigmodel_intrinsics = np.array([
+  [sbigmodel_fl,  0.0,  0.5 * SBIGMODEL_INPUT_SIZE[0]],
+  [0.0,  sbigmodel_fl,      0.5 * (256 + MEDMODEL_CY)],
+  [0.0,  0.0,                                     1.0]])
 
 model_frame_from_road_frame = np.dot(model_intrinsics,
   get_view_frame_from_road_frame(0, 0, 0, model_height))
@@ -73,23 +84,27 @@ bigmodel_frame_from_road_frame = np.dot(bigmodel_intrinsics,
 medmodel_frame_from_road_frame = np.dot(medmodel_intrinsics,
   get_view_frame_from_road_frame(0, 0, 0, model_height))
 
+medmodel_frame_from_calib_frame = np.dot(medmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
 model_frame_from_bigmodel_frame = np.dot(model_intrinsics, np.linalg.inv(bigmodel_intrinsics))
 medmodel_frame_from_bigmodel_frame = np.dot(medmodel_intrinsics, np.linalg.inv(bigmodel_intrinsics))
 
+
 # 'camera from model camera'
 def get_model_height_transform(camera_frame_from_road_frame, height):
   camera_frame_from_road_ground = np.dot(camera_frame_from_road_frame, np.array([
-      [1, 0, 0],
-      [0, 1, 0],
-      [0, 0, 0],
-      [0, 0, 1],
+    [1, 0, 0],
+    [0, 1, 0],
+    [0, 0, 0],
+    [0, 0, 1],
   ]))
 
   camera_frame_from_road_high = np.dot(camera_frame_from_road_frame, np.array([
-      [1, 0, 0],
-      [0, 1, 0],
-      [0, 0, height - model_height],
-      [0, 0, 1],
+    [1, 0, 0],
+    [0, 1, 0],
+    [0, 0, height - model_height],
+    [0, 0, 1],
   ]))
 
   road_high_from_camera_frame = np.linalg.inv(camera_frame_from_road_high)
@@ -100,13 +115,14 @@ def get_model_height_transform(camera_frame_from_road_frame, height):
 
 # camera_frame_from_model_frame aka 'warp matrix'
 # was: calibration.h/CalibrationTransform
-def get_camera_frame_from_model_frame(camera_frame_from_road_frame, height=model_height):
+def get_camera_frame_from_model_frame(camera_frame_from_road_frame, height=model_height, camera_fl=FOCAL):
   vp = vp_from_ke(camera_frame_from_road_frame)
 
+  model_zoom = camera_fl / model_fl
   model_camera_from_model_frame = np.array([
-    [model_zoom,         0., vp[0] - MODEL_CX * model_zoom],
-    [        0., model_zoom, vp[1] - MODEL_CY * model_zoom],
-    [        0.,         0.,                            1.],
+    [model_zoom,  0.0,  vp[0] - MODEL_CX * model_zoom],
+    [0.0,  model_zoom,  vp[1] - MODEL_CY * model_zoom],
+    [0.0,  0.0,                                   1.0],
   ])
 
   # This function is super slow, so skip it if height is very close to canonical

common/transformations/orientation.cc

@@ -39,7 +39,7 @@ Eigen::Matrix3d quat2rot(Eigen::Quaterniond quat){
   return quat.toRotationMatrix();
 }
 
-Eigen::Quaterniond rot2quat(Eigen::Matrix3d rot){
+Eigen::Quaterniond rot2quat(const Eigen::Matrix3d &rot){
   return ensure_unique(Eigen::Quaterniond(rot));
 }
 
@@ -47,7 +47,7 @@ Eigen::Matrix3d euler2rot(Eigen::Vector3d euler){
   return quat2rot(euler2quat(euler));
 }
 
-Eigen::Vector3d rot2euler(Eigen::Matrix3d rot){
+Eigen::Vector3d rot2euler(const Eigen::Matrix3d &rot){
   return quat2euler(rot2quat(rot));
 }
 
@@ -141,7 +141,3 @@ Eigen::Vector3d ned_euler_from_ecef(ECEF ecef_init, Eigen::Vector3d ecef_pose){
   return {phi, theta, psi};
 }
 
-
-
-int main(void){
-}

common/transformations/orientation.hpp

@@ -8,9 +8,9 @@ Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat);
 Eigen::Quaterniond euler2quat(Eigen::Vector3d euler);
 Eigen::Vector3d quat2euler(Eigen::Quaterniond quat);
 Eigen::Matrix3d quat2rot(Eigen::Quaterniond quat);
-Eigen::Quaterniond rot2quat(Eigen::Matrix3d rot);
+Eigen::Quaterniond rot2quat(const Eigen::Matrix3d &rot);
 Eigen::Matrix3d euler2rot(Eigen::Vector3d euler);
-Eigen::Vector3d rot2euler(Eigen::Matrix3d rot);
+Eigen::Vector3d rot2euler(const Eigen::Matrix3d &rot);
 Eigen::Matrix3d rot_matrix(double roll, double pitch, double yaw);
 Eigen::Matrix3d rot(Eigen::Vector3d axis, double angle);
 Eigen::Vector3d ecef_euler_from_ned(ECEF ecef_init, Eigen::Vector3d ned_pose);

common/transformations/orientation.py

@@ -46,7 +46,6 @@ quats_from_rotations = rot2quat
 quat_from_rot = rot2quat
 rotations_from_quats = quat2rot
 rot_from_quat = quat2rot
-rot_from_quat = quat2rot
 euler_from_rot = rot2euler
 euler_from_quat = quat2euler
 rot_from_euler = euler2rot

common/transformations/setup.py

@@ -1,42 +0,0 @@
-import os
-import numpy
-import sysconfig
-
-from Cython.Build import cythonize
-from Cython.Distutils import build_ext
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-
-def get_ext_filename_without_platform_suffix(filename):
-  name, ext = os.path.splitext(filename)
-  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
-
-  if ext_suffix == ext:
-    return filename
-
-  ext_suffix = ext_suffix.replace(ext, '')
-  idx = name.find(ext_suffix)
-
-  if idx == -1:
-    return filename
-  else:
-    return name[:idx] + ext
-
-
-class BuildExtWithoutPlatformSuffix(build_ext):
-  def get_ext_filename(self, ext_name):
-    filename = super().get_ext_filename(ext_name)
-    return get_ext_filename_without_platform_suffix(filename)
-
-
-setup(
-  name='Cython transformations wrapper',
-  cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-  ext_modules=cythonize(
-  Extension(
-    "transformations",
-    sources=["transformations.pyx"],
-    language="c++",
-    extra_compile_args=["-std=c++14"],
-    include_dirs=[numpy.get_include()],
-  )
-))

common/transformations/transformations.pxd

@@ -1,3 +1,4 @@
+#cython: language_level=3
 from libcpp cimport bool
 
 cdef extern from "orientation.cc":

common/transformations/transformations.pyx

@@ -1,18 +1,20 @@
-from transformations cimport Matrix3, Vector3, Quaternion
-from transformations cimport ECEF, NED, Geodetic
+# distutils: language = c++
+# cython: language_level = 3
+from common.transformations.transformations cimport Matrix3, Vector3, Quaternion
+from common.transformations.transformations cimport ECEF, NED, Geodetic
 
-from transformations cimport euler2quat as euler2quat_c
-from transformations cimport quat2euler as quat2euler_c
-from transformations cimport quat2rot as quat2rot_c
-from transformations cimport rot2quat as rot2quat_c
-from transformations cimport euler2rot as euler2rot_c
-from transformations cimport rot2euler as rot2euler_c
-from transformations cimport rot_matrix as rot_matrix_c
-from transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
-from transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
-from transformations cimport geodetic2ecef as geodetic2ecef_c
-from transformations cimport ecef2geodetic as ecef2geodetic_c
-from transformations cimport LocalCoord_c
+from common.transformations.transformations cimport euler2quat as euler2quat_c
+from common.transformations.transformations cimport quat2euler as quat2euler_c
+from common.transformations.transformations cimport quat2rot as quat2rot_c
+from common.transformations.transformations cimport rot2quat as rot2quat_c
+from common.transformations.transformations cimport euler2rot as euler2rot_c
+from common.transformations.transformations cimport rot2euler as rot2euler_c
+from common.transformations.transformations cimport rot_matrix as rot_matrix_c
+from common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
+from common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
+from common.transformations.transformations cimport LocalCoord_c
 
 
 import cython
@@ -26,7 +28,7 @@ cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m):
         [m(2, 0), m(2, 1), m(2, 2)],
     ])
 
-cdef Matrix3 numpy2matrix (np.ndarray[double, ndim=2, mode="fortran"] m):
+cdef Matrix3 numpy2matrix(np.ndarray[double, ndim=2, mode="fortran"] m):
     assert m.shape[0] == 3
     assert m.shape[1] == 3
     return Matrix3(<double*>m.data)

installer/updater/Makefile

@@ -10,7 +10,7 @@ WARN_FLAGS = -Werror=implicit-function-declaration \
              -Werror=format-extra-args
 
 CFLAGS = -std=gnu11 -g -fPIC -O2 $(WARN_FLAGS)
-CXXFLAGS = -std=c++11 -g -fPIC -O2 $(WARN_FLAGS)
+CXXFLAGS = -std=c++1z -g -fPIC -O2 $(WARN_FLAGS)
 
 CURL_FLAGS = -I$(PHONELIBS)/curl/include
 CURL_LIBS = $(PHONELIBS)/curl/lib/libcurl.a \
@@ -34,6 +34,7 @@ all: updater
 OBJS = opensans_regular.ttf.o \
 			 opensans_semibold.ttf.o \
 			 opensans_bold.ttf.o \
+       ../../selfdrive/common/util.o \
        ../../selfdrive/common/touch.o \
        ../../selfdrive/common/framebuffer.o \
        $(PHONELIBS)/json11/json11.o \

installer/updater/update.json

@@ -1,7 +1,7 @@
 {
-  "ota_url": "https://commadist.azureedge.net/neosupdate/ota-signed-efdf7de63b1aef63d68301e6175930991bf9a5927d16ec6fcc69287e2ee7ca4a.zip",
-  "ota_hash": "efdf7de63b1aef63d68301e6175930991bf9a5927d16ec6fcc69287e2ee7ca4a",
-  "recovery_url": "https://commadist.azureedge.net/neosupdate/recovery-97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e.img",
-  "recovery_len": 15861036,
-  "recovery_hash": "97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e"
+  "ota_url": "https://commadist.azureedge.net/neosupdate/ota-signed-969e22c42e5c6314e54bc3ccaa5c6a684f3130a53a7a70e0cea9f1453ceb0b06.zip",
+  "ota_hash": "969e22c42e5c6314e54bc3ccaa5c6a684f3130a53a7a70e0cea9f1453ceb0b06",
+  "recovery_url": "https://commadist.azureedge.net/neosupdate/recovery-9c784a24826c25df315d0ace864224478e9c0e86b904f5d1f8e18ea1037e842b.img",
+  "recovery_len": 15209772,
+  "recovery_hash": "9c784a24826c25df315d0ace864224478e9c0e86b904f5d1f8e18ea1037e842b"
 }

installer/updater/update_kernel.json

@@ -1,7 +0,0 @@
-{
-  "ota_url": "https://commadist.azureedge.net/neosupdate/ota-signed-3bd2b3bdd6a501569e00b8f12786d65e0fd2788c0dd238f8c986e3e2e504683a-kernel.zip",
-  "ota_hash": "3bd2b3bdd6a501569e00b8f12786d65e0fd2788c0dd238f8c986e3e2e504683a",
-  "recovery_url": "https://commadist.azureedge.net/neosupdate/recovery-97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e.img",
-  "recovery_len": 15861036,
-  "recovery_hash": "97c27e6ed04ed6bb0608b845a2d4100912093f9380c3f2ba6b56bccd608e5f6e"
-}

installer/updater/updater.cc

@@ -10,7 +10,9 @@
 #include <string>
 #include <sstream>
 #include <fstream>
+#include <iostream>
 #include <mutex>
+#include <memory>
 #include <thread>
 
 #include <curl/curl.h>
@@ -29,14 +31,14 @@
 
 #include "common/framebuffer.h"
 #include "common/touch.h"
-#include "common/utilpp.h"
+#include "common/util.h"
 
 #define USER_AGENT "NEOSUpdater-0.2"
 
-#define MANIFEST_URL_EON_STAGING "https://github.com/commaai/eon-neos/raw/master/update.staging.json"
-#define MANIFEST_URL_EON_LOCAL "http://192.168.5.1:8000/neosupdate/update.local.json"
-#define MANIFEST_URL_EON "https://github.com/commaai/eon-neos/raw/master/update.json"
-const char *manifest_url = MANIFEST_URL_EON;
+#define MANIFEST_URL_NEOS_STAGING "https://github.com/commaai/eon-neos/raw/master/update.staging.json"
+#define MANIFEST_URL_NEOS_LOCAL "http://192.168.5.1:8000/neosupdate/update.local.json"
+#define MANIFEST_URL_NEOS "https://github.com/commaai/eon-neos/raw/master/update.json"
+const char *manifest_url = MANIFEST_URL_NEOS;
 
 #define RECOVERY_DEV "/dev/block/bootdevice/by-name/recovery"
 #define RECOVERY_COMMAND "/cache/recovery/command"
@@ -96,7 +98,7 @@ std::string download_string(CURL *curl, std::string url) {
 
   curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
   curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
-  curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+  curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 0);
   curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT);
   curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
   curl_easy_setopt(curl, CURLOPT_RESUME_FROM, 0);
@@ -149,6 +151,32 @@ static void start_settings_activity(const char* name) {
   system(launch_cmd);
 }
 
+bool is_settings_active() {
+  FILE *fp;
+  char sys_output[4096];
+
+  fp = popen("/bin/dumpsys window windows", "r");
+  if (fp == NULL) {
+    return false;
+  }
+
+  bool active = false;
+  while (fgets(sys_output, sizeof(sys_output), fp) != NULL) {
+    if (strstr(sys_output, "mCurrentFocus=null")  != NULL) {
+      break;
+    }
+
+    if (strstr(sys_output, "mCurrentFocus=Window") != NULL) {
+      active = true;
+      break;
+    }
+  }
+
+  pclose(fp);
+
+  return active;
+}
+
 struct Updater {
   bool do_exit = false;
 
@@ -156,7 +184,7 @@ struct Updater {
 
   int fb_w, fb_h;
 
-  FramebufferState *fb = NULL;
+  std::unique_ptr<FrameBuffer> fb;
   NVGcontext *vg = NULL;
   int font_regular;
   int font_semibold;
@@ -166,7 +194,6 @@ struct Updater {
 
   std::mutex lock;
 
-  // i hate state machines give me coroutines already
   enum UpdateState {
     CONFIRMATION,
     LOW_BATTERY,
@@ -190,14 +217,20 @@ struct Updater {
   int b_x, b_w, b_y, b_h;
   int balt_x;
 
+  // download stage writes these for the installation stage
+  int recovery_len;
+  std::string recovery_hash;
+  std::string recovery_fn;
+  std::string ota_fn;
+
   CURL *curl = NULL;
 
-  Updater() {
+  void ui_init() {
     touch_init(&touch);
 
-    fb = framebuffer_init("updater", 0x00001000, false,
-                          &fb_w, &fb_h);
-    assert(fb);
+    fb = std::make_unique<FrameBuffer>("updater", 0x00001000, false, &fb_w, &fb_h);
+
+    fb->set_power(HWC_POWER_MODE_NORMAL);
 
     vg = nvgCreateGLES3(NVG_ANTIALIAS | NVG_STENCIL_STROKES | NVG_DEBUG);
     assert(vg);
@@ -218,7 +251,6 @@ struct Updater {
     b_h = 220;
 
     state = CONFIRMATION;
-
   }
 
   int download_file_xferinfo(curl_off_t dltotal, curl_off_t dlno,
@@ -251,7 +283,7 @@ struct Updater {
 
       curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
       curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
-      curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+      curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 0);
       curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT);
       curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
       curl_easy_setopt(curl, CURLOPT_RESUME_FROM, resume_from);
@@ -319,92 +351,78 @@ struct Updater {
     state = RUNNING;
   }
 
-  std::string stage_download(std::string url, std::string hash, std::string name) {
+  std::string download(std::string url, std::string hash, std::string name) {
     std::string out_fn = UPDATE_DIR "/" + util::base_name(url);
 
-    set_progress("Downloading " + name + "...");
-    bool r = download_file(url, out_fn);
-    if (!r) {
-      set_error("failed to download " + name);
-      return "";
+    // start or resume downloading if hash doesn't match
+    std::string fn_hash = sha256_file(out_fn);
+    if (hash.compare(fn_hash) != 0) {
+      set_progress("Downloading " + name + "...");
+      bool r = download_file(url, out_fn);
+      if (!r) {
+        set_error("failed to download " + name);
+        unlink(out_fn.c_str());
+        return "";
+      }
+      fn_hash = sha256_file(out_fn);
     }
 
     set_progress("Verifying " + name + "...");
-    std::string fn_hash = sha256_file(out_fn);
     printf("got %s hash: %s\n", name.c_str(), hash.c_str());
     if (fn_hash != hash) {
       set_error(name + " was corrupt");
       unlink(out_fn.c_str());
       return "";
     }
-
     return out_fn;
   }
 
-  void run_stages() {
+  bool download_stage() {
     curl = curl_easy_init();
     assert(curl);
 
-    if (!check_battery()) {
-      set_battery_low();
-      int battery_cap = battery_capacity();
-      while(battery_cap < min_battery_cap) {
-        battery_cap = battery_capacity();
-        battery_cap_text = std::to_string(battery_cap);
-        usleep(1000000);
-      }
-      set_running();
-    }
+    // ** quick checks before download **
 
     if (!check_space()) {
       set_error("2GB of free space required to update");
-      return;
+      return false;
     }
 
     mkdir(UPDATE_DIR, 0777);
 
-    const int EON = (access("/EON", F_OK) != -1);
-
     set_progress("Finding latest version...");
-    std::string manifest_s;
-    if (EON) {
-      manifest_s = download_string(curl, manifest_url);
-    } else {
-      // don't update NEO
-      exit(0);
-    }
-
+    std::string manifest_s = download_string(curl, manifest_url);
     printf("manifest: %s\n", manifest_s.c_str());
 
     std::string err;
     auto manifest = json11::Json::parse(manifest_s, err);
     if (manifest.is_null() || !err.empty()) {
       set_error("failed to load update manifest");
-      return;
+      return false;
     }
 
     std::string ota_url = manifest["ota_url"].string_value();
     std::string ota_hash = manifest["ota_hash"].string_value();
 
     std::string recovery_url = manifest["recovery_url"].string_value();
-    std::string recovery_hash = manifest["recovery_hash"].string_value();
-    int recovery_len = manifest["recovery_len"].int_value();
+    recovery_hash = manifest["recovery_hash"].string_value();
+    recovery_len = manifest["recovery_len"].int_value();
 
     // std::string installer_url = manifest["installer_url"].string_value();
     // std::string installer_hash = manifest["installer_hash"].string_value();
 
     if (ota_url.empty() || ota_hash.empty()) {
       set_error("invalid update manifest");
-      return;
+      return false;
     }
 
-    // std::string installer_fn = stage_download(installer_url, installer_hash, "installer");
+    // std::string installer_fn = download(installer_url, installer_hash, "installer");
     // if (installer_fn.empty()) {
     //   //error'd
     //   return;
     // }
 
-    std::string recovery_fn;
+    // ** handle recovery download **
     if (recovery_url.empty() || recovery_hash.empty() || recovery_len == 0) {
       set_progress("Skipping recovery flash...");
     } else {
@@ -414,27 +432,57 @@ struct Updater {
       printf("existing recovery hash: %s\n", existing_recovery_hash.c_str());
 
       if (existing_recovery_hash != recovery_hash) {
-        recovery_fn = stage_download(recovery_url, recovery_hash, "recovery");
+        recovery_fn = download(recovery_url, recovery_hash, "recovery");
         if (recovery_fn.empty()) {
           // error'd
-          return;
+          return false;
         }
       }
     }
 
-    std::string ota_fn = stage_download(ota_url, ota_hash, "update");
+    // ** handle ota download **
+    ota_fn = download(ota_url, ota_hash, "update");
     if (ota_fn.empty()) {
       //error'd
-      return;
+      return false;
     }
 
+    // download sucessful
+    return true;
+  }
+
+  // thread that handles downloading and installing the update
+  void run_stages() {
+    printf("run_stages start\n");
+
+
+    // ** download update **
+
     if (!check_battery()) {
       set_battery_low();
       int battery_cap = battery_capacity();
       while(battery_cap < min_battery_cap) {
         battery_cap = battery_capacity();
         battery_cap_text = std::to_string(battery_cap);
-        usleep(1000000);
+        util::sleep_for(1000);
+      }
+      set_running();
+    }
+
+    bool sucess = download_stage();
+    if (!sucess) {
+      return;
+    }
+
+    // ** install update **
+
+    if (!check_battery()) {
+      set_battery_low();
+      int battery_cap = battery_capacity();
+      while(battery_cap < min_battery_cap) {
+        battery_cap = battery_capacity();
+        battery_cap_text = std::to_string(battery_cap);
+        util::sleep_for(1000);
       }
       set_running();
     }
@@ -601,7 +649,7 @@ struct Updater {
       int powerprompt_y = 312;
       nvgFontFace(vg, "opensans_regular");
       nvgFontSize(vg, 64.0f);
-      nvgText(vg, fb_w/2, 740, "Ensure EON is connected to power.", NULL);
+      nvgText(vg, fb_w/2, 740, "Ensure your device remains connected to a power source.", NULL);
 
       NVGpaint paint = nvgBoxGradient(
           vg, progress_x + 1, progress_y + 1,
@@ -657,9 +705,7 @@ struct Updater {
   void ui_update() {
     std::lock_guard<std::mutex> guard(lock);
 
-    switch (state) {
-    case ERROR:
-    case CONFIRMATION: {
+    if (state == ERROR || state == CONFIRMATION) {
       int touch_x = -1, touch_y = -1;
       int res = touch_poll(&touch, &touch_x, &touch_y, 0);
       if (res == 1 && !is_settings_active()) {
@@ -678,13 +724,11 @@ struct Updater {
         }
       }
     }
-    default:
-      break;
-    }
   }
 
-
   void go() {
+    ui_init();
+
     while (!do_exit) {
       ui_update();
 
@@ -706,63 +750,49 @@ struct Updater {
 
       glDisable(GL_BLEND);
 
-      framebuffer_swap(fb);
+      fb->swap();
 
       assert(glGetError() == GL_NO_ERROR);
 
       // no simple way to do 30fps vsync with surfaceflinger...
-      usleep(30000);
+      util::sleep_for(30);
     }
 
     if (update_thread_handle.joinable()) {
       update_thread_handle.join();
     }
 
+    // reboot
     system("service call power 16 i32 0 i32 0 i32 1");
   }
 
-  bool is_settings_active() {
-    FILE *fp;
-    char sys_output[4096];
-
-    fp = popen("/bin/dumpsys window windows", "r");
-    if (fp == NULL) {
-      return false;
-    }
-
-    bool active = false;
-    while (fgets(sys_output, sizeof(sys_output), fp) != NULL) {
-      if (strstr(sys_output, "mCurrentFocus=null")  != NULL) {
-        break;
-      }
-
-      if (strstr(sys_output, "mCurrentFocus=Window") != NULL) {
-        active = true;
-        break;
-      }
-    }
-
-    pclose(fp);
-
-    return active;
-  }
-
 };
 
 }
+
 int main(int argc, char *argv[]) {
+  bool background_cache = false;
   if (argc > 1) {
     if (strcmp(argv[1], "local") == 0) {
-      manifest_url = MANIFEST_URL_EON_LOCAL;
+      manifest_url = MANIFEST_URL_NEOS_LOCAL;
     } else if (strcmp(argv[1], "staging") == 0) {
-      manifest_url = MANIFEST_URL_EON_STAGING;
+      manifest_url = MANIFEST_URL_NEOS_STAGING;
+    } else if (strcmp(argv[1], "bgcache") == 0) {
+      manifest_url = argv[2];
+      background_cache = true;
     } else {
       manifest_url = argv[1];
     }
   }
+
   printf("updating from %s\n", manifest_url);
   Updater updater;
-  updater.go();
 
-  return 0;
+  int err = 0;
+  if (background_cache) {
+    err = !updater.download_stage();
+  } else {
+    updater.go();
+  }
+  return err;
 }

launch_chffrplus.sh

@@ -1,25 +1,170 @@
 #!/usr/bin/bash
 
-export OMP_NUM_THREADS=1
-export MKL_NUM_THREADS=1
-export NUMEXPR_NUM_THREADS=1
-export OPENBLAS_NUM_THREADS=1
-export VECLIB_MAXIMUM_THREADS=1
-
 if [ -z "$BASEDIR" ]; then
   BASEDIR="/data/openpilot"
 fi
 
-if [ -z "$PASSIVE" ]; then
-  export PASSIVE="1"
-fi
+source "$BASEDIR/launch_env.sh"
 
-STAGING_ROOT="/data/safe_staging"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+function two_init {
 
-function launch {
   # Wifi scan
   wpa_cli IFNAME=wlan0 SCAN
 
+  # *** shield cores 2-3 ***
+
+  # android gets two cores
+  echo 0-1 > /dev/cpuset/background/cpus
+  echo 0-1 > /dev/cpuset/system-background/cpus
+  echo 0-1 > /dev/cpuset/foreground/cpus
+  echo 0-1 > /dev/cpuset/foreground/boost/cpus
+  echo 0-1 > /dev/cpuset/android/cpus
+
+  # openpilot gets all the cores
+  echo 0-3 > /dev/cpuset/app/cpus
+
+  # mask off 2-3 from RPS and XPS - Receive/Transmit Packet Steering
+  echo 3 | tee  /sys/class/net/*/queues/*/rps_cpus
+  echo 3 | tee  /sys/class/net/*/queues/*/xps_cpus
+
+  # *** set up governors ***
+
+  # +50mW offroad, +500mW onroad for 30% more RAM bandwidth
+  echo "performance" > /sys/class/devfreq/soc:qcom,cpubw/governor
+  echo 1056000 > /sys/class/devfreq/soc:qcom,m4m/max_freq
+  echo "performance" > /sys/class/devfreq/soc:qcom,m4m/governor
+
+  # unclear if these help, but they don't seem to hurt
+  echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu0/governor
+  echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu2/governor
+
+  # GPU
+  echo "performance" > /sys/class/devfreq/b00000.qcom,kgsl-3d0/governor
+
+  # /sys/class/devfreq/soc:qcom,mincpubw is the only one left at "powersave"
+  # it seems to gain nothing but a wasted 500mW
+
+  # *** set up IRQ affinities ***
+
+  # Collect RIL and other possibly long-running I/O interrupts onto CPU 1
+  echo 1 > /proc/irq/78/smp_affinity_list # qcom,smd-modem (LTE radio)
+  echo 1 > /proc/irq/33/smp_affinity_list # ufshcd (flash storage)
+  echo 1 > /proc/irq/35/smp_affinity_list # wifi (wlan_pci)
+  echo 1 > /proc/irq/6/smp_affinity_list  # MDSS
+
+  # USB traffic needs realtime handling on cpu 3
+  [ -d "/proc/irq/733" ] && echo 3 > /proc/irq/733/smp_affinity_list
+
+  # GPU and camera get cpu 2
+  CAM_IRQS="177 178 179 180 181 182 183 184 185 186 192"
+  for irq in $CAM_IRQS; do
+    echo 2 > /proc/irq/$irq/smp_affinity_list
+  done
+  echo 2 > /proc/irq/193/smp_affinity_list # GPU
+
+  # give GPU threads RT priority
+  for pid in $(pgrep "kgsl"); do
+    chrt -f -p 52 $pid
+  done
+
+  # the flippening!
+  LD_LIBRARY_PATH="" content insert --uri content://settings/system --bind name:s:user_rotation --bind value:i:1
+
+  # disable bluetooth
+  service call bluetooth_manager 8
+
+  # Check for NEOS update
+  if [ $(< /VERSION) != "$REQUIRED_NEOS_VERSION" ]; then
+    if [ -f "$DIR/scripts/continue.sh" ]; then
+      cp "$DIR/scripts/continue.sh" "/data/data/com.termux/files/continue.sh"
+    fi
+
+    if [ ! -f "$BASEDIR/prebuilt" ]; then
+      # Clean old build products, but preserve the scons cache
+      cd $DIR
+      scons --clean
+      git clean -xdf
+      git submodule foreach --recursive git clean -xdf
+    fi
+
+    "$DIR/installer/updater/updater" "file://$DIR/installer/updater/update.json"
+  fi
+}
+
+function tici_init {
+  sudo su -c 'echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu0/governor'
+  sudo su -c 'echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu4/governor'
+
+  # set success flag for current boot slot
+  sudo abctl --set_success
+
+  # Check if AGNOS update is required
+  if [ $(< /VERSION) != "$AGNOS_VERSION" ]; then
+    # Get number of slot to switch to
+    CUR_SLOT=$(abctl --boot_slot)
+    if [[ "$CUR_SLOT" == "_a" ]]; then
+      OTHER_SLOT="_b"
+      OTHER_SLOT_NUMBER="1"
+    else
+      OTHER_SLOT="_a"
+      OTHER_SLOT_NUMBER="0"
+    fi
+    echo "Cur slot $CUR_SLOT, target $OTHER_SLOT"
+
+    # Get expected hashes from manifest
+    MANIFEST="$DIR/selfdrive/hardware/tici/agnos.json"
+    SYSTEM_HASH_EXPECTED=$(jq -r ".[] | select(.name == \"system\") | .hash_raw" $MANIFEST)
+    SYSTEM_SIZE=$(jq -r ".[] | select(.name == \"system\") | .size" $MANIFEST)
+    BOOT_HASH_EXPECTED=$(jq -r ".[] | select(.name == \"boot\") | .hash_raw" $MANIFEST)
+    BOOT_SIZE=$(jq -r ".[] | select(.name == \"boot\") | .size" $MANIFEST)
+    echo "Expected hashes:"
+    echo "System: $SYSTEM_HASH_EXPECTED"
+    echo "Boot: $BOOT_HASH_EXPECTED"
+
+    # Read hashes from alternate partitions, should already be flashed by updated
+    SYSTEM_HASH=$(dd if="/dev/disk/by-partlabel/system$OTHER_SLOT" bs=1 skip="$SYSTEM_SIZE" count=64 2>/dev/null)
+    BOOT_HASH=$(dd if="/dev/disk/by-partlabel/boot$OTHER_SLOT" bs=1 skip="$BOOT_SIZE" count=64 2>/dev/null)
+    echo "Found hashes:"
+    echo "System: $SYSTEM_HASH"
+    echo "Boot: $BOOT_HASH"
+
+    if [[ "$SYSTEM_HASH" == "$SYSTEM_HASH_EXPECTED" && "$BOOT_HASH" == "$BOOT_HASH_EXPECTED" ]]; then
+      echo "Swapping active slot to $OTHER_SLOT_NUMBER"
+
+      # Clean hashes before swapping to prevent looping
+      dd if=/dev/zero of="/dev/disk/by-partlabel/system$OTHER_SLOT" bs=1 seek="$SYSTEM_SIZE" count=64
+      dd if=/dev/zero of="/dev/disk/by-partlabel/boot$OTHER_SLOT" bs=1 seek="$BOOT_SIZE" count=64
+      sync
+
+      abctl --set_active "$OTHER_SLOT_NUMBER"
+
+      sleep 1
+      sudo reboot
+    else
+      echo "Hash mismatch, downloading agnos"
+      if $DIR/selfdrive/hardware/tici/agnos.py $MANIFEST; then
+        echo "Download done, swapping active slot to $OTHER_SLOT_NUMBER"
+
+        # Clean hashes before swapping to prevent looping
+        dd if=/dev/zero of="/dev/disk/by-partlabel/system$OTHER_SLOT" bs=1 seek="$SYSTEM_SIZE" count=64
+        dd if=/dev/zero of="/dev/disk/by-partlabel/boot$OTHER_SLOT" bs=1 seek="$BOOT_SIZE" count=64
+        sync
+
+        abctl --set_active "$OTHER_SLOT_NUMBER"
+      fi
+
+      sleep 1
+      sudo reboot
+    fi
+  fi
+}
+
+function launch {
+  # Remove orphaned git lock if it exists on boot
+  [ -f "$DIR/.git/index.lock" ] && rm -f $DIR/.git/index.lock
+
   # Check to see if there's a valid overlay-based update available. Conditions
   # are as follows:
   #
@@ -49,6 +194,8 @@ function launch {
           git submodule foreach --recursive git reset --hard
 
           echo "Restarting launch script ${LAUNCHER_LOCATION}"
+          unset REQUIRED_NEOS_VERSION
+          unset AGNOS_VERSION
           exec "${LAUNCHER_LOCATION}"
         else
           echo "openpilot backup found, not updating"
@@ -58,70 +205,23 @@ function launch {
     fi
   fi
 
-  # Android and other system processes are not permitted to run on CPU 3
-  # NEOS installed app processes can run anywhere
-  echo 0-2 > /dev/cpuset/background/cpus
-  echo 0-2 > /dev/cpuset/system-background/cpus
-  [ -d "/dev/cpuset/foreground/boost/cpus" ] && echo 0-2 > /dev/cpuset/foreground/boost/cpus  # Not present in < NEOS 15
-  echo 0-2 > /dev/cpuset/foreground/cpus
-  echo 0-2 > /dev/cpuset/android/cpus
-  echo 0-3 > /dev/cpuset/app/cpus
-
-  # Collect RIL and other possibly long-running I/O interrupts onto CPU 1
-  echo 1 > /proc/irq/78/smp_affinity_list # qcom,smd-modem (LTE radio)
-  echo 1 > /proc/irq/33/smp_affinity_list # ufshcd (flash storage)
-  echo 1 > /proc/irq/35/smp_affinity_list # wifi (wlan_pci)
-  # USB traffic needs realtime handling on cpu 3
-  [ -d "/proc/irq/733" ] && echo 3 > /proc/irq/733/smp_affinity_list # USB for LeEco
-  [ -d "/proc/irq/736" ] && echo 3 > /proc/irq/736/smp_affinity_list # USB for OP3T
-
-  DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-  # Remove old NEOS update file
-  # TODO: move this code to the updater
-  if [ -d /data/neoupdate ]; then
-    rm -rf /data/neoupdate
-  fi
-
-  # Check for NEOS update
-  if [ $(< /VERSION) != "14" ]; then
-    if [ -f "$DIR/scripts/continue.sh" ]; then
-      cp "$DIR/scripts/continue.sh" "/data/data/com.termux/files/continue.sh"
-    fi
-
-    if [ ! -f "$BASEDIR/prebuilt" ]; then
-      echo "Clearing build products and resetting scons state prior to NEOS update"
-      cd $BASEDIR && scons --clean
-      rm -rf /tmp/scons_cache
-      rm -r $BASEDIR/.sconsign.dblite
-    fi
-    "$DIR/installer/updater/updater" "file://$DIR/installer/updater/update.json"
-  else
-    if [[ $(uname -v) == "#1 SMP PREEMPT Wed Jun 10 12:40:53 PDT 2020" ]]; then
-      "$DIR/installer/updater/updater" "file://$DIR/installer/updater/update_kernel.json"
-    fi
-  fi
-
-  # One-time fix for a subset of OP3T with gyro orientation offsets.
-  # Remove and regenerate qcom sensor registry. Only done on OP3T mainboards.
-  # Performed exactly once. The old registry is preserved just-in-case, and
-  # doubles as a flag denoting we've already done the reset.
-  # TODO: we should really grow per-platform detect and setup routines
-  if ! $(grep -q "letv" /proc/cmdline) && [ ! -f "/persist/comma/op3t-sns-reg-backup" ]; then
-    echo "Performing OP3T sensor registry reset"
-    mv /persist/sensors/sns.reg /persist/comma/op3t-sns-reg-backup &&
-      rm -f /persist/sensors/sensors_settings /persist/sensors/error_log /persist/sensors/gyro_sensitity_cal &&
-      echo "restart" > /sys/kernel/debug/msm_subsys/slpi &&
-      sleep 5  # Give Android sensor subsystem a moment to recover
-  fi
-
   # handle pythonpath
   ln -sfn $(pwd) /data/pythonpath
   export PYTHONPATH="$PWD"
 
+  # hardware specific init
+  if [ -f /EON ]; then
+    two_init
+  elif [ -f /TICI ]; then
+    tici_init
+  fi
+
+  # write tmux scrollback to a file
+  tmux capture-pane -pq -S-1000 > /tmp/launch_log
+
   # start manager
-  cd selfdrive
-  ./manager.py
+  cd selfdrive/manager
+  ./build.py && ./manager.py
 
   # if broken, keep on screen error
   while true; do sleep 1; done

launch_env.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/bash
+
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export NUMEXPR_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export VECLIB_MAXIMUM_THREADS=1
+
+if [ -z "$REQUIRED_NEOS_VERSION" ]; then
+  export REQUIRED_NEOS_VERSION="16.2"
+fi
+
+if [ -z "$AGNOS_VERSION" ]; then
+  export AGNOS_VERSION="0.11"
+fi
+
+if [ -z "$PASSIVE" ]; then
+  export PASSIVE="1"
+fi
+
+export STAGING_ROOT="/data/safe_staging"

opendbc/.gitignore

@@ -1,6 +1,7 @@
 .mypy_cache/
 *.pyc
 *.os
+*.o
 *.tmp
 *.dylib
 .*.swp
@@ -14,4 +15,3 @@ can/packer_pyx.cpp
 can/parser_pyx.cpp
 can/packer_pyx.html
 can/parser_pyx.html
-can/packer_impl.cpp

opendbc/acura_ilx_2016_can_generated.dbc

@@ -1,7 +1,7 @@
-CM_ "AUTOGENERATED FILE, DO NOT EDIT"
+CM_ "AUTOGENERATED FILE, DO NOT EDIT";
 
 
-CM_ "Imported file _comma.dbc starts here"
+CM_ "Imported file _comma.dbc starts here";
 BO_ 512 GAS_COMMAND: 6 EON
  SG_ GAS_COMMAND : 7|16@0+ (0.253984064,-83.3) [0|1] "" INTERCEPTOR
  SG_ GAS_COMMAND2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" INTERCEPTOR
@@ -19,7 +19,7 @@ BO_ 513 GAS_SENSOR: 6 INTERCEPTOR
 VAL_ 513 STATE 5 "FAULT_TIMEOUT" 4 "FAULT_STARTUP" 3 "FAULT_SCE" 2 "FAULT_SEND" 1 "FAULT_BAD_CHECKSUM" 0 "NO_FAULT" ;
 
 
-CM_ "Imported file _honda_2017.dbc starts here"
+CM_ "Imported file _honda_2017.dbc starts here";
 VERSION ""
 
 
@@ -251,7 +251,7 @@ VAL_ 780 CRUISE_SPEED 255 "no_speed" 252 "stopped" ;
 VAL_ 780 HUD_LEAD 3 "acc_off" 2 "solid_car" 1 "dashed_car" 0 "no_car" ;
 VAL_ 829 BEEP 3 "single_beep" 2 "triple_beep" 1 "repeated_beep" 0 "no_beep" ;
 
-CM_ "acura_ilx_2016_can.dbc starts here"
+CM_ "acura_ilx_2016_can.dbc starts here";
 
 
 

opendbc/acura_rdx_2018_can_generated.dbc

@@ -1,7 +1,7 @@
-CM_ "AUTOGENERATED FILE, DO NOT EDIT"
+CM_ "AUTOGENERATED FILE, DO NOT EDIT";
 
 
-CM_ "Imported file _comma.dbc starts here"
+CM_ "Imported file _comma.dbc starts here";
 BO_ 512 GAS_COMMAND: 6 EON
  SG_ GAS_COMMAND : 7|16@0+ (0.253984064,-83.3) [0|1] "" INTERCEPTOR
  SG_ GAS_COMMAND2 : 23|16@0+ (0.126992032,-83.3) [0|1] "" INTERCEPTOR
@@ -19,7 +19,7 @@ BO_ 513 GAS_SENSOR: 6 INTERCEPTOR
 VAL_ 513 STATE 5 "FAULT_TIMEOUT" 4 "FAULT_STARTUP" 3 "FAULT_SCE" 2 "FAULT_SEND" 1 "FAULT_BAD_CHECKSUM" 0 "NO_FAULT" ;
 
 
-CM_ "Imported file _honda_2017.dbc starts here"
+CM_ "Imported file _honda_2017.dbc starts here";
 VERSION ""
 
 
@@ -251,7 +251,7 @@ VAL_ 780 CRUISE_SPEED 255 "no_speed" 252 "stopped" ;
 VAL_ 780 HUD_LEAD 3 "acc_off" 2 "solid_car" 1 "dashed_car" 0 "no_car" ;
 VAL_ 829 BEEP 3 "single_beep" 2 "triple_beep" 1 "repeated_beep" 0 "no_beep" ;
 
-CM_ "acura_rdx_2018_can.dbc starts here"
+CM_ "acura_rdx_2018_can.dbc starts here";
 
 
 

opendbc/acura_rdx_2020_can_generated.dbc

@@ -0,0 +1,434 @@
+CM_ "AUTOGENERATED FILE, DO NOT EDIT";
+
+
+CM_ "Imported file _bosch_2020.dbc starts here";
+VERSION ""
+
+
+NS_ :
+	NS_DESC_
+	CM_
+	BA_DEF_
+	BA_
+	VAL_
+	CAT_DEF_
+	CAT_
+	FILTER
+	BA_DEF_DEF_
+	EV_DATA_
+	ENVVAR_DATA_
+	SGTYPE_
+	SGTYPE_VAL_
+	BA_DEF_SGTYPE_
+	BA_SGTYPE_
+	SIG_TYPE_REF_
+	VAL_TABLE_
+	SIG_GROUP_
+	SIG_VALTYPE_
+	SIGTYPE_VALTYPE_
+	BO_TX_BU_
+	BA_DEF_REL_
+	BA_REL_
+	BA_DEF_DEF_REL_
+	BU_SG_REL_
+	BU_EV_REL_
+	BU_BO_REL_
+	SG_MUL_VAL_
+
+BU_: EBCM EON CAM RADAR PCM EPS VSA SCM BDY XXX EPB
+
+BO_ 148 KINEMATICS: 8 XXX
+ SG_ LAT_ACCEL : 7|10@0+ (-0.035,17.92) [-20|20] "m/s2" EON
+ SG_ LONG_ACCEL : 25|10@0+ (-0.035,17.92) [-20|20] "m/s2" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+
+BO_ 228 STEERING_CONTROL: 5 EON
+ SG_ STEER_TORQUE_REQUEST : 23|1@0+ (1,0) [0|1] "" EPS
+ SG_ SET_ME_X00 : 22|7@0+ (1,0) [0|127] "" EPS
+ SG_ SET_ME_X00_2 : 31|8@0+ (1,0) [0|0] "" EPS
+ SG_ STEER_TORQUE : 7|16@0- (1,0) [-4096|4096] "" EPS
+ SG_ STEER_DOWN_TO_ZERO : 38|1@0+ (1,0) [0|1] "" EPS
+ SG_ COUNTER : 37|2@0+ (1,0) [0|3] "" EPS
+ SG_ CHECKSUM : 35|4@0+ (1,0) [0|15] "" EPS
+
+BO_ 229 BOSCH_SUPPLEMENTAL_1: 8 XXX
+ SG_ SET_ME_X04 : 0|8@1+ (1,0) [0|255] "" XXX
+ SG_ SET_ME_X00 : 8|8@1+ (1,0) [0|255] "" XXX
+ SG_ SET_ME_X80 : 16|8@1+ (1,0) [0|255] "" XXX
+ SG_ SET_ME_X10 : 24|8@1+ (1,0) [0|255] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 232 BRAKE_HOLD: 7 XXX
+ SG_ XMISSION_SPEED : 7|14@0- (1,0) [1|0] "" XXX
+ SG_ COMPUTER_BRAKE : 39|16@0+ (1,0) [0|0] "" XXX
+ SG_ COMPUTER_BRAKE_REQUEST : 29|1@0+ (1,0) [0|0] "" XXX
+ SG_ COUNTER : 53|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 51|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 342 STEERING_SENSORS: 6 EPS
+ SG_ STEER_ANGLE : 7|16@0- (-0.1,0) [-500|500] "deg" EON
+ SG_ STEER_ANGLE_RATE : 23|16@0- (1,0) [-3000|3000] "deg/s" EON
+ SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 43|4@0+ (1,0) [0|15] "" EON
+
+BO_ 344 ENGINE_DATA: 8 PCM
+ SG_ XMISSION_SPEED : 7|16@0+ (0.01,0) [0|250] "kph" EON
+ SG_ ENGINE_RPM : 23|16@0+ (1,0) [0|15000] "rpm" EON
+ SG_ XMISSION_SPEED2 : 39|16@0+ (0.01,0) [0|250] "kph" EON
+ SG_ ODOMETER : 55|8@0+ (10,0) [0|2550] "m" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 380 POWERTRAIN_DATA: 8 PCM
+ SG_ PEDAL_GAS : 7|8@0+ (1,0) [0|255] "" EON
+ SG_ ENGINE_RPM : 23|16@0+ (1,0) [0|15000] "rpm" EON
+ SG_ GAS_PRESSED : 39|1@0+ (1,0) [0|1] "" EON
+ SG_ ACC_STATUS : 38|1@0+ (1,0) [0|1] "" EON
+ SG_ BOH_17C : 37|5@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_SWITCH : 32|1@0+ (1,0) [0|1] "" EON
+ SG_ BOH2_17C : 47|10@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_PRESSED : 53|1@0+ (1,0) [0|1] "" EON
+ SG_ BOH3_17C : 52|5@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 399 STEER_STATUS: 7 EPS
+ SG_ STEER_TORQUE_SENSOR : 7|16@0- (-1,0) [-31000|31000] "tbd" EON
+ SG_ STEER_ANGLE_RATE : 23|16@0- (-0.1,0) [-31000|31000] "deg/s" EON
+ SG_ STEER_STATUS : 39|4@0+ (1,0) [0|15] "" EON
+ SG_ STEER_CONTROL_ACTIVE : 35|1@0+ (1,0) [0|1] "" EON
+ SG_ STEER_CONFIG_INDEX : 43|4@0+ (1,0) [0|15] "" EON
+ SG_ COUNTER : 53|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 51|4@0+ (1,0) [0|15] "" EON
+
+BO_ 420 VSA_STATUS: 8 VSA
+ SG_ ESP_DISABLED : 28|1@0+ (1,0) [0|1] "" EON
+ SG_ USER_BRAKE : 7|16@0+ (0.015625,-1.609375) [0|1000] "" EON
+ SG_ BRAKE_HOLD_ACTIVE : 46|1@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_HOLD_ENABLED : 45|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 427 STEER_MOTOR_TORQUE: 3 EPS
+ SG_ CONFIG_VALID : 7|1@0+ (1,0) [0|1] "" EON
+ SG_ MOTOR_TORQUE : 1|10@0+ (1,0) [0|256] "" EON
+ SG_ OUTPUT_DISABLED : 22|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 21|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 19|4@0+ (1,0) [0|15] "" EON
+
+BO_ 450 EPB_STATUS: 8 EPB
+ SG_ EPB_ACTIVE : 3|1@0+ (1,0) [0|1] "" EON
+ SG_ EPB_STATE : 29|2@0+ (1,0) [0|3] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 464 WHEEL_SPEEDS: 8 VSA
+ SG_ WHEEL_SPEED_FL : 7|15@0+ (0.01,0) [0|250] "kph" EON
+ SG_ WHEEL_SPEED_FR : 8|15@0+ (0.01,0) [0|250] "kph" EON
+ SG_ WHEEL_SPEED_RL : 25|15@0+ (0.01,0) [0|250] "kph" EON
+ SG_ WHEEL_SPEED_RR : 42|15@0+ (0.01,0) [0|250] "kph" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+
+BO_ 479 ACC_CONTROL: 8 EON
+ SG_ SET_TO_0 : 20|5@0+ (1,0) [0|1] "" XXX
+ SG_ CONTROL_ON : 23|3@0+ (1,0) [0|5] "" XXX
+ SG_ GAS_COMMAND : 7|16@0- (1,0) [0|0] "" XXX
+ SG_ ACCEL_COMMAND : 31|11@0- (0.01,0) [0|0] "m/s2" XXX
+ SG_ BRAKE_LIGHTS : 62|1@0+ (1,0) [0|1] "" XXX
+ SG_ BRAKE_REQUEST : 34|1@0+ (1,0) [0|1] "" XXX
+ SG_ STANDSTILL : 35|1@0+ (1,0) [0|1] "" XXX
+ SG_ STANDSTILL_RELEASE : 36|1@0+ (1,0) [0|1] "" XXX
+ SG_ AEB_STATUS : 33|1@0+ (1,0) [0|1] "" XXX
+ SG_ AEB_BRAKING : 47|1@0+ (1,0) [0|1] "" XXX
+ SG_ AEB_PREPARE : 43|1@0+ (1,0) [0|1] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 490 VEHICLE_DYNAMICS: 8 VSA
+ SG_ LAT_ACCEL : 7|16@0- (0.0015,0) [-20|20] "m/s2" EON
+ SG_ LONG_ACCEL : 23|16@0- (0.0015,0) [-20|20] "m/s2" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+
+BO_ 495 ACC_CONTROL_ON: 8 XXX
+ SG_ SET_TO_75 : 31|8@0+ (1,0) [0|255] "" XXX
+ SG_ SET_TO_30 : 39|8@0+ (1,0) [0|255] "" XXX
+ SG_ ZEROS_BOH : 23|8@0+ (1,0) [0|255] "" XXX
+ SG_ ZEROS_BOH2 : 47|16@0+ (1,0) [0|255] "" XXX
+ SG_ SET_TO_FF : 15|8@0+ (1,0) [0|255] "" XXX
+ SG_ SET_TO_3 : 6|7@0+ (1,0) [0|4095] "" XXX
+ SG_ CONTROL_ON : 7|1@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+
+BO_ 545 XXX_16: 6 SCM
+ SG_ ECON_ON : 23|1@0+ (1,0) [0|1] "" XXX
+ SG_ DRIVE_MODE : 37|2@0+ (1,0) [0|3] "" XXX
+ SG_ COUNTER : 45|2@0+ (1,0) [0|3] "" BDY
+ SG_ CHECKSUM : 43|4@0+ (1,0) [0|15] "" BDY
+
+BO_ 576 LEFT_LANE_LINE_1: 8 CAM
+ SG_ LINE_DISTANCE_VISIBLE : 39|9@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_PROBABILITY : 46|6@0+ (0.015625,0) [0|1] "" XXX
+ SG_ LINE_OFFSET : 23|12@0+ (0.004,-8.192) [0|1] "Meters" XXX
+ SG_ LINE_ANGLE : 7|12@0+ (0.0005,-1.024) [0|1] "" XXX
+ SG_ FRAME_INDEX : 8|4@1+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 577 LEFT_LANE_LINE_2: 8 CAM
+ SG_ LINE_FAR_EDGE_POSITION : 55|8@0+ (1,-128) [0|1] "" XXX
+ SG_ LINE_SOLID : 13|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_DASHED : 14|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_CURVATURE : 23|12@0+ (0.00001,-0.02048) [0|1] "" XXX
+ SG_ LINE_PARAMETER : 39|12@0+ (1,0) [0|1] "" XXX
+ SG_ FRAME_INDEX : 7|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 579 RIGHT_LANE_LINE_1: 8 CAM
+ SG_ LINE_DISTANCE_VISIBLE : 39|9@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_PROBABILITY : 46|6@0+ (0.015625,0) [0|1] "" XXX
+ SG_ LINE_OFFSET : 23|12@0+ (0.004,-8.192) [0|1] "Meters" XXX
+ SG_ LINE_ANGLE : 7|12@0+ (0.0005,-1.024) [0|1] "" XXX
+ SG_ FRAME_INDEX : 8|4@1+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 580 RIGHT_LANE_LINE_2: 8 CAM
+ SG_ LINE_FAR_EDGE_POSITION : 55|8@0+ (1,-128) [0|1] "" XXX
+ SG_ LINE_SOLID : 13|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_DASHED : 14|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_CURVATURE : 23|12@0+ (0.00001,-0.02048) [0|1] "" XXX
+ SG_ LINE_PARAMETER : 39|12@0+ (1,0) [0|1] "" XXX
+ SG_ FRAME_INDEX : 7|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 582 ADJACENT_LEFT_LANE_LINE_1: 8 CAM
+ SG_ LINE_DISTANCE_VISIBLE : 39|9@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_PROBABILITY : 46|6@0+ (0.015625,0) [0|1] "" XXX
+ SG_ LINE_OFFSET : 23|12@0+ (0.004,-8.192) [0|1] "Meters" XXX
+ SG_ LINE_ANGLE : 7|12@0+ (0.0005,-1.024) [0|1] "" XXX
+ SG_ FRAME_INDEX : 8|4@1+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 583 ADJACENT_LEFT_LANE_LINE_2: 8 CAM
+ SG_ LINE_FAR_EDGE_POSITION : 55|8@0+ (1,-128) [0|1] "" XXX
+ SG_ LINE_SOLID : 13|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_DASHED : 14|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_CURVATURE : 23|12@0+ (0.00001,-0.02048) [0|1] "" XXX
+ SG_ LINE_PARAMETER : 39|12@0+ (1,0) [0|1] "" XXX
+ SG_ FRAME_INDEX : 7|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 585 ADJACENT_RIGHT_LANE_LINE_1: 8 CAM
+ SG_ LINE_DISTANCE_VISIBLE : 39|9@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_PROBABILITY : 46|6@0+ (0.015625,0) [0|1] "" XXX
+ SG_ LINE_OFFSET : 23|12@0+ (0.004,-8.192) [0|1] "Meters" XXX
+ SG_ LINE_ANGLE : 7|12@0+ (0.0005,-1.024) [0|1] "" XXX
+ SG_ FRAME_INDEX : 8|4@1+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 586 ADJACENT_RIGHT_LANE_LINE_2: 8 CAM
+ SG_ LINE_FAR_EDGE_POSITION : 55|8@0+ (1,-128) [0|1] "" XXX
+ SG_ LINE_SOLID : 13|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_DASHED : 14|1@0+ (1,0) [0|1] "" XXX
+ SG_ LINE_CURVATURE : 23|12@0+ (0.00001,-0.02048) [0|1] "" XXX
+ SG_ LINE_PARAMETER : 39|12@0+ (1,0) [0|1] "" XXX
+ SG_ FRAME_INDEX : 7|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|1] "" XXX
+
+BO_ 597 ROUGH_WHEEL_SPEED: 8 VSA
+ SG_ WHEEL_SPEED_FL : 7|8@0+ (1,0) [0|255] "mph" EON
+ SG_ WHEEL_SPEED_FR : 15|8@0+ (1,0) [0|255] "mph" EON
+ SG_ WHEEL_SPEED_RL : 23|8@0+ (1,0) [0|255] "mph" EON
+ SG_ WHEEL_SPEED_RR : 31|8@0+ (1,0) [0|255] "mph" EON
+ SG_ SET_TO_X55 : 39|8@0+ (1,0) [0|255] "" XXX
+ SG_ SET_TO_X55_2 : 47|8@0+ (1,0) [0|255] "" EON
+ SG_ LONG_COUNTER : 55|8@0+ (1,0) [0|255] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+
+ BO_ 662 SCM_BUTTONS: 4 SCM
+  SG_ CRUISE_BUTTONS : 7|3@0+ (1,0) [0|7] "" EON
+  SG_ CRUISE_SETTING : 3|2@0+ (1,0) [0|3] "" EON
+  SG_ COUNTER : 29|2@0+ (1,0) [0|3] "" EON
+  SG_ CHECKSUM : 27|4@0+ (1,0) [0|15] "" EON
+
+BO_ 773 SEATBELT_STATUS: 7 BDY
+ SG_ SEATBELT_DRIVER_LAMP : 7|1@0+ (1,0) [0|1] "" EON
+ SG_ SEATBELT_PASS_UNLATCHED : 10|1@0+ (1,0) [0|1] "" EON
+ SG_ SEATBELT_PASS_LATCHED : 11|1@0+ (1,0) [0|1] "" EON
+ SG_ SEATBELT_DRIVER_UNLATCHED : 12|1@0+ (1,0) [0|1] "" EON
+ SG_ SEATBELT_DRIVER_LATCHED : 13|1@0+ (1,0) [0|1] "" EON
+ SG_ PASS_AIRBAG_OFF : 14|1@0+ (1,0) [0|1] "" EON
+ SG_ PASS_AIRBAG_ON : 15|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 53|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 51|4@0+ (1,0) [0|3] "" EON
+
+BO_ 777 CAR_SPEED: 8 PCM
+ SG_ ROUGH_CAR_SPEED : 23|8@0+ (1,0) [0|255] "mph" XXX
+ SG_ CAR_SPEED : 7|16@0+ (0.01,0) [0|65535] "kph" XXX
+ SG_ ROUGH_CAR_SPEED_3 : 39|16@0+ (0.01,0) [0|65535] "kph" XXX
+ SG_ ROUGH_CAR_SPEED_2 : 31|8@0+ (1,0) [0|255] "mph" XXX
+ SG_ LOCK_STATUS : 55|2@0+ (1,0) [0|255] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 780 ACC_HUD: 8 ADAS
+ SG_ CRUISE_SPEED : 31|8@0+ (1,0) [0|255] "kph" BDY
+ SG_ DTC_MODE : 39|1@0+ (1,0) [0|1] "" BDY
+ SG_ BOH : 38|1@0+ (1,0) [0|1] "" BDY
+ SG_ FCM_PROBLEM : 34|1@0+ (1,0) [0|1] "" BDY
+ SG_ RADAR_OBSTRUCTED : 33|1@0+ (1,0) [0|1] "" BDY
+ SG_ ENABLE_MINI_CAR : 32|1@0+ (1,0) [0|1] "" BDY
+ SG_ BOH_3 : 43|1@0+ (1,0) [0|3] "" BDY
+ SG_ BOH_4 : 42|1@0+ (1,0) [0|3] "" BDY
+ SG_ BOH_5 : 41|1@0+ (1,0) [0|3] "" BDY
+ SG_ CRUISE_CONTROL_LABEL : 40|1@0+ (1,0) [0|3] "" BDY
+ SG_ ZEROS_BOH : 7|24@0+ (0.002759506,0) [0|100] "m/s" BDY
+ SG_ FCM_OFF : 35|1@0+ (1,0) [0|1] "" BDY
+ SG_ SET_TO_1 : 36|1@0+ (1,0) [0|1] "" XXX
+ SG_ HUD_DISTANCE : 47|2@0+ (1,0) [0|3] "" BDY
+ SG_ HUD_LEAD : 45|2@0+ (1,0) [0|3] "" BDY
+ SG_ ACC_PROBLEM : 37|1@0+ (1,0) [0|1] "" BDY
+ SG_ ACC_ON : 52|1@0+ (1,0) [0|1] "" XXX
+ SG_ BOH_6 : 51|4@0+ (1,0) [0|15] "" XXX
+ SG_ SET_TO_X1 : 55|1@0+ (1,0) [0|1] "" XXX
+ SG_ IMPERIAL_UNIT : 54|1@0+ (1,0) [0|1] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 804 CRUISE: 8 PCM
+ SG_ TRIP_FUEL_CONSUMED : 23|16@0+ (1,0) [0|255] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 806 SCM_FEEDBACK: 8 SCM
+ SG_ DRIVERS_DOOR_OPEN : 17|1@0+ (1,0) [0|1] "" XXX
+ SG_ MAIN_ON : 28|1@0+ (1,0) [0|1] "" EON
+ SG_ RIGHT_BLINKER : 27|1@0+ (1,0) [0|1] "" EON
+ SG_ LEFT_BLINKER : 26|1@0+ (1,0) [0|1] "" EON
+ SG_ CMBS_STATES : 22|2@0+ (1,0) [0|3] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 829 LKAS_HUD: 5 ADAS
+ SG_ CAM_TEMP_HIGH : 7|1@0+ (1,0) [0|255] "" BDY
+ SG_ SET_ME_X41 : 6|7@0+ (1,0) [0|127] "" BDY
+ SG_ BOH : 6|7@0+ (1,0) [0|127] "" BDY
+ SG_ DASHED_LANES : 14|1@0+ (1,0) [0|1] "" BDY
+ SG_ DTC : 13|1@0+ (1,0) [0|1] "" BDY
+ SG_ LKAS_PROBLEM : 12|1@0+ (1,0) [0|1] "" BDY
+ SG_ LKAS_OFF : 11|1@0+ (1,0) [0|1] "" BDY
+ SG_ SOLID_LANES : 10|1@0+ (1,0) [0|1] "" BDY
+ SG_ LDW_RIGHT : 9|1@0+ (1,0) [0|1] "" BDY
+ SG_ STEERING_REQUIRED : 8|1@0+ (1,0) [0|1] "" BDY
+ SG_ BOH : 23|2@0+ (1,0) [0|4] "" BDY
+ SG_ LDW_PROBLEM : 21|1@0+ (1,0) [0|1] "" BDY
+ SG_ BEEP : 17|2@0+ (1,0) [0|1] "" BDY
+ SG_ LDW_ON : 28|1@0+ (1,0) [0|1] "" BDY
+ SG_ LDW_OFF : 27|1@0+ (1,0) [0|1] "" BDY
+ SG_ CLEAN_WINDSHIELD : 26|1@0+ (1,0) [0|1] "" BDY
+ SG_ SET_ME_X48 : 31|8@0+ (1,0) [0|255] "" BDY
+ SG_ COUNTER : 37|2@0+ (1,0) [0|3] "" BDY
+ SG_ CHECKSUM : 35|4@0+ (1,0) [0|15] "" BDY
+
+BO_ 862 CAMERA_MESSAGES: 8 CAM
+ SG_ ZEROS_BOH : 7|50@0+ (1,0) [0|127] "" BDY
+ SG_ AUTO_HIGHBEAMS_ACTIVE : 53|1@0+ (1,0) [0|1] "" XXX
+ SG_ HIGHBEAMS_ON : 52|1@0+ (1,0) [0|1] "" XXX
+ SG_ ZEROS_BOH_2 : 51|4@0+ (1,0) [0|15] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 884 STALK_STATUS: 8 XXX
+ SG_ AUTO_HEADLIGHTS : 46|1@0+ (1,0) [0|1] "" EON
+ SG_ HIGH_BEAM_HOLD : 47|1@0+ (1,0) [0|1] "" EON
+ SG_ HIGH_BEAM_FLASH : 45|1@0+ (1,0) [0|1] "" EON
+ SG_ HEADLIGHTS_ON : 54|1@0+ (1,0) [0|1] "" EON
+ SG_ WIPER_SWITCH : 53|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+
+BO_ 891 STALK_STATUS_2: 8 XXX
+ SG_ WIPERS : 17|2@0+ (1,0) [0|3] "" EON
+ SG_ LOW_BEAMS : 35|1@0+ (1,0) [0|1] "" XXX
+ SG_ HIGH_BEAMS : 34|1@0+ (1,0) [0|1] "" XXX
+ SG_ PARK_LIGHTS : 36|1@0+ (1,0) [0|1] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+
+CM_ SG_ 479 AEB_STATUS "set for the duration of AEB event";
+CM_ SG_ 479 AEB_BRAKING "set when braking is commanded during AEB event";
+CM_ SG_ 479 AEB_PREPARE "set 1s before AEB";
+CM_ SG_ 576 LINE_DISTANCE_VISIBLE "Length of line visible, undecoded";
+CM_ SG_ 577 LINE_FAR_EDGE_POSITION "Appears to be a measure of line thickness, indicates location of the portion of the line furthest from the car, undecoded";
+CM_ SG_ 577 LINE_PARAMETER "Unclear if this is low quality line curvature rate or if this is something else, but it is correlated with line curvature, undecoded";
+CM_ SG_ 577 LINE_DASHED "1 = line is dashed";
+CM_ SG_ 577 LINE_SOLID "1 = line is solid";
+VAL_ 399 STEER_STATUS 6 "tmp_fault" 5 "fault_1" 4 "no_torque_alert_2" 3 "low_speed_lockout" 2 "no_torque_alert_1" 0 "normal" ;
+
+CM_ "acura_rdx_2020_can.dbc starts here";
+
+
+BO_ 304 GAS_PEDAL_2: 8 PCM
+ SG_ ENGINE_TORQUE_ESTIMATE : 7|16@0- (1,0) [-1000|1000] "Nm" EON
+ SG_ ENGINE_TORQUE_REQUEST : 23|16@0- (1,0) [-1000|1000] "Nm" EON
+ SG_ CAR_GAS : 39|8@0+ (1,0) [0|255] "" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" EON
+
+BO_ 419 GEARBOX: 8 PCM
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+ SG_ GEAR_SHIFTER : 29|6@0+ (1,0) [0|63] "" EON
+ SG_ GEAR : 7|8@0+ (1,0) [0|255] "" EON
+
+BO_ 432 STANDSTILL: 7 VSA
+ SG_ BRAKE_ERROR_1 : 11|1@0+ (1,0) [0|1] "" EON
+ SG_ BRAKE_ERROR_2 : 9|1@0+ (1,0) [0|1] "" EON
+ SG_ COUNTER : 53|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 51|4@0+ (1,0) [0|15] "" EON
+
+BO_ 446 BRAKE_MODULE: 3 VSA
+ SG_ BRAKE_PRESSED : 4|1@0+ (1,0) [0|1] "" XXX
+ SG_ COUNTER : 21|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 19|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 927 RADAR_HUD: 8 RADAR
+ SG_ ZEROS_BOH : 7|10@0+ (1,0) [0|127] "" BDY
+ SG_ CMBS_OFF : 12|1@0+ (1,0) [0|1] "" BDY
+ SG_ RESUME_INSTRUCTION : 21|1@0+ (1,0) [0|1] "" XXX
+ SG_ SET_TO_1 : 13|1@0+ (1,0) [0|1] "" BDY
+ SG_ ZEROS_BOH2 : 11|4@0+ (1,0) [0|1] "" XXX
+ SG_ APPLY_BRAKES_FOR_CANC : 23|1@0+ (1,0) [0|1] "" XXX
+ SG_ ACC_ALERTS : 20|5@0+ (1,0) [0|1] "" BDY
+ SG_ SET_TO_0 : 22|1@0+ (1,0) [0|1] "" XXX
+ SG_ LEAD_DISTANCE : 39|8@0+ (1,0) [0|255] "" XXX
+ SG_ BOH : 40|1@0+ (1,0) [0|1] "" XXX
+ SG_ BOH_2 : 30|1@0+ (1,0) [0|1] "" XXX
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" XXX
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|15] "" XXX
+
+BO_ 1302 ODOMETER: 8 XXX
+ SG_ ODOMETER : 7|24@0+ (1,0) [0|16777215] "km" EON
+ SG_ COUNTER : 61|2@0+ (1,0) [0|3] "" EON
+ SG_ CHECKSUM : 59|4@0+ (1,0) [0|3] "" EON
+
+VAL_ 419 GEAR_SHIFTER  32 "D" 8 "R" 4 "P" ;
+VAL_ 545 ECON_ON_2 0 "off" 3 "on" ;
+VAL_ 662 CRUISE_BUTTONS 7 "tbd" 6 "tbd" 5 "tbd" 4 "accel_res" 3 "decel_set" 2 "cancel" 1 "main" 0 "none" ;
+VAL_ 662 CRUISE_SETTING 3 "distance_adj" 2 "tbd" 1 "lkas_button" 0 "none" ;
+VAL_ 806 CMBS_BUTTON 3 "pressed" 0 "released" ;
+VAL_ 891 WIPERS 4 "High" 2 "Low" 0 "Off" ;
+VAL_ 829 BEEP 3 "single_beep" 2 "triple_beep" 1 "repeated_beep" 0 "no_beep" ;
+
+CM_ "CHFFR_METRIC 330 STEER_ANGLE STEER_ANGLE 0.36 180; CHFFR_METRIC 380 ENGINE_RPM ENGINE_RPM 1 0; CHFFR_METRIC 804 ENGINE_TEMPERATURE ENGINE_TEMPERATURE 1 0";

opendbc/can/SConscript

@@ -1,4 +1,4 @@
-Import('env', 'cereal')
+Import('env', 'envCython', 'cereal')
 
 import os
 from opendbc.can.process_dbc import process
@@ -13,11 +13,13 @@ for x in sorted(os.listdir('../')):
     dbc = env.Command(out_fn, in_fn, compile_dbc)
     dbcs.append(dbc)
 
-
 libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
 
 # Build packer and parser
+lenv = envCython.Clone()
+lenv["LINKFLAGS"] += [libdbc[0].get_labspath()]
+parser = lenv.Program('parser_pyx.so', 'parser_pyx.pyx')
+packer = lenv.Program('packer_pyx.so', 'packer_pyx.pyx')
 
-env.Command(['packer_pyx.so', 'parser_pyx.so'],
-  [libdbc, cereal, 'common_pyx_setup.py', 'packer_pyx.pyx', 'parser_pyx.pyx', 'common.pxd'],
-  "cd opendbc/can && python3 common_pyx_setup.py build_ext --inplace")
+lenv.Depends(parser, libdbc)
+lenv.Depends(packer, libdbc)

opendbc/can/common.cc

@@ -97,23 +97,22 @@ unsigned int volkswagen_crc(unsigned int address, uint64_t d, int l) {
   // a magic variable padding byte tacked onto the end of the payload.
   // https://www.autosar.org/fileadmin/user_upload/standards/classic/4-3/AUTOSAR_SWS_CRCLibrary.pdf
 
-  uint8_t *dat = (uint8_t *)&d;
   uint8_t crc = 0xFF; // Standard init value for CRC8 8H2F/AUTOSAR
 
   // CRC the payload first, skipping over the first byte where the CRC lives.
   for (int i = 1; i < l; i++) {
-    crc ^= dat[i];
+    crc ^= (d >> (i*8)) & 0xFF;
     crc = crc8_lut_8h2f[crc];
   }
 
   // Look up and apply the magic final CRC padding byte, which permutes by CAN
   // address, and additionally (for SOME addresses) by the message counter.
-  uint8_t counter = dat[1] & 0x0F;
+  uint8_t counter = ((d >> 8) & 0xFF) & 0x0F;
   switch(address) {
     case 0x86:  // LWI_01 Steering Angle
       crc ^= (uint8_t[]){0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86,0x86}[counter];
       break;
-    case 0x9F:  // EPS_01 Electric Power Steering
+    case 0x9F:  // LH_EPS_03 Electric Power Steering
       crc ^= (uint8_t[]){0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5,0xF5}[counter];
       break;
     case 0xAD:  // Getriebe_11 Automatic Gearbox
@@ -126,7 +125,7 @@ unsigned int volkswagen_crc(unsigned int address, uint64_t d, int l) {
       crc ^= (uint8_t[]){0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07,0x07}[counter];
       break;
     case 0x117: // ACC_10 Automatic Cruise Control
-      crc ^= (uint8_t[]){0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC,0xAC}[counter];
+      crc ^= (uint8_t[]){0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16}[counter];
       break;
     case 0x120: // TSK_06 Drivetrain Coordinator
       crc ^= (uint8_t[]){0xC4,0xE2,0x4F,0xE4,0xF8,0x2F,0x56,0x81,0x9F,0xE5,0x83,0x44,0x05,0x3F,0x97,0xDF}[counter];
@@ -152,6 +151,9 @@ unsigned int volkswagen_crc(unsigned int address, uint64_t d, int l) {
     case 0x30F: // SWA_01 Lane Change Assist (SpurWechselAssistent)
       crc ^= (uint8_t[]){0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C,0x0C}[counter];
       break;
+    case 0x324: // ACC_04 Automatic Cruise Control
+      crc ^= (uint8_t[]){0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27,0x27}[counter];
+      break;
     case 0x3C0: // Klemmen_Status_01 ignition and starting status
       crc ^= (uint8_t[]){0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3,0xC3}[counter];
       break;
@@ -176,11 +178,9 @@ unsigned int pedal_checksum(uint64_t d, int l) {
   d >>= ((8-l)*8); // remove padding
   d >>= 8; // remove checksum
 
-  uint8_t *dat = (uint8_t *)&d;
-
   int i, j;
   for (i = 0; i < l - 1; i++) {
-    crc ^= dat[i];
+    crc ^= (d >> (i*8)) & 0xFF;
     for (j = 0; j < 8; j++) {
       if ((crc & 0x80) != 0) {
         crc = (uint8_t)((crc << 1) ^ poly);

opendbc/can/common.h

@@ -5,8 +5,12 @@
 #include <unordered_map>
 
 #include "common_dbc.h"
+#include <capnp/dynamic.h>
 #include <capnp/serialize.h>
+
+#ifndef DYNAMIC_CAPNP
 #include "cereal/gen/cpp/log.capnp.h"
+#endif
 
 #define MAX_BAD_COUNTER 5
 
@@ -36,6 +40,9 @@ public:
   uint8_t counter;
   uint8_t counter_fail;
 
+  bool ignore_checksum = false;
+  bool ignore_counter = false;
+
   bool parse(uint64_t sec, uint16_t ts_, uint8_t * dat);
   bool update_counter_generic(int64_t v, int cnt_size);
 };
@@ -43,6 +50,7 @@ public:
 class CANParser {
 private:
   const int bus;
+  kj::Array<capnp::word> aligned_buf;
 
   const DBC *dbc = NULL;
   std::unordered_map<uint32_t, MessageState> message_states;
@@ -54,9 +62,13 @@ public:
   CANParser(int abus, const std::string& dbc_name,
             const std::vector<MessageParseOptions> &options,
             const std::vector<SignalParseOptions> &sigoptions);
+  CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter);
+  #ifndef DYNAMIC_CAPNP
+  void update_string(const std::string &data, bool sendcan);
   void UpdateCans(uint64_t sec, const capnp::List<cereal::CanData>::Reader& cans);
+  #endif
+  void UpdateCans(uint64_t sec, const capnp::DynamicStruct::Reader& cans);
   void UpdateValid(uint64_t sec);
-  void update_string(std::string data, bool sendcan);
   std::vector<SignalValue> query_latest();
 };
 
@@ -68,5 +80,6 @@ private:
 
 public:
   CANPacker(const std::string& dbc_name);
-  uint64_t pack(uint32_t address, const std::vector<SignalPackValue> &signals, int counter);
+  uint64_t pack(uint32_t address, const std::vector<SignalPackValue> &values, int counter);
+  Msg* lookup_message(uint32_t address);
 };

opendbc/can/common_dbc.h

@@ -3,6 +3,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <string>
+#include <vector>
 
 #define ARRAYSIZE(x) (sizeof(x)/sizeof(x[0]))
 
@@ -74,6 +75,7 @@ struct DBC {
   size_t num_vals;
 };
 
+std::vector<const DBC*>& get_dbcs();
 const DBC* dbc_lookup(const std::string& dbc_name);
 
 void dbc_register(const DBC* dbc);

opendbc/can/common_pyx_setup.py

@@ -1,94 +0,0 @@
-import os
-import subprocess
-import sysconfig
-import platform
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-
-from Cython.Build import cythonize
-from Cython.Distutils import build_ext
-
-
-ANNOTATE = os.getenv('ANNOTATE') is not None
-BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
-
-
-def get_ext_filename_without_platform_suffix(filename):
-  name, ext = os.path.splitext(filename)
-  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
-
-  if ext_suffix == ext:
-    return filename
-
-  ext_suffix = ext_suffix.replace(ext, '')
-  idx = name.find(ext_suffix)
-
-  if idx == -1:
-    return filename
-  else:
-    return name[:idx] + ext
-
-
-class BuildExtWithoutPlatformSuffix(build_ext):
-  def get_ext_filename(self, ext_name):
-    filename = super().get_ext_filename(ext_name)
-    return get_ext_filename_without_platform_suffix(filename)
-
-
-extra_compile_args = ["-std=c++14"]
-ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()  # pylint: disable=unexpected-keyword-arg
-if ARCH == "aarch64":
-  extra_compile_args += ["-Wno-deprecated-register"]
-
-if platform.system() == "Darwin":
-  libdbc = "libdbc.dylib"
-else:
-  libdbc = "libdbc.so"
-
-extra_link_args = [os.path.join(BASEDIR, 'opendbc', 'can', libdbc)]
-include_dirs = [
-  BASEDIR,
-  os.path.join(BASEDIR, 'phonelibs'),
-]
-
-# Build CAN Parser
-
-setup(name='CAN parser',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "parser_pyx",
-          language="c++",
-          sources=['parser_pyx.pyx'],
-          extra_compile_args=extra_compile_args,
-          include_dirs=include_dirs,
-          extra_link_args=extra_link_args,
-        ),
-        annotate=ANNOTATE
-      ),
-      nthreads=4,
-)
-
-if platform.system() == "Darwin":
-  os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib parser_pyx.so")
-
-
-# Build CAN Packer
-
-setup(name='CAN packer',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "packer_pyx",
-          language="c++",
-          sources=['packer_pyx.pyx'],
-          extra_compile_args=extra_compile_args,
-          include_dirs=include_dirs,
-          extra_link_args=extra_link_args,
-        ),
-        annotate=ANNOTATE
-      ),
-      nthreads=4,
-)
-
-if platform.system() == "Darwin":
-  os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib packer_pyx.so")

opendbc/can/dbc.cc

@@ -2,15 +2,11 @@
 
 #include "common_dbc.h"
 
-namespace {
-
 std::vector<const DBC*>& get_dbcs() {
   static std::vector<const DBC*> vec;
   return vec;
 }
 
-}
-
 const DBC* dbc_lookup(const std::string& dbc_name) {
   for (const auto& dbci : get_dbcs()) {
     if (dbc_name == dbci->name) {

opendbc/can/packer.cc

@@ -20,7 +20,7 @@ uint64_t ReverseBytes(uint64_t x) {
           ((x & 0x00000000000000ffull) << 56);
 }
 
-uint64_t set_value(uint64_t ret, Signal sig, int64_t ival){
+static uint64_t set_value(uint64_t ret, const Signal& sig, int64_t ival) {
   int shift = sig.is_little_endian? sig.b1 : sig.bo;
   uint64_t mask = ((1ULL << sig.b2)-1) << shift;
   uint64_t dat = (ival & ((1ULL << sig.b2)-1)) << shift;
@@ -59,7 +59,7 @@ uint64_t CANPacker::pack(uint32_t address, const std::vector<SignalPackValue> &s
       WARN("undefined signal %s - %d\n", name.c_str(), address);
       continue;
     }
-    auto sig = sig_it->second;
+    const auto& sig = sig_it->second;
 
     int64_t ival = (int64_t)(round((value - sig.offset) / sig.factor));
     if (ival < 0) {
@@ -75,7 +75,7 @@ uint64_t CANPacker::pack(uint32_t address, const std::vector<SignalPackValue> &s
       WARN("COUNTER not defined\n");
       return ret;
     }
-    auto sig = sig_it->second;
+    const auto& sig = sig_it->second;
 
     if ((sig.type != SignalType::HONDA_COUNTER) && (sig.type != SignalType::VOLKSWAGEN_COUNTER)) {
       WARN("COUNTER signal type not valid\n");
@@ -86,7 +86,7 @@ uint64_t CANPacker::pack(uint32_t address, const std::vector<SignalPackValue> &s
 
   auto sig_it_checksum = signal_lookup.find(std::make_pair(address, "CHECKSUM"));
   if (sig_it_checksum != signal_lookup.end()) {
-    auto sig = sig_it_checksum->second;
+    const auto& sig = sig_it_checksum->second;
     if (sig.type == SignalType::HONDA_CHECKSUM) {
       unsigned int chksm = honda_checksum(address, ret, message_lookup[address].size);
       ret = set_value(ret, sig, chksm);
@@ -112,3 +112,7 @@ uint64_t CANPacker::pack(uint32_t address, const std::vector<SignalPackValue> &s
 
   return ret;
 }
+
+Msg* CANPacker::lookup_message(uint32_t address) {
+  return &message_lookup[address];
+}

opendbc/can/packer_pyx.pyx

@@ -8,8 +8,8 @@ from libcpp.string cimport string
 from libcpp cimport bool
 from posix.dlfcn cimport dlopen, dlsym, RTLD_LAZY
 
-from common cimport CANPacker as cpp_CANPacker
-from common cimport dbc_lookup, SignalPackValue, DBC
+from .common cimport CANPacker as cpp_CANPacker
+from .common cimport dbc_lookup, SignalPackValue, DBC
 
 
 cdef class CANPacker:
@@ -20,9 +20,11 @@ cdef class CANPacker:
     map[int, int] address_to_size
 
   def __init__(self, dbc_name):
-    self.packer = new cpp_CANPacker(dbc_name)
     self.dbc = dbc_lookup(dbc_name)
+    if not self.dbc:
+      raise RuntimeError(f"Can't lookup {dbc_name}")
 
+    self.packer = new cpp_CANPacker(dbc_name)
     num_msgs = self.dbc[0].num_msgs
     for i in range(num_msgs):
       msg = self.dbc[0].msgs[i]
@@ -37,7 +39,7 @@ cdef class CANPacker:
 
     for name, value in values.iteritems():
       n = name.encode('utf8')
-      names.append(n) # TODO: find better way to keep reference to temp string arround
+      names.append(n) # TODO: find better way to keep reference to temp string around
 
       spv.name = n
       spv.value = value

opendbc/can/parser.cc

@@ -13,7 +13,6 @@
 // #define DEBUG printf
 #define INFO printf
 
-
 bool MessageState::parse(uint64_t sec, uint16_t ts_, uint8_t * dat) {
   uint64_t dat_le = read_u64_le(dat);
   uint64_t dat_be = read_u64_be(dat);
@@ -34,47 +33,52 @@ bool MessageState::parse(uint64_t sec, uint16_t ts_, uint8_t * dat) {
 
     DEBUG("parse 0x%X %s -> %lld\n", address, sig.name, tmp);
 
-    if (sig.type == SignalType::HONDA_CHECKSUM) {
-      if (honda_checksum(address, dat_be, size) != tmp) {
-        INFO("0x%X CHECKSUM FAIL\n", address);
-        return false;
+    if (!ignore_checksum) {
+      if (sig.type == SignalType::HONDA_CHECKSUM) {
+        if (honda_checksum(address, dat_be, size) != tmp) {
+          INFO("0x%X CHECKSUM FAIL\n", address);
+          return false;
+        }
+      } else if (sig.type == SignalType::TOYOTA_CHECKSUM) {
+        if (toyota_checksum(address, dat_be, size) != tmp) {
+          INFO("0x%X CHECKSUM FAIL\n", address);
+          return false;
+        }
+      } else if (sig.type == SignalType::VOLKSWAGEN_CHECKSUM) {
+        if (volkswagen_crc(address, dat_le, size) != tmp) {
+          INFO("0x%X CRC FAIL\n", address);
+          return false;
+        }
+      } else if (sig.type == SignalType::SUBARU_CHECKSUM) {
+        if (subaru_checksum(address, dat_be, size) != tmp) {
+          INFO("0x%X CHECKSUM FAIL\n", address);
+          return false;
+        }
+      } else if (sig.type == SignalType::CHRYSLER_CHECKSUM) {
+        if (chrysler_checksum(address, dat_le, size) != tmp) {
+          INFO("0x%X CHECKSUM FAIL\n", address);
+          return false;
+        }
+      } else if (sig.type == SignalType::PEDAL_CHECKSUM) {
+        if (pedal_checksum(dat_be, size) != tmp) {
+          INFO("0x%X PEDAL CHECKSUM FAIL\n", address);
+          return false;
+        }
       }
-    } else if (sig.type == SignalType::HONDA_COUNTER) {
-      if (!update_counter_generic(tmp, sig.b2)) {
-        return false;
-      }
-    } else if (sig.type == SignalType::TOYOTA_CHECKSUM) {
-      if (toyota_checksum(address, dat_be, size) != tmp) {
-        INFO("0x%X CHECKSUM FAIL\n", address);
-        return false;
-      }
-    } else if (sig.type == SignalType::VOLKSWAGEN_CHECKSUM) {
-      if (volkswagen_crc(address, dat_le, size) != tmp) {
-        INFO("0x%X CRC FAIL\n", address);
-        return false;
-      }
-    } else if (sig.type == SignalType::VOLKSWAGEN_COUNTER) {
+    }
+    if (!ignore_counter) {
+      if (sig.type == SignalType::HONDA_COUNTER) {
         if (!update_counter_generic(tmp, sig.b2)) {
-        return false;
-      }
-    } else if (sig.type == SignalType::SUBARU_CHECKSUM) {
-      if (subaru_checksum(address, dat_be, size) != tmp) {
-        INFO("0x%X CHECKSUM FAIL\n", address);
-        return false;
-      }
-    } else if (sig.type == SignalType::CHRYSLER_CHECKSUM) {
-      if (chrysler_checksum(address, dat_le, size) != tmp) {
-        INFO("0x%X CHECKSUM FAIL\n", address);
-        return false;
-      }
-    } else if (sig.type == SignalType::PEDAL_CHECKSUM) {
-      if (pedal_checksum(dat_be, size) != tmp) {
-        INFO("0x%X PEDAL CHECKSUM FAIL\n", address);
-        return false;
-      }
-    } else if (sig.type == SignalType::PEDAL_COUNTER) {
-      if (!update_counter_generic(tmp, sig.b2)) {
-        return false;
+          return false;
+        }
+      } else if (sig.type == SignalType::VOLKSWAGEN_COUNTER) {
+          if (!update_counter_generic(tmp, sig.b2)) {
+          return false;
+        }
+      } else if (sig.type == SignalType::PEDAL_COUNTER) {
+        if (!update_counter_generic(tmp, sig.b2)) {
+          return false;
+        }
       }
     }
 
@@ -108,26 +112,24 @@ bool MessageState::update_counter_generic(int64_t v, int cnt_size) {
 CANParser::CANParser(int abus, const std::string& dbc_name,
           const std::vector<MessageParseOptions> &options,
           const std::vector<SignalParseOptions> &sigoptions)
-  : bus(abus) {
+  : bus(abus), aligned_buf(kj::heapArray<capnp::word>(1024)) {
 
   dbc = dbc_lookup(dbc_name);
   assert(dbc);
   init_crc_lookup_tables();
 
   for (const auto& op : options) {
-    MessageState state = {
-      .address = op.address,
-      // .check_frequency = op.check_frequency,
-    };
+    MessageState &state = message_states[op.address];
+    state.address = op.address;
+    // state.check_frequency = op.check_frequency,
 
     // msg is not valid if a message isn't received for 10 consecutive steps
     if (op.check_frequency > 0) {
       state.check_threshold = (1000000000ULL / op.check_frequency) * 10;
     }
 
-
     const Msg* msg = NULL;
-    for (int i=0; i<dbc->num_msgs; i++) {
+    for (int i = 0; i < dbc->num_msgs; i++) {
       if (dbc->msgs[i].address == op.address) {
         msg = &dbc->msgs[i];
         break;
@@ -141,7 +143,7 @@ CANParser::CANParser(int abus, const std::string& dbc_name,
     state.size = msg->size;
 
     // track checksums and counters for this message
-    for (int i=0; i<msg->num_sigs; i++) {
+    for (int i = 0; i < msg->num_sigs; i++) {
       const Signal *sig = &msg->sigs[i];
       if (sig->type != SignalType::DEFAULT) {
         state.parse_sigs.push_back(*sig);
@@ -153,7 +155,7 @@ CANParser::CANParser(int abus, const std::string& dbc_name,
     for (const auto& sigop : sigoptions) {
       if (sigop.address != op.address) continue;
 
-      for (int i=0; i<msg->num_sigs; i++) {
+      for (int i = 0; i < msg->num_sigs; i++) {
         const Signal *sig = &msg->sigs[i];
         if (strcmp(sig->name, sigop.name) == 0
             && sig->type == SignalType::DEFAULT) {
@@ -162,37 +164,105 @@ CANParser::CANParser(int abus, const std::string& dbc_name,
           break;
         }
       }
+    }
+  }
+}
 
+CANParser::CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter)
+  : bus(abus) {
+  // Add all messages and signals
+
+  dbc = dbc_lookup(dbc_name);
+  assert(dbc);
+  init_crc_lookup_tables();
+
+  for (int i = 0; i < dbc->num_msgs; i++) {
+    const Msg* msg = &dbc->msgs[i];
+    MessageState state = {
+      .address = msg->address,
+      .size = msg->size,
+      .ignore_checksum = ignore_checksum,
+      .ignore_counter = ignore_counter,
+    };
+
+    for (int j = 0; j < msg->num_sigs; j++) {
+      const Signal *sig = &msg->sigs[j];
+      state.parse_sigs.push_back(*sig);
+      state.vals.push_back(0);
     }
 
     message_states[state.address] = state;
   }
 }
 
+#ifndef DYNAMIC_CAPNP
+void CANParser::update_string(const std::string &data, bool sendcan) {
+  // format for board, make copy due to alignment issues.
+  const size_t buf_size = (data.length() / sizeof(capnp::word)) + 1;
+  if (aligned_buf.size() < buf_size) {
+    aligned_buf = kj::heapArray<capnp::word>(buf_size);
+  }
+  memcpy(aligned_buf.begin(), data.data(), data.length());
+
+  // extract the messages
+  capnp::FlatArrayMessageReader cmsg(aligned_buf.slice(0, buf_size));
+  cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
+
+  last_sec = event.getLogMonoTime();
+
+  auto cans = sendcan? event.getSendcan() : event.getCan();
+  UpdateCans(last_sec, cans);
+
+  UpdateValid(last_sec);
+}
+
 void CANParser::UpdateCans(uint64_t sec, const capnp::List<cereal::CanData>::Reader& cans) {
-    int msg_count = cans.size();
+  int msg_count = cans.size();
 
-    DEBUG("got %d messages\n", msg_count);
+  DEBUG("got %d messages\n", msg_count);
 
+  for (int i = 0; i < msg_count; i++) {
+    auto cmsg = cans[i];
     // parse the messages
-    for (int i = 0; i < msg_count; i++) {
-      auto cmsg = cans[i];
-      if (cmsg.getSrc() != bus) {
-        // DEBUG("skip %d: wrong bus\n", cmsg.getAddress());
-        continue;
-      }
-      auto state_it = message_states.find(cmsg.getAddress());
-      if (state_it == message_states.end()) {
-        // DEBUG("skip %d: not specified\n", cmsg.getAddress());
-        continue;
-      }
-
-      if (cmsg.getDat().size() > 8) continue; //shouldnt ever happen
-      uint8_t dat[8] = {0};
-      memcpy(dat, cmsg.getDat().begin(), cmsg.getDat().size());
-
-      state_it->second.parse(sec, cmsg.getBusTime(), dat);
+    if (cmsg.getSrc() != bus) {
+      // DEBUG("skip %d: wrong bus\n", cmsg.getAddress());
+      continue;
     }
+    auto state_it = message_states.find(cmsg.getAddress());
+    if (state_it == message_states.end()) {
+      // DEBUG("skip %d: not specified\n", cmsg.getAddress());
+      continue;
+    }
+
+    if (cmsg.getDat().size() > 8) continue; //shouldn't ever happen
+    uint8_t dat[8] = {0};
+    memcpy(dat, cmsg.getDat().begin(), cmsg.getDat().size());
+
+    state_it->second.parse(sec, cmsg.getBusTime(), dat);
+  }
+}
+#endif
+
+void CANParser::UpdateCans(uint64_t sec, const capnp::DynamicStruct::Reader& cmsg) {
+  // assume message struct is `cereal::CanData` and parse
+  assert(cmsg.has("address") && cmsg.has("src") && cmsg.has("dat") && cmsg.has("busTime"));
+
+  if (cmsg.get("src").as<uint8_t>() != bus) {
+    DEBUG("skip %d: wrong bus\n", cmsg.get("address").as<uint32_t>());
+    return;
+  }
+
+  auto state_it = message_states.find(cmsg.get("address").as<uint32_t>());
+  if (state_it == message_states.end()) {
+    DEBUG("skip %d: not specified\n", cmsg.get("address").as<uint32_t>());
+    return;
+  }
+
+  auto dat = cmsg.get("dat").as<capnp::Data>();
+  if (dat.size() > 8) return; //shouldn't ever happen
+  uint8_t data[8] = {0};
+  memcpy(data, dat.begin(), dat.size());
+  state_it->second.parse(sec, cmsg.get("busTime").as<uint16_t>(), data);
 }
 
 void CANParser::UpdateValid(uint64_t sec) {
@@ -208,24 +278,6 @@ void CANParser::UpdateValid(uint64_t sec) {
   }
 }
 
-void CANParser::update_string(std::string data, bool sendcan) {
-  // format for board, make copy due to alignment issues, will be freed on out of scope
-  auto amsg = kj::heapArray<capnp::word>((data.length() / sizeof(capnp::word)) + 1);
-  memcpy(amsg.begin(), data.data(), data.length());
-
-  // extract the messages
-  capnp::FlatArrayMessageReader cmsg(amsg);
-  cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
-  last_sec = event.getLogMonoTime();
-
-  auto cans = sendcan? event.getSendcan() : event.getCan();
-  UpdateCans(last_sec, cans);
-
-  UpdateValid(last_sec);
-}
-
-
 std::vector<SignalValue> CANParser::query_latest() {
   std::vector<SignalValue> ret;
 

opendbc/can/parser.py

@@ -1,2 +1,2 @@
-from opendbc.can.parser_pyx import CANParser  # pylint: disable=no-name-in-module, import-error
-assert CANParser
+from opendbc.can.parser_pyx import CANParser, CANDefine  # pylint: disable=no-name-in-module, import-error
+assert CANParser, CANDefine

opendbc/can/parser_pyx.pyx

@@ -3,24 +3,20 @@
 
 from libcpp.string cimport string
 from libcpp.vector cimport vector
-from libcpp cimport bool
 from libcpp.unordered_set cimport unordered_set
 from libc.stdint cimport uint32_t, uint64_t, uint16_t
 from libcpp.map cimport map
-
-from collections import defaultdict
-
-from common cimport CANParser as cpp_CANParser
-from common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
-
-
 from libcpp cimport bool
+
+from .common cimport CANParser as cpp_CANParser
+from .common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
+
 import os
 import numbers
+from collections import defaultdict
 
 cdef int CAN_INVALID_CNT = 5
 
-
 cdef class CANParser:
   cdef:
     cpp_CANParser *can
@@ -30,7 +26,7 @@ cdef class CANParser:
     vector[SignalValue] can_values
     bool test_mode_enabled
 
-  cdef public:
+  cdef readonly:
     string dbc_name
     dict vl
     dict ts
@@ -40,10 +36,11 @@ cdef class CANParser:
   def __init__(self, dbc_name, signals, checks=None, bus=0):
     if checks is None:
       checks = []
-
     self.can_valid = True
     self.dbc_name = dbc_name
     self.dbc = dbc_lookup(dbc_name)
+    if not self.dbc:
+      raise RuntimeError("Can't lookup" + dbc_name)
     self.vl = {}
     self.ts = {}
 
@@ -113,7 +110,7 @@ cdef class CANParser:
 
 
     for cv in can_values:
-      # Cast char * directly to unicde
+      # Cast char * directly to unicode
       name = <unicode>self.address_to_msg_name[cv.address].c_str()
       cv_name = <unicode>cv.name
 
@@ -151,7 +148,9 @@ cdef class CANDefine():
   def __init__(self, dbc_name):
     self.dbc_name = dbc_name
     self.dbc = dbc_lookup(dbc_name)
-
+    if not self.dbc:
+      raise RuntimeError("Can't lookup" + dbc_name)
+      
     num_vals = self.dbc[0].num_vals
 
     address_to_msg_name = {}