openpilot v0.9.8 toyota-test-build

date: 2024-11-27T07:44:05
master commit: 2b6b8414700cbb048f8a76329ff52d0112f86bc2

.clang-tidy

@@ -0,0 +1,19 @@
+---
+Checks: '
+bugprone-*,
+-bugprone-integer-division,
+-bugprone-narrowing-conversions,
+performance-*,
+clang-analyzer-*,
+misc-*,
+-misc-unused-parameters,
+modernize-*,
+-modernize-avoid-c-arrays,
+-modernize-deprecated-headers,
+-modernize-use-auto,
+-modernize-use-using,
+-modernize-use-nullptr,
+-modernize-use-trailing-return-type,
+'
+CheckOptions:
+...

.codespellignore

@@ -1,6 +0,0 @@
-Wen
-REGIST
-PullRequest
-cancelled
-FOF
-NoO

.dockerignore

@@ -0,0 +1,39 @@
+**/.git
+.DS_Store
+*.dylib
+*.DSYM
+*.d
+*.pyc
+*.pyo
+.*.swp
+.*.swo
+.*.un~
+*.tmp
+*.o
+*.o-*
+*.os
+*.os-*
+*.so
+*.a
+
+venv/
+.venv/
+
+notebooks
+phone
+massivemap
+neos
+installer
+chffr/app2
+chffr/backend/env
+selfdrive/nav
+selfdrive/baseui
+selfdrive/test/simulator2
+**/cache_data
+xx/plus
+xx/community
+xx/projects
+!xx/projects/eon_testing_master
+!xx/projects/map3d
+xx/ops
+xx/junk

.editorconfig

@@ -5,7 +5,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{py,pyx,pxd}]
+[{*.py, *.pyx, *.pxd}]
 charset = utf-8
 indent_style = space
 indent_size = 2

.gitignore

@@ -10,23 +10,15 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
+model2.png
 a.out
 .hypothesis
-.cache/
-bin/
 
-# created at launch for TICI PYTHONPATH (PC uses editable installs via pyproject.toml)
-/msgq
-/opendbc
-/rednose
-/teleoprtc
-/tinygrad
+/docs_site/
 
-*.mp4
 *.dylib
 *.DSYM
 *.d
-*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -43,62 +35,71 @@ bin/
 *.class
 *.pyxbldc
 *.vcd
-*.mo
-*_pyx.cpp
-*.stats
-*.pkl
-*.pkl*
+*.qm
 config.json
+clcache
 compile_commands.json
 compare_runtime*.html
-openpilot/selfdrive/modeld/models/tg_input_devices.json
 
-# build artifacts
-docs_site/
-openpilot/selfdrive/pandad/pandad
-openpilot/cereal/services.h
-openpilot/cereal/gen
-openpilot/cereal/messaging/bridge
-openpilot/selfdrive/ui/translations/tmp
-openpilot/selfdrive/car/tests/cars_dump
-openpilot/system/camerad/camerad
-openpilot/system/camerad/test/ae_gray_test
+persist
+selfdrive/pandad/pandad
+cereal/services.h
+cereal/gen
+cereal/messaging/bridge
+selfdrive/logcatd/logcatd
+selfdrive/mapd/default_speeds_by_region.json
+system/proclogd/proclogd
+selfdrive/ui/translations/alerts_generated.h
+selfdrive/ui/translations/tmp
+selfdrive/test/longitudinal_maneuvers/out
+selfdrive/car/tests/cars_dump
+system/camerad/camerad
+system/camerad/test/ae_gray_test
+
+notebooks
+hyperthneed
+provisioning
 
 .coverage*
 coverage.xml
 htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/thneed/compile
+selfdrive/modeld/models/*.thneed
+selfdrive/modeld/models/*.pkl
 
-# openpilot log files
 *.bz2
 *.zst
-*.rlog
 
 build/
 
 !**/.gitkeep
 
+poetry.toml
+Pipfile
 
 ### VisualStudioCode ###
-*.vsix
-.history
-.ionide
 .vscode/*
-.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# agents
-.claude/
-.context/
-PLAN.md
-TASK.md
-CLAUDE.md
-SKILL.md
+# Local History for Visual Studio Code
+.history/
 
-### JetBrains ###
-!.idea/customTargets.xml
-!.idea/tools/*
-!.run/*
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide

.python-version

@@ -1 +0,0 @@
-3.12.13

.vscode/extensions.json

@@ -1,8 +0,0 @@
-{
-  "recommendations": [
-    "ms-python.python",
-    "ms-vscode.cpptools",
-    "elagil.pre-commit-helper",
-    "charliermarsh.ruff",
-  ]
-}

.vscode/launch.json

@@ -1,88 +0,0 @@
-{
-  "version": "0.2.0",
-  "inputs": [
-    {
-      "id": "python_process",
-      "type": "pickString",
-      "description": "Select the process to debug",
-      "options": [
-        "openpilot/selfdrive/controls/controlsd.py",
-        "openpilot/system/timed/timed.py",
-        "openpilot/tools/sim/run_bridge.py"
-      ]
-    },
-    {
-      "id": "cpp_process",
-      "type": "pickString",
-      "description": "Select the process to debug",
-      "options": [
-        "openpilot/selfdrive/ui/ui"
-      ]
-    },
-    {
-      "id": "args",
-      "description": "Arguments to pass to the process",
-      "type": "promptString"
-    },
-    {
-      "id": "replayArg",
-      "type": "promptString",
-      "description": "Enter route or segment to replay."
-    }
-  ],
-  "configurations": [
-    {
-      "name": "Python: openpilot Process",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${input:python_process}",
-      "console": "integratedTerminal",
-      "justMyCode": true,
-      "args": "${input:args}"
-    },
-    {
-      "name": "C++: openpilot Process",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}"
-    },
-    {
-      "name": "Attach LLDB to Replay drive",
-      "type": "lldb",
-      "request": "attach",
-      "pid": "${command:pickMyProcess}",
-      "sourceMap": {
-        ".": "${workspaceFolder}/opendbc/safety"
-      },
-      "initCommands": [
-        "script import time; time.sleep(3)"
-      ]
-    },
-    {
-      "name": "Replay drive",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
-      "args": [
-        "${input:replayArg}"
-      ],
-      "console": "integratedTerminal",
-      "justMyCode": false,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}"
-      },
-      "subProcess": true,
-      "stopOnEntry": false
-    }
-  ],
-  "compounds": [
-    {
-      "name": "Replay drive + Safety LLDB",
-      "configurations": [
-        "Replay drive",
-        "Attach LLDB to Replay drive"
-      ]
-    }
-  ]
-}

.vscode/settings.json

@@ -1,21 +0,0 @@
-{
-  "editor.tabSize": 2,
-  "editor.insertSpaces": true,
-  "editor.renderWhitespace": "trailing",
-  "files.trimTrailingWhitespace": true,
-  "search.exclude": {
-    "**/.git": true,
-    "**/.venv": true,
-    "**/__pycache__": true
-  },
-  "files.exclude": {
-    "**/.git": true,
-    "**/.venv": true,
-    "**/__pycache__": true
-  },
-  "python.analysis.exclude": [
-    "**/.git",
-    "**/.venv",
-    "**/__pycache__",
-  ]
-}

Jenkinsfile

@@ -0,0 +1,266 @@
+def retryWithDelay(int maxRetries, int delay, Closure body) {
+  for (int i = 0; i < maxRetries; i++) {
+    try {
+      return body()
+    } catch (Exception e) {
+      sleep(delay)
+    }
+  }
+  throw Exception("Failed after ${maxRetries} retries")
+}
+
+def device(String ip, String step_label, String cmd) {
+  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
+    def ssh_cmd = """
+ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
+
+set -e
+
+export TERM=xterm-256color
+
+shopt -s huponexit # kill all child processes when the shell exits
+
+export CI=1
+export PYTHONWARNINGS=error
+export LOGPRINT=debug
+export TEST_DIR=${env.TEST_DIR}
+export SOURCE_DIR=${env.SOURCE_DIR}
+export GIT_BRANCH=${env.GIT_BRANCH}
+export GIT_COMMIT=${env.GIT_COMMIT}
+export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
+export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
+export AZURE_TOKEN='${env.AZURE_TOKEN}'
+# only use 1 thread for tici tests since most require HIL
+export PYTEST_ADDOPTS="-n 0"
+
+
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
+
+source ~/.bash_profile
+if [ -f /TICI ]; then
+  source /etc/profile
+
+  rm -rf /tmp/tmp*
+  rm -rf ~/.commacache
+  rm -rf /dev/shm/*
+  rm -rf /dev/tmp/tmp*
+
+  if ! systemctl is-active --quiet systemd-resolved; then
+    echo "restarting resolved"
+    sudo systemctl start systemd-resolved
+    sleep 3
+  fi
+
+  # restart aux USB
+  if [ -e /sys/bus/usb/drivers/hub/3-0:1.0 ]; then
+    echo "restarting aux usb"
+    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/unbind
+    sleep 0.5
+    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/bind
+  fi
+fi
+if [ -f /data/openpilot/launch_env.sh ]; then
+  source /data/openpilot/launch_env.sh
+fi
+
+ln -snf ${env.TEST_DIR} /data/pythonpath
+
+cd ${env.TEST_DIR} || true
+time ${cmd}
+END"""
+
+    sh script: ssh_cmd, label: step_label
+  }
+}
+
+def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
+  stage(stageName) {
+    if (currentBuild.result != null) {
+        return
+    }
+
+    def extra = extra_env.collect { "export ${it}" }.join('\n');
+    def branch = env.BRANCH_NAME ?: 'master';
+    def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes'
+
+    lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
+      docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
+        timeout(time: 35, unit: 'MINUTES') {
+          retry (3) {
+            def date = sh(script: 'date', returnStdout: true).trim();
+            device(device_ip, "set time", "date -s '" + date + "'")
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          }
+          steps.each { item ->
+            def name = item[0]
+            def cmd = item[1]
+
+            def args = item[2]
+            def diffPaths = args.diffPaths ?: []
+            def cmdTimeout = args.timeout ?: 9999
+
+            if (branch != "master" && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
+              println "Skipping ${name}: no changes in ${diffPaths}."
+              return
+            } else {
+              timeout(time: cmdTimeout, unit: 'SECONDS') {
+                device(device_ip, name, cmd)
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+def hasPathChanged(String gitDiff, List<String> paths) {
+  for (path in paths) {
+    if (gitDiff.contains(path)) {
+      return true
+    }
+  }
+  return false
+}
+
+def setupCredentials() {
+  withCredentials([
+    string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
+  ]) {
+    env.AZURE_TOKEN = "${AZURE_TOKEN}"
+  }
+
+  withCredentials([
+    string(credentialsId: 'ci_artifacts_pat', variable: 'CI_ARTIFACTS_TOKEN'),
+  ]) {
+    env.CI_ARTIFACTS_TOKEN = "${CI_ARTIFACTS_TOKEN}"
+  }
+
+  withCredentials([
+    string(credentialsId: 'post_comments_github_pat', variable: 'GITHUB_COMMENTS_TOKEN'),
+  ]) {
+    env.GITHUB_COMMENTS_TOKEN = "${GITHUB_COMMENTS_TOKEN}"
+  }
+}
+
+def step(String name, String cmd, Map args = [:]) {
+  return [name, cmd, args]
+}
+
+node {
+  env.CI = "1"
+  env.PYTHONWARNINGS = "error"
+  env.TEST_DIR = "/data/openpilot"
+  env.SOURCE_DIR = "/data/openpilot_source/"
+  setupCredentials()
+
+  env.GIT_BRANCH = checkout(scm).GIT_BRANCH
+  env.GIT_COMMIT = checkout(scm).GIT_COMMIT
+
+  def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging',
+                         'testing-closet*', 'hotfix-*']
+  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
+
+  if (env.BRANCH_NAME != 'master') {
+    properties([
+        disableConcurrentBuilds(abortPrevious: true)
+    ])
+  }
+
+  try {
+    if (env.BRANCH_NAME == 'devel-staging') {
+      deviceStage("build release3-staging", "tici-needs-can", [], [
+        step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"),
+      ])
+    }
+
+    if (env.BRANCH_NAME == 'master-ci') {
+      parallel (
+        'nightly': {
+          deviceStage("build nightly", "tici-needs-can", [], [
+            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
+          ])
+        },
+        'nightly-dev': {
+          deviceStage("build nightly-dev", "tici-needs-can", [], [
+            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
+          ])
+        },
+      )
+    }
+
+    if (!env.BRANCH_NAME.matches(excludeRegex)) {
+    parallel (
+      // tici tests
+      'onroad tests': {
+        deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [
+          // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
+          //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("check dirty", "release/check-dirty.sh"),
+          step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
+          //["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
+        ])
+      },
+      'HW + Unit Tests': {
+        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
+          step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
+          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"),
+          step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"),
+          step("test manager", "pytest system/manager/test/test_manager.py"),
+        ])
+      },
+      'loopback': {
+        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+        ])
+      },
+      'camerad': {
+        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+      },
+      'sensord': {
+        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+        ])
+        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+        ])
+      },
+      'replay': {
+        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+        ])
+      },
+      'tizi': {
+        deviceStage("tizi", "tizi", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
+          step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
+        ])
+      },
+
+    )
+    }
+  } catch (Exception e) {
+    currentBuild.result = 'FAILED'
+    throw e
+  }
+}

LICENSE

@@ -1,7 +0,0 @@
-Copyright (c) 2018, Comma.ai, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.md

@@ -1,21 +0,0 @@
-# Custom MIT License
-
-Copyright (c) 2024, Haibin Wen, SUNNYPILOT LLC
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to view and modify the Software, subject to the following conditions:
-
-1. **Permission Required**: Permission Required for Commercial, For-Profit, or Closed Source Use: Use of the Software, in whole or in part, for any commercial purposes, for-profit projects, or in closed source projects requires explicit written permission from the original author(s).
-
-2. **Redistribution**: Any redistribution of the Software, modified or unmodified, must retain this license notice and the following acknowledgment:
-   "This software is licensed under a custom license requiring permission for use."
-
-3. **Visibility**: Any project that uses the Software must visibly mention the following acknowledgment:
-   "This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use."
-
-4. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-Contact sunnypilot Support <support@sunnypilot.ai> for permission requests.
-
----
-
-Haibin Wen, SUNNYPILOT LLC

README.md

@@ -1,74 +1,98 @@
-![](https://user-images.githubusercontent.com/47793918/233812617-beab2e71-57b9-479e-8bff-c3931347ca40.png)
+<div align="center" style="text-align: center;">
 
-## 🌞 What is sunnypilot?
-[sunnypilot](https://github.com/sunnyhaibin/sunnypilot) is a fork of comma.ai's openpilot, an open source driver assistance system. sunnypilot offers the user a unique driving experience for over 300+ supported car makes and models with modified behaviors of driving assist engagements. sunnypilot complies with comma.ai's safety rules as accurately as possible.
+<h1>openpilot</h1>
 
-## 💭 Join our Community Forum
-Join the official sunnypilot community forum to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
-* https://community.sunnypilot.ai/
+<p>
+  <b>openpilot is an operating system for robotics.</b>
+  <br>
+  Currently, it upgrades the driver assistance system in 275+ supported cars.
+</p>
 
-## Documentation
-https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
+<h3>
+  <a href="https://docs.comma.ai">Docs</a>
+  <span> · </span>
+  <a href="https://docs.comma.ai/contributing/roadmap/">Roadmap</a>
+  <span> · </span>
+  <a href="https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md">Contribute</a>
+  <span> · </span>
+  <a href="https://discord.comma.ai">Community</a>
+  <span> · </span>
+  <a href="https://comma.ai/shop">Try it on a comma 3X</a>
+</h3>
 
-## 🚘 Running on a dedicated device in a car
-First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
+Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
 
-## Installation
-Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
+![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
+[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
 
-## 🎆 Pull Requests
-We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.
+</div>
 
-Pull requests should be against the most current `master` branch.
+<table>
+  <tr>
+    <td><a href="https://youtu.be/NmBfgOanCyk" title="Video By Greer Viau"><img src="https://github.com/commaai/openpilot/assets/8762862/2f7112ae-f748-4f39-b617-fabd689c3772"></a></td>
+    <td><a href="https://youtu.be/VHKyqZ7t8Gw" title="Video By Logan LeGrand"><img src="https://github.com/commaai/openpilot/assets/8762862/92351544-2833-40d7-9e0b-7ef7ae37ec4c"></a></td>
+    <td><a href="https://youtu.be/SUIZYzxtMQs" title="A drive to Taco Bell"><img src="https://github.com/commaai/openpilot/assets/8762862/05ceefc5-2628-439c-a9b2-89ce77dc6f63"></a></td>
+  </tr>
+</table>
 
-## 📊 User Data
+To start using openpilot in a car
+------
 
-By default, sunnypilot uploads the driving data to comma servers. You can also access your data through [comma connect](https://connect.comma.ai/).
+To use openpilot in a car, you need four things:
+1. **Supported Device:** a comma 3/3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x).
+2. **Software:** The setup procedure for the comma 3/3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
+3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
+4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3/3X to your car.
 
-sunnypilot is open source software. The user is free to disable data collection if they wish to do so.
+We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
 
-sunnypilot logs the road-facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
+To start developing openpilot
+------
 
-By using this software, you understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
+openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot).
 
-## Licensing
+* Join the [community Discord](https://discord.comma.ai)
+* Check out [the contributing docs](docs/CONTRIBUTING.md)
+* Check out the [openpilot tools](tools/)
+* Read about the [development workflow](docs/WORKFLOW.md)
+* Code documentation lives at https://docs.comma.ai
+* Information about running openpilot lives on the [community wiki](https://github.com/commaai/openpilot/wiki)
 
-sunnypilot is released under the [MIT License](LICENSE). This repository includes original work as well as significant portions of code derived from [openpilot by comma.ai](https://github.com/commaai/openpilot), which is also released under the MIT license with additional disclaimers.
+Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs#open-positions) and offers lots of [bounties](https://comma.ai/bounties) for external contributors.
 
-The original openpilot license notice, including comma.ai’s indemnification and alpha software disclaimer, is reproduced below as required:
+Safety and Testing
+----
 
-> openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
->
-> Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
->
-> **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
-> YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
-> NO WARRANTY EXPRESSED OR IMPLIED.**
+* openpilot observes [ISO26262](https://en.wikipedia.org/wiki/ISO_26262) guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
+* openpilot has software-in-the-loop [tests](.github/workflows/selfdrive_tests.yaml) that run on every commit.
+* The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
+* panda has software-in-the-loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
+* Internally, we have a hardware-in-the-loop Jenkins test suite that builds and unit tests the various processes.
+* panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
+* We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
 
-For full license terms, please see the [`LICENSE`](LICENSE) file.
+Licensing
+------
 
-## 💰 Support sunnypilot
-If you find any of the features useful, consider becoming a [sponsor on GitHub](https://github.com/sponsors/sunnyhaibin) to support future feature development and improvements.
+openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
+Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
 
-By becoming a sponsor, you will gain access to exclusive content, early access to new features, and the opportunity to directly influence the project's development.
+**THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
+YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
+NO WARRANTY EXPRESSED OR IMPLIED.**
 
+User Data and comma Account
+------
 
-<h3>GitHub Sponsor</h3>
+By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
 
-<a href="https://github.com/sponsors/sunnyhaibin">
-  <img src="https://user-images.githubusercontent.com/47793918/244135584-9800acbd-69fd-4b2b-bec9-e5fa2d85c817.png" alt="Become a Sponsor" width="300" style="max-width: 100%; height: auto;">
-</a>
-<br>
+openpilot is open source software: the user is free to disable data collection if they wish to do so.
 
-<h3>PayPal</h3>
+openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
+The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
 
-<a href="https://paypal.me/sunnyhaibin0850" target="_blank">
-<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" alt="PayPal this" title="PayPal - The safer, easier way to pay online!" border="0" />
-</a>
-<br></br>
-
-Your continuous love and support are greatly appreciated! Enjoy 🥰
-
-<span>-</span> Jason, Founder of sunnypilot
+By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.

RELEASES.md

@@ -1,94 +1,16 @@
-Version 0.11.2 (2026-06-15)
-========================
-
-
-Version 0.11.1 (2026-05-18)
-========================
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Improved thermal policy for comma four
-* Acura MDX 2022-24 support thanks to mvl-boston!
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-Version 0.11.0 (2026-03-17)
-========================
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-
-Version 0.10.3 (2025-12-17)
-========================
-* New driving model #36249
-  * New temporal policy architecture
-  * New on-policy training physics noise model
-* New driver monitoring model #36409
-  * Trained on a new dataset, including comma four data
-* Improved inter-process communication memory efficiency
-
-Version 0.10.2 (2025-11-19)
-========================
-* comma four support
-
-Version 0.10.1 (2025-09-08)
-========================
-* New driving model #36276
-  * World Model: removed global localization inputs
-  * World Model: 2x the number of parameters
-  * World Model: trained on 4x the number of segments
-  * VAE Compression Model: new architecture and training objective
-  * Driving Vision Model: trained on 4x the number of segments
-* New Driver Monitoring model #36198
-* Acura TLX 2021 support thanks to MVL!
-* Honda City 2023 support thanks to vanillagorillaa and drFritz!
-* Honda N-Box 2018 support thanks to miettal!
-* Honda Odyssey 2021-25 support thanks to csouers and MVL!
-* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
-
-Version 0.10.0 (2025-08-05)
+Version 0.9.8 (2024-XX-XX)
 ========================
 * New driving model
-  * New training architecture
-     * Described in our CVPR paper: "Learning to Drive from a World Model"
-     * Longitudinal MPC replaced by E2E planning from World Model in Experimental Mode
-     * Action from lateral MPC as training objective replaced by E2E planning from World Model
-  * Low-speed lead car ground-truth fixes
-* Enable live-learned steering actuation delay
-* Opt-in audio recording for dashcam video
-* Acura MDX 2025 support thanks to vanillagorillaa and MVL!
-* Honda Accord 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda CR-V 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda Pilot 2023-25 support thanks to vanillagorillaa and MVL!
-
-Version 0.9.9 (2025-05-23)
-========================
-* New driving model
-  * New training architecture using parts from MLSIM
-* Steering actuation delay is now learned online
-* Ford Escape 2023-24 support thanks to incognitojam!
-* Ford Kuga 2024 support thanks to incognitojam!
-* Hyundai Nexo 2021 support thanks to sunnyhaibin!
-* Tesla Model 3 and Y support thanks to lukasloetkolben!
-* Lexus RC 2023 support thanks to nelsonjchen!
-
-Version 0.9.8 (2025-02-28)
-========================
-* New driving model
-  * Model now gates applying positive acceleration in Chill mode
-* New driver monitoring model
+  * Trained in brand new ML simulator
+  * Model now gates applying positive accel in Chill mode
+* New driving monitoring model
   * Reduced false positives related to passengers
 * Image processing pipeline moved to the ISP
-  * More GPU time for bigger driving models
-  * Power draw reduced 0.5W, which means your device runs cooler
+  * More GPU time for driving models
+  * Power draw reduced 0.5W, which means your device runs cooler 
 * Added toggle to enable driver monitoring even when openpilot is not engaged
-* Localizer rewritten to remove GPS dependency at runtime
-* Firehose Mode for maximizing your training data uploads
 * Enable openpilot longitudinal control for Ford Q3 vehicles
 * New Toyota TSS2 longitudinal tune
-* Rivian R1S and R1T support thanks to lukasloetkolben!
-* Ford F-150, F-150 Hybrid, Mach-E, and Ranger support
 
 Version 0.9.7 (2024-06-13)
 ========================
@@ -1104,7 +1026,7 @@ Version 0.2.1  (2016-12-14)
 
 Version 0.2  (2016-12-12)
 =========================
- * Car/Radar abstraction layers have shipped, see openpilot/cereal/car.capnp
+ * Car/Radar abstraction layers have shipped, see cereal/car.capnp
  * controlsd has been refactored
  * Shipped plant model and testing maneuvers
  * visiond exits more gracefully now

SConstruct

@@ -0,0 +1,394 @@
+import os
+import subprocess
+import sys
+import sysconfig
+import platform
+import numpy as np
+
+import SCons.Errors
+
+SCons.Warnings.warningAsException(True)
+
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
+TICI = os.path.isfile('/TICI')
+AGNOS = TICI
+
+Decider('MD5-timestamp')
+
+SetOption('num_jobs', int(os.cpu_count()/2))
+
+AddOption('--kaitai',
+          action='store_true',
+          help='Regenerate kaitai struct parsers')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--compile_db',
+          action='store_true',
+          help='build clang compilation database')
+
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
+
+AddOption('--snpe',
+          action='store_true',
+          help='use SNPE on PC')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+AddOption('--pc-thneed',
+          action='store_true',
+          dest='pc_thneed',
+          help='use thneed on pc')
+
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
+
+AddOption('--minimal',
+          action='store_false',
+          dest='extras',
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
+          help='the minimum build to run openpilot. no tests, tools, etc.')
+
+## Architecture name breakdown (arch)
+## - larch64: linux tici aarch64
+## - aarch64: linux pc aarch64
+## - x86_64:  linux pc x64
+## - Darwin:  mac x64 or arm64
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and AGNOS:
+  arch = "larch64"
+assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
+
+lenv = {
+  "PATH": os.environ['PATH'],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+}
+
+rpath = lenv["LD_LIBRARY_PATH"].copy()
+
+if arch == "larch64":
+  cpppath = [
+    "#third_party/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/local/lib",
+    "/system/vendor/lib64",
+    f"#third_party/acados/{arch}/lib",
+  ]
+
+  libpath += [
+    "#third_party/snpe/larch64",
+    "#third_party/libyuv/larch64/lib",
+    "/usr/lib/aarch64-linux-gnu"
+  ]
+  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  rpath += ["/usr/local/lib"]
+else:
+  cflags = []
+  cxxflags = []
+  cpppath = []
+  rpath += []
+
+  # MacOS
+  if arch == "Darwin":
+    libpath = [
+      f"#third_party/libyuv/{arch}/lib",
+      f"#third_party/acados/{arch}/lib",
+      f"{brew_prefix}/lib",
+      f"{brew_prefix}/opt/openssl@3.0/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += [
+      f"{brew_prefix}/include",
+      f"{brew_prefix}/opt/openssl@3.0/include",
+    ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
+  # Linux
+  else:
+    libpath = [
+      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{arch}/lib",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
+
+if GetOption('asan'):
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
+else:
+  ccflags = []
+  ldflags = []
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
+
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
+
+env = Environment(
+  ENV=lenv,
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Wunused",
+    "-Werror",
+    "-Wshadow",
+    "-Wno-unknown-warning-option",
+    "-Wno-inconsistent-missing-override",
+    "-Wno-c99-designator",
+    "-Wno-reorder-init-list",
+    "-Wno-vla-cxx-extension",
+  ] + cflags + ccflags,
+
+  CPPPATH=cpppath + [
+    "#",
+    "#third_party/acados/include",
+    "#third_party/acados/include/blasfeo/include",
+    "#third_party/acados/include/hpipm/include",
+    "#third_party/catch2/include",
+    "#third_party/libyuv/include",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/snpe/include",
+    "#third_party",
+    "#msgq",
+  ],
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
+  LIBPATH=libpath + [
+    "#msgq_repo",
+    "#third_party",
+    "#selfdrive/pandad",
+    "#common",
+    "#rednose/helpers",
+  ],
+  CYTHONCFILESUFFIX=".cpp",
+  COMPILATIONDB_USE_ABSPATH=True,
+  REDNOSE_ROOT="#",
+  tools=["default", "cython", "compilation_db", "rednose_filter"],
+  toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
+)
+
+if arch == "Darwin":
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
+
+if GetOption('compile_db'):
+  env.CompilationDatabase('compile_commands.json')
+
+# Setup cache dir
+cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
+
+node_interval = 5
+node_count = 0
+def progress_function(node):
+  global node_count
+  node_count += node_interval
+  sys.stderr.write("progress: %d\n" % node_count)
+
+if os.environ.get('SCONS_PROGRESS'):
+  Progress(progress_function, interval=node_interval)
+
+# Cython build environment
+py_include = sysconfig.get_paths()['include']
+envCython = env.Clone()
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove("-Werror")
+
+envCython["LIBS"] = []
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+np_version = SCons.Script.Value(np.__version__)
+Export('envCython', 'np_version')
+
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
+
+qt_libs = []
+if arch == "Darwin":
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
+else:
+  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
+  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
+
+  qt_env['QTDIR'] = qt_install_prefix
+  qt_dirs = [
+    f"{qt_install_headers}",
+  ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
+  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+qt_env['QT3DIR'] = qt_env['QTDIR']
+
+# compatibility for older SCons versions
+try:
+  qt_env.Tool('qt3')
+except SCons.Errors.UserError:
+  qt_env.Tool('qt')
+
+qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_CORE_LIB",
+  "-DQT_MESSAGELOGCONTEXT",
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui', ]
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+
+Export('env', 'qt_env', 'arch', 'real_arch')
+
+# Build common module
+SConscript(['common/SConscript'])
+Import('_common', '_gpucommon')
+
+common = [_common, 'json11', 'zmq']
+gpucommon = [_gpucommon]
+
+Export('common', 'gpucommon')
+
+# Build messaging (cereal + msgq + socketmaster + their dependencies)
+# Enable swaglog include in submodules
+env_swaglog = env.Clone()
+env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
+SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
+
+SConscript(['cereal/SConscript'])
+
+Import('socketmaster', 'msgq')
+messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
+Export('messaging')
+
+
+# Build other submodules
+SConscript(['panda/SConscript'])
+
+# Build rednose library
+SConscript(['rednose/SConscript'])
+
+# Build system services
+SConscript([
+  'system/proclogd/SConscript',
+  'system/ubloxd/SConscript',
+  'system/loggerd/SConscript',
+])
+if arch != "Darwin":
+  SConscript([
+    'system/sensord/SConscript',
+    'system/logcatd/SConscript',
+  ])
+
+if arch == "larch64":
+  SConscript(['system/camerad/SConscript'])
+
+# Build openpilot
+SConscript(['third_party/SConscript'])
+
+SConscript(['selfdrive/SConscript'])
+
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
+  SConscript(['tools/replay/SConscript'])
+  if arch != "larch64":
+    SConscript(['tools/cabana/SConscript'])
+
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

cereal/README.md

@@ -0,0 +1,51 @@
+# What is cereal?
+
+cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
+
+
+## Messaging Spec
+
+You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
+
+All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
+
+### Best Practices
+
+- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
+- In the context of the message they are in, field names should be completely unambiguous.
+- All values should be easy to plot and be human-readable with minimal parsing.
+
+### Maintaining backwards-compatibility
+
+When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
+be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
+things are not. Read more details [here](https://capnproto.org/language.html).
+
+### Custom forks
+
+Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
+spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
+then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
+[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
+fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
+
+Example
+---
+```python
+import cereal.messaging as messaging
+
+# in subscriber
+sm = messaging.SubMaster(['sensorEvents'])
+while 1:
+  sm.update()
+  print(sm['sensorEvents'])
+
+```
+
+```python
+# in publisher
+pm = messaging.PubMaster(['sensorEvents'])
+dat = messaging.new_message('sensorEvents', size=1)
+dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
+pm.send('sensorEvents', dat)
+```

cereal/SConscript

@@ -0,0 +1,20 @@
+Import('env', 'common', 'msgq')
+
+cereal_dir = Dir('.')
+gen_dir = Dir('gen')
+
+# Build cereal
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
+env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
+            schema_files,
+            f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
+
+cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
+
+# Build messaging
+services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+
+socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
+
+Export('cereal', 'socketmaster')

cereal/__init__.py

@@ -0,0 +1,9 @@
+import os
+import capnp
+
+CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
+capnp.remove_import_hook()
+
+log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/car.capnp

@@ -0,0 +1 @@
+../opendbc_repo/opendbc/car/car.capnp

cereal/custom.capnp

@@ -0,0 +1,55 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0xb526ba661d550a59;
+
+# custom.capnp: a home for empty structs reserved for custom forks
+# These structs are guaranteed to remain reserved and empty in mainline
+# cereal, so use these if you want custom events in your fork.
+
+# you can rename the struct, but don't change the identifier
+struct SelfdriveStateSP @0x81c2f05a394cf4af {
+  mads @0 :ModularAssistiveDrivingSystem;
+
+  struct ModularAssistiveDrivingSystem {
+    state @0 :ModularAssistiveDrivingSystemState;
+    enabled @1 :Bool;
+    active @2 :Bool;
+    available @3 :Bool;
+
+    enum ModularAssistiveDrivingSystemState {
+      disabled @0;
+      paused @1;
+      enabled @2;
+      softDisabling @3;
+      overriding @4;
+    }
+  }
+}
+
+struct CustomReserved1 @0xaedffd8f31e7b55d {
+}
+
+struct CustomReserved2 @0xf35cc4560bbf6ec2 {
+}
+
+struct CustomReserved3 @0xda96579883444c35 {
+}
+
+struct CustomReserved4 @0x80ae746ee2596b11 {
+}
+
+struct CustomReserved5 @0xa5cd762cd951a455 {
+}
+
+struct CustomReserved6 @0xf98d843bfd7004a3 {
+}
+
+struct CustomReserved7 @0xb86e6369214c01c8 {
+}
+
+struct CustomReserved8 @0xf416ec09499d9d19 {
+}
+
+struct CustomReserved9 @0xa1680744031fdb2d {
+}

cereal/legacy.capnp

@@ -0,0 +1,574 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0x80ef1ec4889c2a63;
+
+# legacy.capnp: a home for deprecated structs
+
+struct LogRotate @0x9811e1f38f62f2d1 {
+  segmentNum @0 :Int32;
+  path @1 :Text;
+}
+
+struct LiveUI @0xc08240f996aefced {
+  rearViewCam @0 :Bool;
+  alertText1 @1 :Text;
+  alertText2 @2 :Text;
+  awarenessStatus @3 :Float32;
+}
+
+struct UiLayoutState @0x88dcce08ad29dda0 {
+  activeApp @0 :App;
+  sidebarCollapsed @1 :Bool;
+  mapEnabled @2 :Bool;
+  mockEngaged @3 :Bool;
+
+  enum App @0x9917470acf94d285 {
+    home @0;
+    music @1;
+    nav @2;
+    settings @3;
+    none @4;
+  }
+}
+
+struct OrbslamCorrection @0x8afd33dc9b35e1aa {
+  correctionMonoTime @0 :UInt64;
+  prePositionECEF @1 :List(Float64);
+  postPositionECEF @2 :List(Float64);
+  prePoseQuatECEF @3 :List(Float32);
+  postPoseQuatECEF @4 :List(Float32);
+  numInliers @5 :UInt32;
+}
+
+struct EthernetPacket @0xa99a9d5b33cf5859 {
+  pkt @0 :Data;
+  ts @1 :Float32;
+}
+
+struct CellInfo @0xcff7566681c277ce {
+  timestamp @0 :UInt64;
+  repr @1 :Text; # android toString() for now
+}
+
+struct WifiScan @0xd4df5a192382ba0b {
+  bssid @0 :Text;
+  ssid @1 :Text;
+  capabilities @2 :Text;
+  frequency @3 :Int32;
+  level @4 :Int32;
+  timestamp @5 :Int64;
+
+  centerFreq0 @6 :Int32;
+  centerFreq1 @7 :Int32;
+  channelWidth @8 :ChannelWidth;
+  operatorFriendlyName @9 :Text;
+  venueName @10 :Text;
+  is80211mcResponder @11 :Bool;
+  passpoint @12 :Bool;
+
+  distanceCm @13 :Int32;
+  distanceSdCm @14 :Int32;
+
+  enum ChannelWidth @0xcb6a279f015f6b51 {
+    w20Mhz @0;
+    w40Mhz @1;
+    w80Mhz @2;
+    w160Mhz @3;
+    w80Plus80Mhz @4;
+  }
+}
+
+struct LiveEventData @0x94b7baa90c5c321e {
+  name @0 :Text;
+  value @1 :Int32;
+}
+
+struct ModelData @0xb8aad62cffef28a9 {
+  frameId @0 :UInt32;
+  frameAge @12 :UInt32;
+  frameDropPerc @13 :Float32;
+  timestampEof @9 :UInt64;
+  modelExecutionTime @14 :Float32;
+  gpuExecutionTime @16 :Float32;
+  rawPred @15 :Data;
+
+  path @1 :PathData;
+  leftLane @2 :PathData;
+  rightLane @3 :PathData;
+  lead @4 :LeadData;
+  freePath @6 :List(Float32);
+
+  settings @5 :ModelSettings;
+  leadFuture @7 :LeadData;
+  speed @8 :List(Float32);
+  meta @10 :MetaData;
+  longitudinal @11 :LongitudinalData;
+
+  struct PathData @0x8817eeea389e9f08 {
+    points @0 :List(Float32);
+    prob @1 :Float32;
+    std @2 :Float32;
+    stds @3 :List(Float32);
+    poly @4 :List(Float32);
+    validLen @5 :Float32;
+  }
+
+  struct LeadData @0xd1c9bef96d26fa91 {
+    dist @0 :Float32;
+    prob @1 :Float32;
+    std @2 :Float32;
+    relVel @3 :Float32;
+    relVelStd @4 :Float32;
+    relY @5 :Float32;
+    relYStd @6 :Float32;
+    relA @7 :Float32;
+    relAStd @8 :Float32;
+  }
+
+  struct ModelSettings @0xa26e3710efd3e914 {
+    bigBoxX @0 :UInt16;
+    bigBoxY @1 :UInt16;
+    bigBoxWidth @2 :UInt16;
+    bigBoxHeight @3 :UInt16;
+    boxProjection @4 :List(Float32);
+    yuvCorrection @5 :List(Float32);
+    inputTransform @6 :List(Float32);
+  }
+
+  struct MetaData @0x9744f25fb60f2bf8 {
+    engagedProb @0 :Float32;
+    desirePrediction @1 :List(Float32);
+    brakeDisengageProb @2 :Float32;
+    gasDisengageProb @3 :Float32;
+    steerOverrideProb @4 :Float32;
+    desireState @5 :List(Float32);
+  }
+
+  struct LongitudinalData @0xf98f999c6a071122 {
+    distances @2 :List(Float32);
+    speeds @0 :List(Float32);
+    accelerations @1 :List(Float32);
+  }
+}
+
+struct ECEFPoint @0xc25bbbd524983447 {
+  x @0 :Float64;
+  y @1 :Float64;
+  z @2 :Float64;
+}
+
+struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 {
+  x @0 :Float32;
+  y @1 :Float32;
+  z @2 :Float32;
+}
+
+struct GPSPlannerPoints @0xab54c59699f8f9f3 {
+  curPosDEPRECATED @0 :ECEFPointDEPRECATED;
+  pointsDEPRECATED @1 :List(ECEFPointDEPRECATED);
+  curPos @6 :ECEFPoint;
+  points @7 :List(ECEFPoint);
+  valid @2 :Bool;
+  trackName @3 :Text;
+  speedLimit @4 :Float32;
+  accelTarget @5 :Float32;
+}
+
+struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b {
+  valid @0 :Bool;
+  poly @1 :List(Float32);
+  trackName @2 :Text;
+  speed @3 :Float32;
+  acceleration @4 :Float32;
+  pointsDEPRECATED @5 :List(ECEFPointDEPRECATED);
+  points @6 :List(ECEFPoint);
+  xLookahead @7 :Float32;
+}
+
+struct UiNavigationEvent @0x90c8426c3eaddd3b {
+  type @0: Type;
+  status @1: Status;
+  distanceTo @2: Float32;
+  endRoadPointDEPRECATED @3: ECEFPointDEPRECATED;
+  endRoadPoint @4: ECEFPoint;
+
+  enum Type @0xe8db07dcf8fcea05 {
+    none @0;
+    laneChangeLeft @1;
+    laneChangeRight @2;
+    mergeLeft @3;
+    mergeRight @4;
+    turnLeft @5;
+    turnRight @6;
+  }
+
+  enum Status @0xb9aa88c75ef99a1f {
+    none @0;
+    passive @1;
+    approaching @2;
+    active @3;
+  }
+}
+
+struct LiveLocationData @0xb99b2bc7a57e8128 {
+  status @0 :UInt8;
+
+  # 3D fix
+  lat @1 :Float64;
+  lon @2 :Float64;
+  alt @3 :Float32;     # m
+
+  # speed
+  speed @4 :Float32;   # m/s
+
+  # NED velocity components
+  vNED @5 :List(Float32);
+
+  # roll, pitch, heading (x,y,z)
+  roll @6 :Float32;     # WRT to center of earth?
+  pitch @7 :Float32;    # WRT to center of earth?
+  heading @8 :Float32;  # WRT to north?
+
+  # what are these?
+  wanderAngle @9 :Float32;
+  trackAngle @10 :Float32;
+
+  # car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png
+
+  # gyro, in car frame, deg/s
+  gyro @11 :List(Float32);
+
+  # accel, in car frame, m/s^2
+  accel @12 :List(Float32);
+
+  accuracy @13 :Accuracy;
+
+  source @14 :SensorSource;
+  # if we are fixing a location in the past
+  fixMonoTime @15 :UInt64;
+
+  gpsWeek @16 :Int32;
+  timeOfWeek @17 :Float64;
+
+  positionECEF @18 :List(Float64);
+  poseQuatECEF @19 :List(Float32);
+  pitchCalibration @20 :Float32;
+  yawCalibration @21 :Float32;
+  imuFrame @22 :List(Float32);
+
+  struct Accuracy @0x943dc4625473b03f {
+    pNEDError @0 :List(Float32);
+    vNEDError @1 :List(Float32);
+    rollError @2 :Float32;
+    pitchError @3 :Float32;
+    headingError @4 :Float32;
+    ellipsoidSemiMajorError @5 :Float32;
+    ellipsoidSemiMinorError @6 :Float32;
+    ellipsoidOrientationError @7 :Float32;
+  }
+
+  enum SensorSource @0xc871d3cc252af657 {
+    applanix @0;
+    kalman @1;
+    orbslam @2;
+    timing @3;
+    dummy @4;
+  }
+}
+
+struct OrbOdometry @0xd7700859ed1f5b76 {
+  # timing first
+  startMonoTime @0 :UInt64;
+  endMonoTime @1 :UInt64;
+
+  # fundamental matrix and error
+  f @2: List(Float64);
+  err @3: Float64;
+
+  # number of inlier points
+  inliers @4: Int32;
+
+  # for debug only
+  # indexed by endMonoTime features
+  # value is startMonoTime feature match
+  # -1 if no match
+  matches @5: List(Int16);
+}
+
+struct OrbFeatures @0xcd60164a8a0159ef {
+  timestampEof @0 :UInt64;
+  # transposed arrays of normalized image coordinates
+  # len(xs) == len(ys) == len(descriptors) * 32
+  xs @1 :List(Float32);
+  ys @2 :List(Float32);
+  descriptors @3 :Data;
+  octaves @4 :List(Int8);
+
+  # match index to last OrbFeatures
+  # -1 if no match
+  timestampLastEof @5 :UInt64;
+  matches @6: List(Int16);
+}
+
+struct OrbFeaturesSummary @0xd500d30c5803fa4f {
+  timestampEof @0 :UInt64;
+  timestampLastEof @1 :UInt64;
+
+  featureCount @2 :UInt16;
+  matchCount @3 :UInt16;
+  computeNs @4 :UInt64;
+}
+
+struct OrbKeyFrame @0xc8233c0345e27e24 {
+  # this is a globally unique id for the KeyFrame
+  id @0: UInt64;
+
+  # this is the location of the KeyFrame
+  pos @1: ECEFPoint;
+
+  # these are the features in the world
+  # len(dpos) == len(descriptors) * 32
+  dpos @2 :List(ECEFPoint);
+  descriptors @3 :Data;
+}
+
+struct KalmanOdometry @0x92e21bb7ea38793a {
+  trans @0 :List(Float32); # m/s in device frame
+  rot @1 :List(Float32); # rad/s in device frame
+  transStd @2 :List(Float32); # std m/s in device frame
+  rotStd @3 :List(Float32); # std rad/s in device frame
+}
+
+struct OrbObservation @0x9b326d4e436afec7 {
+  observationMonoTime @0 :UInt64;
+  normalizedCoordinates @1 :List(Float32);
+  locationECEF @2 :List(Float64);
+  matchDistance @3: UInt32;
+}
+
+struct CalibrationFeatures @0x8fdfadb254ea867a {
+  frameId @0 :UInt32;
+
+  p0 @1 :List(Float32);
+  p1 @2 :List(Float32);
+  status @3 :List(Int8);
+}
+
+struct NavStatus @0xbd8822120928120c {
+  isNavigating @0 :Bool;
+  currentAddress @1 :Address;
+
+  struct Address @0xce7cd672cacc7814 {
+    title @0 :Text;
+    lat @1 :Float64;
+    lng @2 :Float64;
+    house @3 :Text;
+    address @4 :Text;
+    street @5 :Text;
+    city @6 :Text;
+    state @7 :Text;
+    country @8 :Text;
+  }
+}
+
+struct NavUpdate @0xdb98be6565516acb {
+  isNavigating @0 :Bool;
+  curSegment @1 :Int32;
+  segments @2 :List(Segment);
+
+  struct LatLng @0x9eaef9187cadbb9b {
+    lat @0 :Float64;
+    lng @1 :Float64;
+  }
+
+  struct Segment @0xa5b39b4fc4d7da3f {
+    from @0 :LatLng;
+    to @1 :LatLng;
+    updateTime @2 :Int32;
+    distance @3 :Int32;
+    crossTime @4 :Int32;
+    exitNo @5 :Int32;
+    instruction @6 :Instruction;
+
+    parts @7 :List(LatLng);
+
+    enum Instruction @0xc5417a637451246f {
+      turnLeft @0;
+      turnRight @1;
+      keepLeft @2;
+      keepRight @3;
+      straight @4;
+      roundaboutExitNumber @5;
+      roundaboutExit @6;
+      roundaboutTurnLeft @7;
+      unkn8 @8;
+      roundaboutStraight @9;
+      unkn10 @10;
+      roundaboutTurnRight @11;
+      unkn12 @12;
+      roundaboutUturn @13;
+      unkn14 @14;
+      arrive @15;
+      exitLeft @16;
+      exitRight @17;
+      unkn18 @18;
+      uturn @19;
+      # ...
+    }
+  }
+}
+
+struct TrafficEvent @0xacfa74a094e62626 {
+  type @0 :Type;
+  distance @1 :Float32;
+  action @2 :Action;
+  resuming @3 :Bool;
+
+  enum Type @0xd85d75253435bf4b {
+    stopSign @0;
+    lightRed @1;
+    lightYellow @2;
+    lightGreen @3;
+    stopLight @4;
+  }
+
+  enum Action @0xa6f6ce72165ccb49 {
+    none @0;
+    yield @1;
+    stop @2;
+    resumeReady @3;
+  }
+
+}
+
+
+struct AndroidGnss @0xdfdf30d03fc485bd {
+  union {
+    measurements @0 :Measurements;
+    navigationMessage @1 :NavigationMessage;
+  }
+
+  struct Measurements @0xa20710d4f428d6cd {
+    clock @0 :Clock;
+    measurements @1 :List(Measurement);
+
+    struct Clock @0xa0e27b453a38f450 {
+      timeNanos @0 :Int64;
+      hardwareClockDiscontinuityCount @1 :Int32;
+
+      hasTimeUncertaintyNanos @2 :Bool;
+      timeUncertaintyNanos @3 :Float64;
+
+      hasLeapSecond @4 :Bool;
+      leapSecond @5 :Int32;
+
+      hasFullBiasNanos @6 :Bool;
+      fullBiasNanos @7 :Int64;
+
+      hasBiasNanos @8 :Bool;
+      biasNanos @9 :Float64;
+
+      hasBiasUncertaintyNanos @10 :Bool;
+      biasUncertaintyNanos @11 :Float64;
+
+      hasDriftNanosPerSecond @12 :Bool;
+      driftNanosPerSecond @13 :Float64;
+
+      hasDriftUncertaintyNanosPerSecond @14 :Bool;
+      driftUncertaintyNanosPerSecond @15 :Float64;
+    }
+
+    struct Measurement @0xd949bf717d77614d {
+      svId @0 :Int32;
+      constellation @1 :Constellation;
+
+      timeOffsetNanos @2 :Float64;
+      state @3 :Int32;
+      receivedSvTimeNanos @4 :Int64;
+      receivedSvTimeUncertaintyNanos @5 :Int64;
+      cn0DbHz @6 :Float64;
+      pseudorangeRateMetersPerSecond @7 :Float64;
+      pseudorangeRateUncertaintyMetersPerSecond @8 :Float64;
+      accumulatedDeltaRangeState @9 :Int32;
+      accumulatedDeltaRangeMeters @10 :Float64;
+      accumulatedDeltaRangeUncertaintyMeters @11 :Float64;
+
+      hasCarrierFrequencyHz @12 :Bool;
+      carrierFrequencyHz @13 :Float32;
+      hasCarrierCycles @14 :Bool;
+      carrierCycles @15 :Int64;
+      hasCarrierPhase @16 :Bool;
+      carrierPhase @17 :Float64;
+      hasCarrierPhaseUncertainty @18 :Bool;
+      carrierPhaseUncertainty @19 :Float64;
+      hasSnrInDb @20 :Bool;
+      snrInDb @21 :Float64;
+
+      multipathIndicator @22 :MultipathIndicator;
+
+      enum Constellation @0x9ef1f3ff0deb5ffb {
+        unknown @0;
+        gps @1;
+        sbas @2;
+        glonass @3;
+        qzss @4;
+        beidou @5;
+        galileo @6;
+      }
+
+      enum State @0xcbb9490adce12d72 {
+        unknown @0;
+        codeLock @1;
+        bitSync @2;
+        subframeSync @3;
+        towDecoded @4;
+        msecAmbiguous @5;
+        symbolSync @6;
+        gloStringSync @7;
+        gloTodDecoded @8;
+        bdsD2BitSync @9;
+        bdsD2SubframeSync @10;
+        galE1bcCodeLock @11;
+        galE1c2ndCodeLock @12;
+        galE1bPageSync @13;
+        sbasSync @14;
+      }
+
+      enum MultipathIndicator @0xc04e7b6231d4caa8 {
+        unknown @0;
+        detected @1;
+        notDetected @2;
+      }
+    }
+  }
+
+  struct NavigationMessage @0xe2517b083095fd4e {
+    type @0 :Int32;
+    svId @1 :Int32;
+    messageId @2 :Int32;
+    submessageId @3 :Int32;
+    data @4 :Data;
+    status @5 :Status;
+
+    enum Status @0xec1ff7996b35366f {
+      unknown @0;
+      parityPassed @1;
+      parityRebuilt @2;
+    }
+  }
+}
+
+struct LidarPts @0xe3d6685d4e9d8f7a {
+  r @0 :List(UInt16);        # uint16   m*500.0
+  theta @1 :List(UInt16);    # uint16 deg*100.0
+  reflect @2 :List(UInt8);   # uint8      0-255
+
+  # For storing out of file.
+  idx @3 :UInt64;
+
+  # For storing in file
+  pkt @4 :Data;
+}
+
+

cereal/maptile.capnp

@@ -0,0 +1,49 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0xa086df597ef5d7a0;
+
+# Geometry
+struct Point {
+  x @0: Float64;
+  y @1: Float64;
+  z @2: Float64;
+}
+
+struct PolyLine {
+  points @0: List(Point);
+}
+
+# Map features
+struct Lane {
+  id @0 :Text;
+
+  leftBoundary @1 :LaneBoundary;
+  rightBoundary @2 :LaneBoundary;
+
+  leftAdjacentId @3 :Text;
+  rightAdjacentId @4 :Text;
+
+  inboundIds @5 :List(Text);
+  outboundIds @6 :List(Text);
+
+  struct LaneBoundary {
+    polyLine @0 :PolyLine;
+    startHeading @1 :Float32; # WRT north
+  }
+}
+
+# Map tiles
+struct TileSummary {
+  version @0 :Text;
+  updatedAt @1 :UInt64; # Millis since epoch
+
+  level @2 :UInt8;
+  x @3 :UInt16;
+  y @4 :UInt16;
+}
+
+struct MapTile {
+  summary @0 :TileSummary;
+  lanes @1 :List(Lane);
+}

cereal/messaging/__init__.py

@@ -0,0 +1,268 @@
+# must be built with scons
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
+import msgq
+
+import os
+import capnp
+import time
+
+from typing import Optional, List, Union, Dict, Deque
+from collections import deque
+
+from cereal import log
+from cereal.services import SERVICE_LIST
+
+NO_TRAVERSAL_LIMIT = 2**64-1
+
+
+def reset_context():
+  msgq.context = Context()
+
+
+def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
+  with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
+    return msg
+
+
+def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
+  args = {
+    'valid': False,
+    'logMonoTime': int(time.monotonic() * 1e9),
+    **kwargs
+  }
+  dat = log.Event.new_message(**args)
+  if service is not None:
+    if size is None:
+      dat.init(service)
+    else:
+      dat.init(service, size)
+  return dat
+
+
+def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
+  """Receive all message currently available on the queue"""
+  msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
+  return [log_from_bytes(m) for m in msgs]
+
+
+# TODO: print when we drop packets?
+def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+  """Same as drain sock, but only returns latest message. Consider using conflate instead."""
+  dat = None
+
+  while 1:
+    if wait and dat is None:
+      recv = sock.receive()
+    else:
+      recv = sock.receive(non_blocking=True)
+
+    if recv is None:  # Timeout hit
+      break
+
+    dat = recv
+
+  if dat is not None:
+    dat = log_from_bytes(dat)
+
+  return dat
+
+
+def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+  dat = sock.receive()
+  if dat is not None:
+    dat = log_from_bytes(dat)
+  return dat
+
+
+def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
+  dat = sock.receive(non_blocking=True)
+  if dat is not None:
+    dat = log_from_bytes(dat)
+  return dat
+
+
+def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
+  """Keep receiving until we get a message"""
+  while True:
+    dat = sock.receive()
+    if dat is not None:
+      return log_from_bytes(dat)
+
+
+class FrequencyTracker:
+  def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
+    freq = max(min(service_freq, update_freq), 1.)
+    if is_poll:
+      min_freq = max_freq = freq
+    else:
+      max_freq = min(freq, update_freq)
+      if service_freq >= 2 * update_freq:
+        min_freq = update_freq
+      elif update_freq >= 2* service_freq:
+        min_freq = freq
+      else:
+        min_freq = min(freq, freq / 2.)
+
+    self.min_freq = min_freq * 0.8
+    self.max_freq = max_freq * 1.2
+    self.recv_dts: Deque[float] = deque(maxlen=int(10 * freq))
+    self.recv_dts_sum = 0.0
+    self.recent_recv_dts: Deque[float] = deque(maxlen=int(freq))
+    self.recent_recv_dts_sum = 0.0
+    self.prev_time = 0.0
+
+  def record_recv_time(self, cur_time: float) -> None:
+    # TODO: Handle case where cur_time is less than prev_time
+    if self.prev_time > 1e-5:
+      dt = cur_time - self.prev_time
+
+      if len(self.recv_dts) == self.recv_dts.maxlen:
+        self.recv_dts_sum -= self.recv_dts[0]
+      self.recv_dts.append(dt)
+      self.recv_dts_sum += dt
+
+      if len(self.recent_recv_dts) == self.recent_recv_dts.maxlen:
+        self.recent_recv_dts_sum -= self.recent_recv_dts[0]
+      self.recent_recv_dts.append(dt)
+      self.recent_recv_dts_sum += dt
+
+    self.prev_time = cur_time
+
+  @property
+  def valid(self) -> bool:
+    if not self.recv_dts:
+      return False
+
+    avg_freq = len(self.recv_dts) / self.recv_dts_sum
+    if self.min_freq <= avg_freq <= self.max_freq:
+      return True
+
+    avg_freq_recent = len(self.recent_recv_dts) / self.recent_recv_dts_sum
+    return self.min_freq <= avg_freq_recent <= self.max_freq
+
+
+class SubMaster:
+  def __init__(self, services: List[str], poll: Optional[str] = None,
+               ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
+               ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
+    self.frame = -1
+    self.services = services
+    self.seen = {s: False for s in services}
+    self.updated = {s: False for s in services}
+    self.recv_time = {s: 0. for s in services}
+    self.recv_frame = {s: 0 for s in services}
+    self.alive = {s: False for s in services}
+    self.freq_ok = {s: False for s in services}
+    self.sock = {}
+    self.data = {}
+    self.valid = {}
+    self.logMonoTime = {}
+
+    self.freq_tracker: Dict[str, FrequencyTracker] = {}
+    self.poller = Poller()
+    polled_services = set([poll, ] if poll is not None else services)
+    self.non_polled_services = set(services) - polled_services
+
+    self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq
+    self.ignore_alive = [] if ignore_alive is None else ignore_alive
+    self.ignore_valid = [] if ignore_valid is None else ignore_valid
+
+    self.simulation = bool(int(os.getenv("SIMULATION", "0")))
+
+    # if freq and poll aren't specified, assume the max to be conservative
+    assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used."
+    self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services])
+
+    for s in services:
+      p = self.poller if s not in self.non_polled_services else None
+      self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True)
+
+      try:
+        data = new_message(s)
+      except capnp.lib.capnp.KjException:
+        data = new_message(s, 0) # lists
+
+      self.data[s] = getattr(data.as_reader(), s)
+      self.logMonoTime[s] = 0
+      self.valid[s] = False
+      self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
+
+  def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
+    return self.data[s]
+
+  def _check_avg_freq(self, s: str) -> bool:
+    return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive)
+
+  def update(self, timeout: int = 100) -> None:
+    msgs = []
+    for sock in self.poller.poll(timeout):
+      msgs.append(recv_one_or_none(sock))
+
+    # non-blocking receive for non-polled sockets
+    for s in self.non_polled_services:
+      msgs.append(recv_one_or_none(self.sock[s]))
+    self.update_msgs(time.monotonic(), msgs)
+
+  def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
+    self.frame += 1
+    self.updated = dict.fromkeys(self.services, False)
+    for msg in msgs:
+      if msg is None:
+        continue
+
+      s = msg.which()
+      self.seen[s] = True
+      self.updated[s] = True
+
+      self.freq_tracker[s].record_recv_time(cur_time)
+      self.recv_time[s] = cur_time
+      self.recv_frame[s] = self.frame
+      self.data[s] = getattr(msg, s)
+      self.logMonoTime[s] = msg.logMonoTime
+      self.valid[s] = msg.valid
+
+    for s in self.services:
+      if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation:
+        # alive if delay is within 10x the expected frequency
+        self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency)
+        self.freq_ok[s] = self.freq_tracker[s].valid
+      else:
+        self.freq_ok[s] = True
+        self.alive[s] = self.seen[s] if self.simulation else True
+
+  def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
+    return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
+
+  def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
+    return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
+
+  def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
+    return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
+
+  def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
+    return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
+
+
+class PubMaster:
+  def __init__(self, services: List[str]):
+    self.sock = {}
+    for s in services:
+      self.sock[s] = pub_sock(s)
+
+  def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None:
+    if not isinstance(dat, bytes):
+      dat = dat.to_bytes()
+    self.sock[s].send(dat)
+
+  def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
+    for _ in range(int(timeout*(1./dt))):
+      if self.sock[s].all_readers_updated():
+        return True
+      time.sleep(dt)
+    return False
+
+  def all_readers_updated(self, s: str) -> bool:
+    return self.sock[s].all_readers_updated()  # type: ignore

cereal/messaging/bridge.cc

@@ -0,0 +1,71 @@
+#include <cassert>
+
+#include "cereal/messaging/msgq_to_zmq.h"
+#include "cereal/services.h"
+#include "common/util.h"
+
+ExitHandler do_exit;
+
+static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
+  std::vector<std::string> service_list;
+  for (const auto& it : services) {
+    std::string name = it.second.name;
+    bool in_whitelist = whitelist_str.find(name) != std::string::npos;
+    if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
+      continue;
+    }
+    service_list.push_back(name);
+  }
+  return service_list;
+}
+
+void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &ip) {
+  MsgqToZmq bridge;
+  bridge.run(endpoints, ip);
+}
+
+void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
+  auto poller = std::make_unique<ZMQPoller>();
+  auto pub_context = std::make_unique<MSGQContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
+  std::map<SubSocket *, PubSocket *> sub2pub;
+
+  for (auto endpoint : endpoints) {
+    auto pub_sock = new MSGQPubSocket();
+    auto sub_sock = new ZMQSubSocket();
+    pub_sock->connect(pub_context.get(), endpoint);
+    sub_sock->connect(sub_context.get(), endpoint, ip, false);
+
+    poller->registerSocket(sub_sock);
+    sub2pub[sub_sock] = pub_sock;
+  }
+
+  while (!do_exit) {
+    for (auto sub_sock : poller->poll(100)) {
+      std::unique_ptr<Message> msg(sub_sock->receive(true));
+      if (msg) {
+        sub2pub[sub_sock]->sendMessage(msg.get());
+      }
+    }
+  }
+
+  // Clean up allocated sockets
+  for (auto &[sub_sock, pub_sock] : sub2pub) {
+    delete sub_sock;
+    delete pub_sock;
+  }
+}
+
+int main(int argc, char **argv) {
+  bool is_zmq_to_msgq = argc > 2;
+  std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1";
+  std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : "";
+  std::vector<std::string> endpoints = get_services(whitelist_str, is_zmq_to_msgq);
+
+  if (is_zmq_to_msgq) {
+    zmq_to_msgq(endpoints, ip);
+  } else {
+    msgq_to_zmq(endpoints, ip);
+  }
+  return 0;
+}

cereal/messaging/messaging.h

@@ -8,7 +8,7 @@
 
 #include <capnp/serialize.h>
 
-#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "cereal/gen/cpp/log.capnp.h"
 #include "common/timing.h"
 #include "msgq/ipc.h"
 

cereal/messaging/msgq_to_zmq.cc

@@ -1,8 +1,7 @@
-#include "openpilot/cereal/messaging/msgq_to_zmq.h"
+#include "cereal/messaging/msgq_to_zmq.h"
 
 #include <cassert>
 
-#include "openpilot/cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -22,14 +21,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
-  msgq_context = std::make_unique<Context>();
+  zmq_context = std::make_unique<ZMQContext>();
+  msgq_context = std::make_unique<MSGQContext>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -49,7 +48,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -72,7 +71,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -109,8 +108,7 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -130,7 +128,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <condition_variable>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#define private public
+#include "msgq/impl_msgq.h"
+#include "msgq/impl_zmq.h"
+
+class MsgqToZmq {
+public:
+  MsgqToZmq() {}
+  void run(const std::vector<std::string> &endpoints, const std::string &ip);
+
+protected:
+  void registerSockets();
+  void zmqMonitorThread();
+
+  struct SocketPair {
+    std::string endpoint;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
+    std::unique_ptr<MSGQSubSocket> sub_sock;
+    int connected_clients = 0;
+  };
+
+  std::unique_ptr<MSGQContext> msgq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
+  std::mutex mutex;
+  std::condition_variable cv;
+  std::unique_ptr<MSGQPoller> msgq_poller;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
+  std::vector<SocketPair> socket_pairs;
+};

cereal/messaging/socketmaster.cc

@@ -3,8 +3,8 @@
 #include <string>
 #include <mutex>
 
-#include "openpilot/cereal/services.h"
-#include "openpilot/cereal/messaging/messaging.h"
+#include "cereal/services.h"
+#include "cereal/messaging/messaging.h"
 
 const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
 
@@ -33,7 +33,7 @@ MessageContext message_context;
 struct SubMaster::SubMessage {
   std::string name;
   SubSocket *socket = nullptr;
-  float freq = 0.0f;
+  int freq = 0;
   bool updated = false, alive = false, valid = false, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,8 +187,7 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    service serv = services.at(std::string(name));
-    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
+    PubSocket *socket = PubSocket::create(message_context.context(), name);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/services.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+from typing import Optional
+
+
+class Service:
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
+    self.should_log = should_log
+    self.frequency = frequency
+    self.decimation = decimation
+
+
+_services: dict[str, tuple] = {
+  # service: (should_log, frequency, qlog decimation (optional))
+  # note: the "EncodeIdx" packets will still be in the log
+  "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
+  "accelerometer": (True, 104., 104),
+  "accelerometer2": (True, 100., 100),
+  "magnetometer": (True, 25.),
+  "lightSensor": (True, 100., 100),
+  "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
+  "gpsNMEA": (True, 9.),
+  "deviceState": (True, 2., 1),
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10),
+  "selfdriveState": (True, 100., 10),
+  "pandaStates": (True, 10., 1),
+  "peripheralState": (True, 2., 1),
+  "radarState": (True, 20., 5),
+  "roadEncodeIdx": (False, 20., 1),
+  "liveTracks": (True, 20.),
+  "sendcan": (True, 100., 139),
+  "logMessage": (True, 0.),
+  "errorLogMessage": (True, 0., 1),
+  "liveCalibration": (True, 4., 4),
+  "liveTorqueParameters": (True, 4., 1),
+  "androidLog": (True, 0.),
+  "carState": (True, 100., 10),
+  "carControl": (True, 100., 10),
+  "carOutput": (True, 100., 10),
+  "longitudinalPlan": (True, 20., 10),
+  "driverAssistance": (True, 20., 20),
+  "procLog": (True, 0.5, 15),
+  "gpsLocationExternal": (True, 10., 10),
+  "gpsLocation": (True, 1., 1),
+  "ubloxGnss": (True, 10.),
+  "qcomGnss": (True, 2.),
+  "gnssMeasurements": (True, 10., 10),
+  "clocks": (True, 0.1, 1),
+  "ubloxRaw": (True, 20.),
+  "livePose": (True, 20., 4),
+  "liveParameters": (True, 20., 5),
+  "cameraOdometry": (True, 20., 10),
+  "thumbnail": (True, 0.2, 1),
+  "onroadEvents": (True, 1., 1),
+  "carParams": (True, 0.02, 1),
+  "roadCameraState": (True, 20., 20),
+  "driverCameraState": (True, 20., 20),
+  "driverEncodeIdx": (False, 20., 1),
+  "driverStateV2": (True, 20., 10),
+  "driverMonitoringState": (True, 20., 10),
+  "wideRoadEncodeIdx": (False, 20., 1),
+  "wideRoadCameraState": (True, 20., 20),
+  "drivingModelData": (True, 20., 10),
+  "modelV2": (True, 20.),
+  "managerState": (True, 2., 1),
+  "uploaderState": (True, 0., 1),
+  "navInstruction": (True, 1., 10),
+  "navRoute": (True, 0.),
+  "navThumbnail": (True, 0.),
+  "qRoadEncodeIdx": (False, 20.),
+  "userFlag": (True, 0., 1),
+  "microphone": (True, 10., 10),
+
+  # sunnypilot
+  "selfdriveStateSP": (True, 100., 10),
+
+  # debug
+  "uiDebug": (True, 0., 1),
+  "testJoystick": (True, 0.),
+  "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
+  "livestreamWideRoadEncodeIdx": (False, 20.),
+  "livestreamRoadEncodeIdx": (False, 20.),
+  "livestreamDriverEncodeIdx": (False, 20.),
+  "livestreamWideRoadEncodeData": (False, 20.),
+  "livestreamRoadEncodeData": (False, 20.),
+  "livestreamDriverEncodeData": (False, 20.),
+  "customReservedRawData0": (True, 0.),
+  "customReservedRawData1": (True, 0.),
+  "customReservedRawData2": (True, 0.),
+}
+SERVICE_LIST = {name: Service(*vals) for
+                idx, (name, vals) in enumerate(_services.items())}
+
+
+def build_header():
+  h = ""
+  h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n"
+  h += "#ifndef __SERVICES_H\n"
+  h += "#define __SERVICES_H\n"
+
+  h += "#include <map>\n"
+  h += "#include <string>\n"
+
+  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
+  h += "static std::map<std::string, service> services = {\n"
+  for k, v in SERVICE_LIST.items():
+    should_log = "true" if v.should_log else "false"
+    decimation = -1 if v.decimation is None else v.decimation
+    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation)
+  h += "};\n"
+
+  h += "#endif\n"
+  return h
+
+
+if __name__ == "__main__":
+  print(build_header())

codecov.yml

@@ -0,0 +1,13 @@
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch: off
+
+ignore:
+  - "**/test_*.py"
+  - "selfdrive/test/**"
+  - "system/version.py" # codecov changes depending on if we are in a branch or not
+  - "tools"

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -0,0 +1,39 @@
+Import('env', 'envCython', 'arch')
+
+common_libs = [
+  'params.cc',
+  'swaglog.cc',
+  'util.cc',
+  'i2c.cc',
+  'watchdog.cc',
+  'ratekeeper.cc'
+]
+
+if arch != "Darwin":
+  common_libs.append('gpio.cc')
+
+_common = env.Library('common', common_libs, LIBS="json11")
+
+files = [
+  'clutil.cc',
+]
+
+_gpucommon = env.Library('gpucommon', files)
+Export('_common', '_gpucommon')
+
+if GetOption('extras'):
+  env.Program('tests/test_common',
+              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+
+# Cython bindings
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
+
+Export('common_python')

common/api/__init__.py

@@ -0,0 +1,46 @@
+import jwt
+import os
+import requests
+from datetime import datetime, timedelta, UTC
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.version import get_version
+
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
+
+class Api:
+  def __init__(self, dongle_id):
+    self.dongle_id = dongle_id
+    with open(Paths.persist_root()+'/comma/id_rsa') as f:
+      self.private_key = f.read()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def get_token(self, expiry_hours=1):
+    now = datetime.now(UTC).replace(tzinfo=None)
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours)
+    }
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+
+
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+  headers = {}
+  if access_token is not None:
+    headers['Authorization'] = "JWT " + access_token
+
+  headers['User-Agent'] = "openpilot-" + get_version()
+
+  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)

common/basedir.py

@@ -0,0 +1,4 @@
+import os
+
+
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))

common/clutil.cc

@@ -0,0 +1,200 @@
+#include "common/clutil.h"
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+
+#include "common/util.h"
+#include "common/swaglog.h"
+
+namespace {  // helper functions
+
+template <typename Func, typename Id, typename Name>
+std::string get_info(Func get_info_func, Id id, Name param_name) {
+  size_t size = 0;
+  CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
+  std::string info(size, '\0');
+  CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
+  return info;
+}
+inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
+inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
+
+void cl_print_info(cl_platform_id platform, cl_device_id device) {
+  size_t work_group_size = 0;
+  cl_device_type device_type = 0;
+  clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
+  clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
+  const char *type_str = "Other...";
+  switch (device_type) {
+    case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
+    case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
+    case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
+  }
+
+  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
+  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
+  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
+  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
+  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
+  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
+  LOGD("max work group size: %zu", work_group_size);
+  LOGD("type = %d, %s", (int)device_type, type_str);
+}
+
+void cl_print_build_errors(cl_program program, cl_device_id device) {
+  cl_build_status status;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
+  size_t log_size;
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
+  std::string log(log_size, '\0');
+  clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
+
+  LOGE("build failed; status=%d, log: %s", status, log.c_str());
+}
+
+}  // namespace
+
+cl_device_id cl_get_device_id(cl_device_type device_type) {
+  cl_uint num_platforms = 0;
+  CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
+  std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
+  CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
+
+  for (size_t i = 0; i < num_platforms; ++i) {
+    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
+
+    // Get first device
+    if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
+      cl_print_info(platform_ids[i], device_id);
+      return device_id;
+    }
+  }
+  LOGE("No valid openCL platform found");
+  assert(0);
+  return nullptr;
+}
+
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
+  return cl_program_from_source(ctx, device_id, util::read_file(path), args);
+}
+
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
+  const char *csrc = src.c_str();
+  cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
+  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
+    cl_print_build_errors(prg, device_id);
+    assert(0);
+  }
+  return prg;
+}
+
+cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args) {
+  cl_program prg = CL_CHECK_ERR(clCreateProgramWithBinary(ctx, 1, &device_id, &length, &binary, NULL, &err));
+  if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
+    cl_print_build_errors(prg, device_id);
+    assert(0);
+  }
+  return prg;
+}
+
+// Given a cl code and return a string representation
+#define CL_ERR_TO_STR(err) case err: return #err
+const char* cl_get_error_string(int err) {
+  switch (err) {
+    CL_ERR_TO_STR(CL_SUCCESS);
+    CL_ERR_TO_STR(CL_DEVICE_NOT_FOUND);
+    CL_ERR_TO_STR(CL_DEVICE_NOT_AVAILABLE);
+    CL_ERR_TO_STR(CL_COMPILER_NOT_AVAILABLE);
+    CL_ERR_TO_STR(CL_MEM_OBJECT_ALLOCATION_FAILURE);
+    CL_ERR_TO_STR(CL_OUT_OF_RESOURCES);
+    CL_ERR_TO_STR(CL_OUT_OF_HOST_MEMORY);
+    CL_ERR_TO_STR(CL_PROFILING_INFO_NOT_AVAILABLE);
+    CL_ERR_TO_STR(CL_MEM_COPY_OVERLAP);
+    CL_ERR_TO_STR(CL_IMAGE_FORMAT_MISMATCH);
+    CL_ERR_TO_STR(CL_IMAGE_FORMAT_NOT_SUPPORTED);
+    CL_ERR_TO_STR(CL_MAP_FAILURE);
+    CL_ERR_TO_STR(CL_MISALIGNED_SUB_BUFFER_OFFSET);
+    CL_ERR_TO_STR(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST);
+    CL_ERR_TO_STR(CL_COMPILE_PROGRAM_FAILURE);
+    CL_ERR_TO_STR(CL_LINKER_NOT_AVAILABLE);
+    CL_ERR_TO_STR(CL_LINK_PROGRAM_FAILURE);
+    CL_ERR_TO_STR(CL_DEVICE_PARTITION_FAILED);
+    CL_ERR_TO_STR(CL_KERNEL_ARG_INFO_NOT_AVAILABLE);
+    CL_ERR_TO_STR(CL_INVALID_VALUE);
+    CL_ERR_TO_STR(CL_INVALID_DEVICE_TYPE);
+    CL_ERR_TO_STR(CL_INVALID_PLATFORM);
+    CL_ERR_TO_STR(CL_INVALID_DEVICE);
+    CL_ERR_TO_STR(CL_INVALID_CONTEXT);
+    CL_ERR_TO_STR(CL_INVALID_QUEUE_PROPERTIES);
+    CL_ERR_TO_STR(CL_INVALID_COMMAND_QUEUE);
+    CL_ERR_TO_STR(CL_INVALID_HOST_PTR);
+    CL_ERR_TO_STR(CL_INVALID_MEM_OBJECT);
+    CL_ERR_TO_STR(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR);
+    CL_ERR_TO_STR(CL_INVALID_IMAGE_SIZE);
+    CL_ERR_TO_STR(CL_INVALID_SAMPLER);
+    CL_ERR_TO_STR(CL_INVALID_BINARY);
+    CL_ERR_TO_STR(CL_INVALID_BUILD_OPTIONS);
+    CL_ERR_TO_STR(CL_INVALID_PROGRAM);
+    CL_ERR_TO_STR(CL_INVALID_PROGRAM_EXECUTABLE);
+    CL_ERR_TO_STR(CL_INVALID_KERNEL_NAME);
+    CL_ERR_TO_STR(CL_INVALID_KERNEL_DEFINITION);
+    CL_ERR_TO_STR(CL_INVALID_KERNEL);
+    CL_ERR_TO_STR(CL_INVALID_ARG_INDEX);
+    CL_ERR_TO_STR(CL_INVALID_ARG_VALUE);
+    CL_ERR_TO_STR(CL_INVALID_ARG_SIZE);
+    CL_ERR_TO_STR(CL_INVALID_KERNEL_ARGS);
+    CL_ERR_TO_STR(CL_INVALID_WORK_DIMENSION);
+    CL_ERR_TO_STR(CL_INVALID_WORK_GROUP_SIZE);
+    CL_ERR_TO_STR(CL_INVALID_WORK_ITEM_SIZE);
+    CL_ERR_TO_STR(CL_INVALID_GLOBAL_OFFSET);
+    CL_ERR_TO_STR(CL_INVALID_EVENT_WAIT_LIST);
+    CL_ERR_TO_STR(CL_INVALID_EVENT);
+    CL_ERR_TO_STR(CL_INVALID_OPERATION);
+    CL_ERR_TO_STR(CL_INVALID_GL_OBJECT);
+    CL_ERR_TO_STR(CL_INVALID_BUFFER_SIZE);
+    CL_ERR_TO_STR(CL_INVALID_MIP_LEVEL);
+    CL_ERR_TO_STR(CL_INVALID_GLOBAL_WORK_SIZE);
+    CL_ERR_TO_STR(CL_INVALID_PROPERTY);
+    CL_ERR_TO_STR(CL_INVALID_IMAGE_DESCRIPTOR);
+    CL_ERR_TO_STR(CL_INVALID_COMPILER_OPTIONS);
+    CL_ERR_TO_STR(CL_INVALID_LINKER_OPTIONS);
+    CL_ERR_TO_STR(CL_INVALID_DEVICE_PARTITION_COUNT);
+    case -69: return "CL_INVALID_PIPE_SIZE";
+    case -70: return "CL_INVALID_DEVICE_QUEUE";
+    case -71: return "CL_INVALID_SPEC_ID";
+    case -72: return "CL_MAX_SIZE_RESTRICTION_EXCEEDED";
+    case -1002: return "CL_INVALID_D3D10_DEVICE_KHR";
+    case -1003: return "CL_INVALID_D3D10_RESOURCE_KHR";
+    case -1004: return "CL_D3D10_RESOURCE_ALREADY_ACQUIRED_KHR";
+    case -1005: return "CL_D3D10_RESOURCE_NOT_ACQUIRED_KHR";
+    case -1006: return "CL_INVALID_D3D11_DEVICE_KHR";
+    case -1007: return "CL_INVALID_D3D11_RESOURCE_KHR";
+    case -1008: return "CL_D3D11_RESOURCE_ALREADY_ACQUIRED_KHR";
+    case -1009: return "CL_D3D11_RESOURCE_NOT_ACQUIRED_KHR";
+    case -1010: return "CL_INVALID_DX9_MEDIA_ADAPTER_KHR";
+    case -1011: return "CL_INVALID_DX9_MEDIA_SURFACE_KHR";
+    case -1012: return "CL_DX9_MEDIA_SURFACE_ALREADY_ACQUIRED_KHR";
+    case -1013: return "CL_DX9_MEDIA_SURFACE_NOT_ACQUIRED_KHR";
+    case -1093: return "CL_INVALID_EGL_OBJECT_KHR";
+    case -1092: return "CL_EGL_RESOURCE_NOT_ACQUIRED_KHR";
+    case -1001: return "CL_PLATFORM_NOT_FOUND_KHR";
+    case -1057: return "CL_DEVICE_PARTITION_FAILED_EXT";
+    case -1058: return "CL_INVALID_PARTITION_COUNT_EXT";
+    case -1059: return "CL_INVALID_PARTITION_NAME_EXT";
+    case -1094: return "CL_INVALID_ACCELERATOR_INTEL";
+    case -1095: return "CL_INVALID_ACCELERATOR_TYPE_INTEL";
+    case -1096: return "CL_INVALID_ACCELERATOR_DESCRIPTOR_INTEL";
+    case -1097: return "CL_ACCELERATOR_TYPE_NOT_SUPPORTED_INTEL";
+    case -1000: return "CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR";
+    case -1098: return "CL_INVALID_VA_API_MEDIA_ADAPTER_INTEL";
+    case -1099: return "CL_INVALID_VA_API_MEDIA_SURFACE_INTEL";
+    case -1100: return "CL_VA_API_MEDIA_SURFACE_ALREADY_ACQUIRED_INTEL";
+    case -1101: return "CL_VA_API_MEDIA_SURFACE_NOT_ACQUIRED_INTEL";
+    default: return "CL_UNKNOWN_ERROR";
+  }
+}

common/clutil.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#include <string>
+
+#define CL_CHECK(_expr)          \
+  do {                           \
+    assert(CL_SUCCESS == (_expr)); \
+  } while (0)
+
+#define CL_CHECK_ERR(_expr)           \
+  ({                                  \
+    cl_int err = CL_INVALID_VALUE;    \
+    __typeof__(_expr) _ret = _expr;   \
+    assert(_ret&& err == CL_SUCCESS); \
+    _ret;                             \
+  })
+
+cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
+cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
+cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
+cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);
+const char* cl_get_error_string(int err);

common/conversions.py

@@ -0,0 +1,19 @@
+import numpy as np
+
+class Conversions:
+  # Speed
+  MPH_TO_KPH = 1.609344
+  KPH_TO_MPH = 1. / MPH_TO_KPH
+  MS_TO_KPH = 3.6
+  KPH_TO_MS = 1. / MS_TO_KPH
+  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
+  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
+  MS_TO_KNOTS = 1.9438
+  KNOTS_TO_MS = 1. / MS_TO_KNOTS
+
+  # Angle
+  DEG_TO_RAD = np.pi / 180.
+  RAD_TO_DEG = 1. / DEG_TO_RAD
+
+  # Mass
+  LB_TO_KG = 0.453592

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/ffi_wrapper.py

@@ -0,0 +1,8 @@
+import platform
+
+
+def suffix():
+  if platform.system() == "Darwin":
+    return ".dylib"
+  else:
+    return ".so"

common/file_helpers.py

@@ -0,0 +1,37 @@
+import os
+import tempfile
+import contextlib
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+                        overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)

common/filter_simple.py

@@ -0,0 +1,18 @@
+class FirstOrderFilter:
+  # first order filter
+  def __init__(self, x0, rc, dt, initialized=True):
+    self.x = x0
+    self.dt = dt
+    self.update_alpha(rc)
+    self.initialized = initialized
+
+  def update_alpha(self, rc):
+    self.alpha = self.dt / (rc + self.dt)
+
+  def update(self, x):
+    if self.initialized:
+      self.x = (1. - self.alpha) * self.x + self.alpha * x
+    else:
+      self.initialized = True
+      self.x = x
+    return self.x

common/git.py

@@ -0,0 +1,42 @@
+from functools import cache
+import subprocess
+from openpilot.common.run import run_cmd, run_cmd_default
+
+
+@cache
+def get_commit(cwd: str = None, branch: str = "HEAD") -> str:
+  return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
+
+
+@cache
+def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str:
+  return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
+
+
+@cache
+def get_short_branch(cwd: str = None) -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
+
+
+@cache
+def get_branch(cwd: str = None) -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
+
+
+@cache
+def get_origin(cwd: str = None) -> str:
+  try:
+    local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
+    tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
+    return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
+  except subprocess.CalledProcessError:  # Not on a branch, fallback
+    return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
+
+
+@cache
+def get_normalized_origin(cwd: str = None) -> str:
+  return get_origin(cwd) \
+    .replace("git@", "", 1) \
+    .replace(".git", "", 1) \
+    .replace("https://", "", 1) \
+    .replace(":", "/", 1)

common/gpio.cc

@@ -0,0 +1,84 @@
+#include "common/gpio.h"
+
+#include <string>
+
+#ifdef __APPLE__
+int gpio_init(int pin_nr, bool output) {
+  return 0;
+}
+
+int gpio_set(int pin_nr, bool high) {
+  return 0;
+}
+
+int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
+  return 0;
+}
+
+#else
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <cstring>
+#include <linux/gpio.h>
+#include <sys/ioctl.h>
+
+#include "common/util.h"
+#include "common/swaglog.h"
+
+int gpio_init(int pin_nr, bool output) {
+  char pin_dir_path[50];
+  int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
+                           "/sys/class/gpio/gpio%d/direction", pin_nr);
+  if (pin_dir_path_len <= 0) {
+    return -1;
+  }
+  const char *value = output ? "out" : "in";
+  return util::write_file(pin_dir_path, (void*)value, strlen(value));
+}
+
+int gpio_set(int pin_nr, bool high) {
+  char pin_val_path[50];
+  int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
+                           "/sys/class/gpio/gpio%d/value", pin_nr);
+  if (pin_val_path_len <= 0) {
+    return -1;
+  }
+  return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);
+}
+
+int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
+
+  // Assumed that all interrupt pins are unexported and rights are given to
+  // read from gpiochip0.
+  std::string gpiochip_path = "/dev/gpiochip" + std::to_string(gpiochiop_id);
+  int fd = open(gpiochip_path.c_str(), O_RDONLY);
+  if (fd < 0) {
+    LOGE("Error opening gpiochip0 fd");
+    return -1;
+  }
+
+  // Setup event
+  struct gpioevent_request rq;
+  rq.lineoffset = pin_nr;
+  rq.handleflags = GPIOHANDLE_REQUEST_INPUT;
+
+  /* Requesting both edges as the data ready pulse from the lsm6ds sensor is
+     very short(75us) and is mostly detected as falling edge instead of rising.
+     So if it is detected as rising the following falling edge is skipped. */
+  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
+
+  strncpy(rq.consumer_label, consumer_label, std::size(rq.consumer_label) - 1);
+  int ret = util::safe_ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
+  if (ret == -1) {
+    LOGE("Unable to get line event from ioctl : %s", strerror(errno));
+    close(fd);
+    return -1;
+  }
+
+  close(fd);
+  return rq.fd;
+}
+
+#endif

common/gpio.h

@@ -0,0 +1,33 @@
+#pragma once
+
+// Pin definitions
+#ifdef QCOM2
+  #define GPIO_HUB_RST_N        30
+  #define GPIO_UBLOX_RST_N      32
+  #define GPIO_UBLOX_SAFEBOOT_N 33
+  #define GPIO_GNSS_PWR_EN      34 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
+  #define GPIO_STM_RST_N        124
+  #define GPIO_STM_BOOT0        134
+  #define GPIO_BMX_ACCEL_INT    21
+  #define GPIO_BMX_GYRO_INT     23
+  #define GPIO_BMX_MAGN_INT     87
+  #define GPIO_LSM_INT          84
+  #define GPIOCHIP_INT          0
+#else
+  #define GPIO_HUB_RST_N        0
+  #define GPIO_UBLOX_RST_N      0
+  #define GPIO_UBLOX_SAFEBOOT_N 0
+  #define GPIO_GNSS_PWR_EN      0 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
+  #define GPIO_STM_RST_N        0
+  #define GPIO_STM_BOOT0        0
+  #define GPIO_BMX_ACCEL_INT    0
+  #define GPIO_BMX_GYRO_INT     0
+  #define GPIO_BMX_MAGN_INT     0
+  #define GPIO_LSM_INT          0
+  #define GPIOCHIP_INT          0
+#endif
+
+int gpio_init(int pin_nr, bool output);
+int gpio_set(int pin_nr, bool high);
+
+int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr);

common/gpio.py

@@ -0,0 +1,54 @@
+import os
+from functools import cache
+
+def gpio_init(pin: int, output: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
+      f.write(b"out" if output else b"in")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} direction: {e}")
+
+def gpio_set(pin: int, high: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
+      f.write(b"1" if high else b"0")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+def gpio_read(pin: int) -> bool | None:
+  val = None
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
+      val = bool(int(f.read().strip()))
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+  return val
+
+def gpio_export(pin: int) -> None:
+  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
+    return
+
+  try:
+    with open("/sys/class/gpio/export", 'w') as f:
+      f.write(str(pin))
+  except Exception:
+    print(f"Failed to export gpio {pin}")
+
+@cache
+def get_irq_action(irq: int) -> list[str]:
+  try:
+    with open(f"/sys/kernel/irq/{irq}/actions") as f:
+      actions = f.read().strip().split(',')
+      return actions
+  except FileNotFoundError:
+    return []
+
+def get_irqs_for_action(action: str) -> list[str]:
+  ret = []
+  with open("/proc/interrupts") as f:
+    for l in f.readlines():
+      irq = l.split(':')[0].strip()
+      if irq.isdigit() and action in get_irq_action(irq):
+        ret.append(irq)
+  return ret

common/i2c.cc

@@ -0,0 +1,92 @@
+#include "common/i2c.h"
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+#include <cassert>
+#include <cstdio>
+#include <stdexcept>
+
+#include "common/swaglog.h"
+#include "common/util.h"
+
+#define UNUSED(x) (void)(x)
+
+#ifdef QCOM2
+// TODO: decide if we want to install libi2c-dev everywhere
+extern "C" {
+  #include <linux/i2c-dev.h>
+  #include <i2c/smbus.h>
+}
+
+I2CBus::I2CBus(uint8_t bus_id) {
+  char bus_name[20];
+  snprintf(bus_name, 20, "/dev/i2c-%d", bus_id);
+
+  i2c_fd = HANDLE_EINTR(open(bus_name, O_RDWR));
+  if (i2c_fd < 0) {
+    throw std::runtime_error("Failed to open I2C bus");
+  }
+}
+
+I2CBus::~I2CBus() {
+  if (i2c_fd >= 0) {
+    close(i2c_fd);
+  }
+}
+
+int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
+  std::lock_guard lk(m);
+
+  int ret = 0;
+
+  ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
+  if (ret < 0) { goto fail; }
+
+  ret = i2c_smbus_read_i2c_block_data(i2c_fd, register_address, len, buffer);
+  if ((ret < 0) || (ret != len)) { goto fail; }
+
+fail:
+  return ret;
+}
+
+int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
+  std::lock_guard lk(m);
+
+  int ret = 0;
+
+  ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
+  if (ret < 0) { goto fail; }
+
+  ret = i2c_smbus_write_byte_data(i2c_fd, register_address, data);
+  if (ret < 0) { goto fail; }
+
+fail:
+  return ret;
+}
+
+#else
+
+I2CBus::I2CBus(uint8_t bus_id) {
+  UNUSED(bus_id);
+  i2c_fd = -1;
+}
+
+I2CBus::~I2CBus() {}
+
+int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
+  UNUSED(device_address);
+  UNUSED(register_address);
+  UNUSED(buffer);
+  UNUSED(len);
+  return -1;
+}
+
+int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
+  UNUSED(device_address);
+  UNUSED(register_address);
+  UNUSED(data);
+  return -1;
+}
+#endif

common/i2c.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+#include <mutex>
+
+#include <sys/types.h>
+
+class I2CBus {
+  private:
+    int i2c_fd;
+    std::mutex m;
+
+  public:
+    I2CBus(uint8_t bus_id);
+    ~I2CBus();
+
+    int read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len);
+    int set_register(uint8_t device_address, uint register_address, uint8_t data);
+};

common/logging_extra.py

@@ -8,7 +8,6 @@ import uuid
 import socket
 import logging
 import traceback
-import numpy as np
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
@@ -16,8 +15,6 @@ from contextlib import contextmanager
 LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
 
 def json_handler(obj):
-  if isinstance(obj, np.bool_):
-    return bool(obj)
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
   return repr(obj)
@@ -199,6 +196,7 @@ class SwagLogger(logging.Logger):
       co = f.f_code
       filename = os.path.normcase(co.co_filename)
 
+      # TODO: is this pylint exception correct?
       if filename == _srcfile:
         f = f.f_back
         continue

common/mat.h

@@ -0,0 +1,85 @@
+#pragma once
+
+typedef struct vec3 {
+  float v[3];
+} vec3;
+
+typedef struct vec4 {
+  float v[4];
+} vec4;
+
+typedef struct mat3 {
+  float v[3*3];
+} mat3;
+
+typedef struct mat4 {
+  float v[4*4];
+} mat4;
+
+static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
+  mat3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      float v = 0.0;
+      for (int k=0; k<3; k++) {
+        v += a.v[r*3+k] * b.v[k*3+c];
+      }
+      ret.v[r*3+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
+  vec3 ret = {{0.0}};
+  for (int r=0; r<3; r++) {
+    for (int c=0; c<3; c++) {
+      ret.v[r] += a.v[r*3+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+static inline mat4 matmul(const mat4 &a, const mat4 &b) {
+  mat4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      float v = 0.0;
+      for (int k=0; k<4; k++) {
+        v += a.v[r*4+k] * b.v[k*4+c];
+      }
+      ret.v[r*4+c] = v;
+    }
+  }
+  return ret;
+}
+
+static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
+  vec4 ret = {{0.0}};
+  for (int r=0; r<4; r++) {
+    for (int c=0; c<4; c++) {
+      ret.v[r] += a.v[r*4+c] * b.v[c];
+    }
+  }
+  return ret;
+}
+
+// scales the input and output space of a transformation matrix
+// that assumes pixel-center origin.
+static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
+  // in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
+
+  mat3 transform_out = {{
+    1.0f/s, 0.0f, 0.5f,
+    0.0f, 1.0f/s, 0.5f,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  mat3 transform_in = {{
+    s,  0.0f, -0.5f*s,
+    0.0f, s, -0.5f*s,
+    0.0f, 0.0f, 1.0f,
+  }};
+
+  return matmul3(transform_in, matmul3(in, transform_out));
+}

common/mock/__init__.py

@@ -0,0 +1,50 @@
+"""
+Utilities for generating mock messages for testing.
+example in common/tests/test_mock.py
+"""
+
+
+import functools
+import threading
+from cereal.messaging import PubMaster
+from cereal.services import SERVICE_LIST
+from openpilot.common.mock.generators import generate_livePose
+from openpilot.common.realtime import Ratekeeper
+
+
+MOCK_GENERATOR = {
+  "livePose": generate_livePose
+}
+
+
+def generate_messages_loop(services: list[str], done: threading.Event):
+  pm = PubMaster(services)
+  rk = Ratekeeper(100)
+  i = 0
+  while not done.is_set():
+    for s in services:
+      should_send = i % (100/SERVICE_LIST[s].frequency) == 0
+      if should_send:
+        message = MOCK_GENERATOR[s]()
+        pm.send(s, message)
+    i += 1
+    rk.keep_time()
+
+
+def mock_messages(services: list[str] | str):
+  if isinstance(services, str):
+    services = [services]
+
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      done = threading.Event()
+      t = threading.Thread(target=generate_messages_loop, args=(services, done))
+      t.start()
+      try:
+        return func(*args, **kwargs)
+      finally:
+        done.set()
+        t.join()
+    return wrapper
+  return decorator

common/mock/generators.py

@@ -1,4 +1,4 @@
-from openpilot.cereal import messaging
+from cereal import messaging
 
 
 def generate_livePose():

common/numpy_fast.py

@@ -0,0 +1,19 @@
+def clip(x, lo, hi):
+  return max(lo, min(hi, x))
+
+def interp(x, xp, fp):
+  N = len(xp)
+
+  def get_interp(xv):
+    hi = 0
+    while hi < N and xv > xp[hi]:
+      hi += 1
+    low = hi - 1
+    return fp[-1] if hi == N and xv > xp[low] else (
+      fp[0] if hi == 0 else
+      (xv - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]) + fp[low])
+
+  return [get_interp(v) for v in x] if hasattr(x, '__iter__') else get_interp(x)
+
+def mean(x):
+  return sum(x) / len(x)

common/params.cc

@@ -0,0 +1,359 @@
+#include "common/params.h"
+
+#include <dirent.h>
+#include <sys/file.h>
+
+#include <algorithm>
+#include <cassert>
+#include <csignal>
+#include <unordered_map>
+
+#include "common/queue.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+
+namespace {
+
+volatile sig_atomic_t params_do_exit = 0;
+void params_sig_handler(int signal) {
+  params_do_exit = 1;
+}
+
+int fsync_dir(const std::string &path) {
+  int result = -1;
+  int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
+  if (fd >= 0) {
+    result = HANDLE_EINTR(fsync(fd));
+    HANDLE_EINTR(close(fd));
+  }
+  return result;
+}
+
+bool create_params_path(const std::string &param_path, const std::string &key_path) {
+  // Make sure params path exists
+  if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
+    return false;
+  }
+
+  // See if the symlink exists, otherwise create it
+  if (!util::file_exists(key_path)) {
+    // 1) Create temp folder
+    // 2) Symlink it to temp link
+    // 3) Move symlink to <params>/d
+
+    std::string tmp_path = param_path + "/.tmp_XXXXXX";
+    // this should be OK since mkdtemp just replaces characters in place
+    char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
+    if (tmp_dir == NULL) {
+      return false;
+    }
+
+    std::string link_path = std::string(tmp_dir) + ".link";
+    if (symlink(tmp_dir, link_path.c_str()) != 0) {
+      return false;
+    }
+
+    // don't return false if it has been created by other
+    if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
+  std::string params_path = path.empty() ? Path::params() : path;
+  if (!create_params_path(params_path, params_path + prefix)) {
+    throw std::runtime_error(util::string_format(
+        "Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
+        errno, params_path.c_str(), prefix.c_str()));
+  }
+  return params_path;
+}
+
+class FileLock {
+public:
+  FileLock(const std::string &fn) {
+    fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
+    if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
+      LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
+    }
+  }
+  ~FileLock() { close(fd_); }
+
+private:
+  int fd_ = -1;
+};
+
+std::unordered_map<std::string, uint32_t> keys = {
+    {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
+    {"AlwaysOnDM", PERSISTENT},
+    {"ApiCache_Device", PERSISTENT},
+    {"AssistNowToken", PERSISTENT},
+    {"AthenadPid", PERSISTENT},
+    {"AthenadUploadQueue", PERSISTENT},
+    {"AthenadRecentlyViewedRoutes", PERSISTENT},
+    {"BootCount", PERSISTENT},
+    {"CalibrationParams", PERSISTENT},
+    {"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
+    {"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
+    {"CarBatteryCapacity", PERSISTENT},
+    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsPersistent", PERSISTENT},
+    {"CarParamsPrevRoute", PERSISTENT},
+    {"CompletedTrainingVersion", PERSISTENT},
+    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CurrentBootlog", PERSISTENT},
+    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisablePowerDown", PERSISTENT},
+    {"DisableUpdates", PERSISTENT},
+    {"DisengageOnAccelerator", PERSISTENT},
+    {"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
+    {"DongleId", PERSISTENT},
+    {"DoReboot", CLEAR_ON_MANAGER_START},
+    {"DoShutdown", CLEAR_ON_MANAGER_START},
+    {"DoUninstall", CLEAR_ON_MANAGER_START},
+    {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
+    {"ExperimentalMode", PERSISTENT},
+    {"ExperimentalModeConfirmed", PERSISTENT},
+    {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ForcePowerDown", PERSISTENT},
+    {"GitBranch", PERSISTENT},
+    {"GitCommit", PERSISTENT},
+    {"GitCommitDate", PERSISTENT},
+    {"GitDiff", PERSISTENT},
+    {"GithubSshKeys", PERSISTENT},
+    {"GithubUsername", PERSISTENT},
+    {"GitRemote", PERSISTENT},
+    {"GsmApn", PERSISTENT},
+    {"GsmMetered", PERSISTENT},
+    {"GsmRoaming", PERSISTENT},
+    {"HardwareSerial", PERSISTENT},
+    {"HasAcceptedTerms", PERSISTENT},
+    {"IMEI", PERSISTENT},
+    {"InstallDate", PERSISTENT},
+    {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
+    {"IsEngaged", PERSISTENT},
+    {"IsLdwEnabled", PERSISTENT},
+    {"IsMetric", PERSISTENT},
+    {"IsOffroad", CLEAR_ON_MANAGER_START},
+    {"IsOnroad", PERSISTENT},
+    {"IsRhdDetected", PERSISTENT},
+    {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
+    {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
+    {"IsTestedBranch", CLEAR_ON_MANAGER_START},
+    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LanguageSetting", PERSISTENT},
+    {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
+    {"LastGPSPosition", PERSISTENT},
+    {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
+    {"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
+    {"LastUpdateException", CLEAR_ON_MANAGER_START},
+    {"LastUpdateTime", PERSISTENT},
+    {"LiveParameters", PERSISTENT},
+    {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
+    {"LocationFilterInitialState", PERSISTENT},
+    {"LongitudinalManeuverMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"LongitudinalPersonality", PERSISTENT},
+    {"NetworkMetered", PERSISTENT},
+    {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
+    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
+    {"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
+    {"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
+    {"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
+    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
+    {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
+    {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
+    {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
+    {"OpenpilotEnabledToggle", PERSISTENT},
+    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSignatures", CLEAR_ON_MANAGER_START},
+    {"PrimeType", PERSISTENT},
+    {"RecordFront", PERSISTENT},
+    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
+    {"SecOCKey", PERSISTENT | DONT_LOG},
+    {"RouteCount", PERSISTENT},
+    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"SshEnabled", PERSISTENT},
+    {"TermsVersion", PERSISTENT},
+    {"TrainingVersion", PERSISTENT},
+    {"UbloxAvailable", PERSISTENT},
+    {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
+    {"UpdaterAvailableBranches", PERSISTENT},
+    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterState", CLEAR_ON_MANAGER_START},
+    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
+    {"UpdaterLastFetchTime", PERSISTENT},
+    {"Version", PERSISTENT},
+
+    // sunnypilot params
+    {"Mads", PERSISTENT},
+    {"MadsCruiseMain", PERSISTENT},
+    {"MadsDisengageLateralOnBrake", PERSISTENT},
+    {"MadsUnifiedEngagementMode", PERSISTENT},
+
+    {"ToyotaAutoHold", PERSISTENT},
+    {"ToyotaEnhancedBsm", PERSISTENT},
+    {"ToyotaTSS2Long", PERSISTENT},
+};
+
+} // namespace
+
+
+Params::Params(const std::string &path) {
+  params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
+  params_path = ensure_params_path(params_prefix, path);
+}
+
+Params::~Params() {
+  if (future.valid()) {
+    future.wait();
+  }
+  assert(queue.empty());
+}
+
+std::vector<std::string> Params::allKeys() const {
+  std::vector<std::string> ret;
+  for (auto &p : keys) {
+    ret.push_back(p.first);
+  }
+  return ret;
+}
+
+bool Params::checkKey(const std::string &key) {
+  return keys.find(key) != keys.end();
+}
+
+ParamKeyType Params::getKeyType(const std::string &key) {
+  return static_cast<ParamKeyType>(keys[key]);
+}
+
+int Params::put(const char* key, const char* value, size_t value_size) {
+  // Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
+  // 1) Create temp file
+  // 2) Write data to temp file
+  // 3) fsync() the temp file
+  // 4) rename the temp file to the real name
+  // 5) fsync() the containing directory
+  std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
+  int tmp_fd = mkstemp((char*)tmp_path.c_str());
+  if (tmp_fd < 0) return -1;
+
+  int result = -1;
+  do {
+    // Write value to temp.
+    ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
+    if (bytes_written < 0 || (size_t)bytes_written != value_size) {
+      result = -20;
+      break;
+    }
+
+    // fsync to force persist the changes.
+    if ((result = fsync(tmp_fd)) < 0) break;
+
+    FileLock file_lock(params_path + "/.lock");
+
+    // Move temp into place.
+    if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
+
+    // fsync parent directory
+    result = fsync_dir(getParamPath());
+  } while (false);
+
+  close(tmp_fd);
+  if (result != 0) {
+    ::unlink(tmp_path.c_str());
+  }
+  return result;
+}
+
+int Params::remove(const std::string &key) {
+  FileLock file_lock(params_path + "/.lock");
+  int result = unlink(getParamPath(key).c_str());
+  if (result != 0) {
+    return result;
+  }
+  return fsync_dir(getParamPath());
+}
+
+std::string Params::get(const std::string &key, bool block) {
+  if (!block) {
+    return util::read_file(getParamPath(key));
+  } else {
+    // blocking read until successful
+    params_do_exit = 0;
+    void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
+    void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
+
+    std::string value;
+    while (!params_do_exit) {
+      if (value = util::read_file(getParamPath(key)); !value.empty()) {
+        break;
+      }
+      util::sleep_for(100);  // 0.1 s
+    }
+
+    std::signal(SIGINT, prev_handler_sigint);
+    std::signal(SIGTERM, prev_handler_sigterm);
+    return value;
+  }
+}
+
+std::map<std::string, std::string> Params::readAll() {
+  FileLock file_lock(params_path + "/.lock");
+  return util::read_files_in_dir(getParamPath());
+}
+
+void Params::clearAll(ParamKeyType key_type) {
+  FileLock file_lock(params_path + "/.lock");
+
+  // 1) delete params of key_type
+  // 2) delete files that are not defined in the keys.
+  if (DIR *d = opendir(getParamPath().c_str())) {
+    struct dirent *de = NULL;
+    while ((de = readdir(d))) {
+      if (de->d_type != DT_DIR) {
+        auto it = keys.find(de->d_name);
+        if (it == keys.end() || (it->second & key_type)) {
+          unlink(getParamPath(de->d_name).c_str());
+        }
+      }
+    }
+    closedir(d);
+  }
+
+  fsync_dir(getParamPath());
+}
+
+void Params::putNonBlocking(const std::string &key, const std::string &val) {
+   queue.push(std::make_pair(key, val));
+  // start thread on demand
+  if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
+    future = std::async(std::launch::async, &Params::asyncWriteThread, this);
+  }
+}
+
+void Params::asyncWriteThread() {
+  // TODO: write the latest one if a key has multiple values in the queue.
+  std::pair<std::string, std::string> p;
+  while (queue.try_pop(p, 0)) {
+    // Params::put is Thread-Safe
+    put(p.first, p.second);
+  }
+}

common/params.h

@@ -2,7 +2,6 @@
 
 #include <future>
 #include <map>
-#include <optional>
 #include <string>
 #include <tuple>
 #include <utility>
@@ -10,34 +9,16 @@
 
 #include "common/queue.h"
 
-enum ParamKeyFlag {
+enum ParamKeyType {
   PERSISTENT = 0x02,
   CLEAR_ON_MANAGER_START = 0x04,
   CLEAR_ON_ONROAD_TRANSITION = 0x08,
   CLEAR_ON_OFFROAD_TRANSITION = 0x10,
   DONT_LOG = 0x20,
   DEVELOPMENT_ONLY = 0x40,
-  CLEAR_ON_IGNITION_ON = 0x80,
-  BACKUP = 0x100,
   ALL = 0xFFFFFFFF
 };
 
-enum ParamKeyType {
-  STRING = 0, // must be utf-8 decodable
-  BOOL = 1,
-  INT = 2,
-  FLOAT = 3,
-  TIME = 4, // ISO 8601
-  JSON = 5,
-  BYTES = 6
-};
-
-struct ParamKeyAttributes {
-  uint32_t flags;
-  ParamKeyType type;
-  std::optional<std::string> default_value = std::nullopt;
-};
-
 class Params {
 public:
   explicit Params(const std::string &path = {});
@@ -46,18 +27,16 @@ public:
   Params(const Params&) = delete;
   Params& operator=(const Params&) = delete;
 
-  std::vector<std::string> allKeys(ParamKeyFlag flag = ALL) const;
+  std::vector<std::string> allKeys() const;
   bool checkKey(const std::string &key);
-  ParamKeyFlag getKeyFlag(const std::string &key);
   ParamKeyType getKeyType(const std::string &key);
-  std::optional<std::string> getKeyDefaultValue(const std::string &key);
   inline std::string getParamPath(const std::string &key = {}) {
     return params_path + params_prefix + (key.empty() ? "" : "/" + key);
   }
 
   // Delete a value
   int remove(const std::string &key);
-  void clearAll(ParamKeyFlag flag);
+  void clearAll(ParamKeyType type);
 
   // helpers for reading values
   std::string get(const std::string &key, bool block = false);

common/params.py

@@ -0,0 +1,18 @@
+from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
+assert Params
+assert ParamKeyType
+assert UnknownKeyName
+
+if __name__ == "__main__":
+  import sys
+
+  params = Params()
+  key = sys.argv[1]
+  assert params.check_key(key), f"unknown param: {key}"
+
+  if len(sys.argv) == 3:
+    val = sys.argv[2]
+    print(f"SET: {key} = {val}")
+    params.put(key, val)
+  elif len(sys.argv) == 2:
+    print(f"GET: {key} = {params.get(key)}")

common/params_pyx.pyx

@@ -0,0 +1,123 @@
+# distutils: language = c++
+# cython: language_level = 3
+from libcpp cimport bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+cdef extern from "common/params.h":
+  cpdef enum ParamKeyType:
+    PERSISTENT
+    CLEAR_ON_MANAGER_START
+    CLEAR_ON_ONROAD_TRANSITION
+    CLEAR_ON_OFFROAD_TRANSITION
+    DEVELOPMENT_ONLY
+    ALL
+
+  cdef cppclass c_Params "Params":
+    c_Params(string) except + nogil
+    string get(string, bool) nogil
+    bool getBool(string, bool) nogil
+    int remove(string) nogil
+    int put(string, string) nogil
+    void putNonBlocking(string, string) nogil
+    void putBoolNonBlocking(string, bool) nogil
+    int putBool(string, bool) nogil
+    bool checkKey(string) nogil
+    string getParamPath(string) nogil
+    void clearAll(ParamKeyType)
+    vector[string] allKeys()
+
+
+def ensure_bytes(v):
+  return v.encode() if isinstance(v, str) else v
+
+class UnknownKeyName(Exception):
+  pass
+
+cdef class Params:
+  cdef c_Params* p
+  cdef str d
+
+  def __cinit__(self, d=""):
+    cdef string path = <string>d.encode()
+    with nogil:
+      self.p = new c_Params(path)
+    self.d = d
+
+  def __reduce__(self):
+    return (type(self), (self.d,))
+
+  def __dealloc__(self):
+    del self.p
+
+  def clear_all(self, tx_type=ParamKeyType.ALL):
+    self.p.clearAll(tx_type)
+
+  def check_key(self, key):
+    key = ensure_bytes(key)
+    if not self.p.checkKey(key):
+      raise UnknownKeyName(key)
+    return key
+
+  def get(self, key, bool block=False, encoding=None):
+    cdef string k = self.check_key(key)
+    cdef string val
+    with nogil:
+      val = self.p.get(k, block)
+
+    if val == b"":
+      if block:
+        # If we got no value while running in blocked mode
+        # it means we got an interrupt while waiting
+        raise KeyboardInterrupt
+      else:
+        return None
+
+    return val if encoding is None else val.decode(encoding)
+
+  def get_bool(self, key, bool block=False):
+    cdef string k = self.check_key(key)
+    cdef bool r
+    with nogil:
+      r = self.p.getBool(k, block)
+    return r
+
+  def put(self, key, dat):
+    """
+    Warning: This function blocks until the param is written to disk!
+    In very rare cases this can take over a second, and your code will hang.
+    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
+    in general try to avoid writing params as much as possible.
+    """
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = ensure_bytes(dat)
+    with nogil:
+      self.p.put(k, dat_bytes)
+
+  def put_bool(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBool(k, val)
+
+  def put_nonblocking(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = ensure_bytes(dat)
+    with nogil:
+      self.p.putNonBlocking(k, dat_bytes)
+
+  def put_bool_nonblocking(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBoolNonBlocking(k, val)
+
+  def remove(self, key):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.remove(k)
+
+  def get_param_path(self, key=""):
+    cdef string key_bytes = ensure_bytes(key)
+    return self.p.getParamPath(key_bytes).decode("utf-8")
+
+  def all_keys(self):
+    return self.p.allKeys()

common/pid.py

@@ -0,0 +1,73 @@
+import numpy as np
+from numbers import Number
+
+from openpilot.common.numpy_fast import clip, interp
+
+
+class PIDController:
+  def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p = k_p
+    self._k_i = k_i
+    self._k_d = k_d
+    self.k_f = k_f   # feedforward gain
+    if isinstance(self._k_p, Number):
+      self._k_p = [[0], [self._k_p]]
+    if isinstance(self._k_i, Number):
+      self._k_i = [[0], [self._k_i]]
+    if isinstance(self._k_d, Number):
+      self._k_d = [[0], [self._k_d]]
+
+    self.pos_limit = pos_limit
+    self.neg_limit = neg_limit
+
+    self.i_unwind_rate = 0.3 / rate
+    self.i_rate = 1.0 / rate
+    self.speed = 0.0
+
+    self.reset()
+
+  @property
+  def k_p(self):
+    return interp(self.speed, self._k_p[0], self._k_p[1])
+
+  @property
+  def k_i(self):
+    return interp(self.speed, self._k_i[0], self._k_i[1])
+
+  @property
+  def k_d(self):
+    return interp(self.speed, self._k_d[0], self._k_d[1])
+
+  @property
+  def error_integral(self):
+    return self.i/self.k_i
+
+  def reset(self):
+    self.p = 0.0
+    self.i = 0.0
+    self.d = 0.0
+    self.f = 0.0
+    self.control = 0
+
+  def update(self, error, error_rate=0.0, speed=0.0, override=False, feedforward=0., freeze_integrator=False):
+    self.speed = speed
+
+    self.p = float(error) * self.k_p
+    self.f = feedforward * self.k_f
+    self.d = error_rate * self.k_d
+
+    if override:
+      self.i -= self.i_unwind_rate * float(np.sign(self.i))
+    else:
+      if not freeze_integrator:
+        self.i = self.i + error * self.k_i * self.i_rate
+
+        # Clip i to prevent exceeding control limits
+        control_no_i = self.p + self.d + self.f
+        control_no_i = clip(control_no_i, self.neg_limit, self.pos_limit)
+        self.i = clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
+
+    control = self.p + self.i + self.d + self.f
+
+    self.control = clip(control, self.neg_limit, self.pos_limit)
+    return self.control

common/prefix.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cassert>
+#include <string>
+
+#include "common/params.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+
+class OpenpilotPrefix {
+public:
+  OpenpilotPrefix(std::string prefix = {}) {
+    if (prefix.empty()) {
+      prefix = util::random_string(15);
+    }
+    msgq_path = "/dev/shm/" + prefix;
+    bool ret = util::create_directories(msgq_path, 0777);
+    assert(ret);
+    setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
+  }
+
+  ~OpenpilotPrefix() {
+    auto param_path = Params().getParamPath();
+    if (util::file_exists(param_path)) {
+      std::string real_path = util::readlink(param_path);
+      system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
+      unlink(param_path.c_str());
+    }
+    if (getenv("COMMA_CACHE") == nullptr) {
+      system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()).c_str());
+    }
+    system(util::string_format("rm %s -rf", Path::comma_home().c_str()).c_str());
+    system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
+    unsetenv("OPENPILOT_PREFIX");
+  }
+
+private:
+  std::string msgq_path;
+};

common/prefix.py

@@ -0,0 +1,53 @@
+import os
+import shutil
+import uuid
+
+
+from openpilot.common.params import Params
+from openpilot.system.hardware import PC
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+
+class OpenpilotPrefix:
+  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+    self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
+    self.msgq_path = os.path.join('/dev/shm', self.prefix)
+    self.clean_dirs_on_exit = clean_dirs_on_exit
+    self.shared_download_cache = shared_download_cache
+
+  def __enter__(self):
+    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
+    os.environ['OPENPILOT_PREFIX'] = self.prefix
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
+
+    if self.shared_download_cache:
+      os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
+
+    return self
+
+  def __exit__(self, exc_type, exc_obj, exc_tb):
+    if self.clean_dirs_on_exit:
+      self.clean_dirs()
+    try:
+      del os.environ['OPENPILOT_PREFIX']
+      if self.original_prefix is not None:
+        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
+    except KeyError:
+      pass
+    return False
+
+  def clean_dirs(self):
+    symlink_path = Params().get_param_path()
+    if os.path.exists(symlink_path):
+      shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
+      os.remove(symlink_path)
+    shutil.rmtree(self.msgq_path, ignore_errors=True)
+    if PC:
+      shutil.rmtree(Paths.log_root(), ignore_errors=True)
+    if not os.environ.get("COMMA_CACHE", False):
+      shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
+    shutil.rmtree(Paths.comma_home(), ignore_errors=True)

common/ratekeeper.cc

@@ -6,9 +6,9 @@
 #include "common/timing.h"
 #include "common/util.h"
 
-RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
-    : name(name_),
-      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
+    : name(name),
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
   interval = 1 / rate;
   last_monitor_time = seconds_since_boot();
   next_frame_time = last_monitor_time + interval;

common/realtime.py

@@ -0,0 +1,94 @@
+"""Utilities for reading real time clocks and keeping soft real time constraints."""
+import gc
+import os
+import time
+from collections import deque
+
+from setproctitle import getproctitle
+
+from openpilot.system.hardware import PC
+
+
+# time step for each process
+DT_CTRL = 0.01  # controlsd
+DT_MDL = 0.05  # model
+DT_HW = 0.5  # hardwared and manager
+DT_DMON = 0.05  # driver monitoring
+
+
+class Priority:
+  # CORE 2
+  # - modeld = 55
+  # - camerad = 54
+  CTRL_LOW = 51 # plannerd & radard
+
+  # CORE 3
+  # - pandad = 55
+  CTRL_HIGH = 53
+
+
+def set_realtime_priority(level: int) -> None:
+  if not PC:
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
+
+
+def set_core_affinity(cores: list[int]) -> None:
+  if not PC:
+    os.sched_setaffinity(0, cores)
+
+
+def config_realtime_process(cores: int | list[int], priority: int) -> None:
+  gc.disable()
+  set_realtime_priority(priority)
+  c = cores if isinstance(cores, list) else [cores, ]
+  set_core_affinity(c)
+
+
+class Ratekeeper:
+  def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
+    """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
+    self._interval = 1. / rate
+    self._next_frame_time = time.monotonic() + self._interval
+    self._print_delay_threshold = print_delay_threshold
+    self._frame = 0
+    self._remaining = 0.0
+    self._process_name = getproctitle()
+    self._dts = deque([self._interval], maxlen=100)
+    self._last_monitor_time = time.monotonic()
+
+  @property
+  def frame(self) -> int:
+    return self._frame
+
+  @property
+  def remaining(self) -> float:
+    return self._remaining
+
+  @property
+  def lagging(self) -> bool:
+    avg_dt = sum(self._dts) / len(self._dts)
+    expected_dt = self._interval * (1 / 0.9)
+    return avg_dt > expected_dt
+
+  # Maintain loop rate by calling this at the end of each loop
+  def keep_time(self) -> bool:
+    lagged = self.monitor_time()
+    if self._remaining > 0:
+      time.sleep(self._remaining)
+    return lagged
+
+  # Monitors the cumulative lag, but does not enforce a rate
+  def monitor_time(self) -> bool:
+    prev = self._last_monitor_time
+    self._last_monitor_time = time.monotonic()
+    self._dts.append(self._last_monitor_time - prev)
+
+    lagged = False
+    remaining = self._next_frame_time - time.monotonic()
+    self._next_frame_time += self._interval
+    if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
+      print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")
+      lagged = True
+    self._frame += 1
+    self._remaining = remaining
+    return lagged

common/retry.py

@@ -0,0 +1,30 @@
+import time
+import functools
+
+from openpilot.common.swaglog import cloudlog
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          cloudlog.exception(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        cloudlog.error(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator
+
+
+if __name__ == "__main__":
+  @retry(attempts=10)
+  def abc():
+    raise ValueError("abc failed :(")
+  abc()

common/run.py

@@ -0,0 +1,13 @@
+import subprocess
+
+
+def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
+  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
+
+
+def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
+  try:
+    return run_cmd(cmd, cwd=cwd, env=env)
+  except subprocess.CalledProcessError:
+    return default
+

common/spinner.py

@@ -0,0 +1,52 @@
+import os
+import subprocess
+from openpilot.common.basedir import BASEDIR
+
+
+class Spinner:
+  def __init__(self):
+    try:
+      self.spinner_proc = subprocess.Popen(["./spinner"],
+                                           stdin=subprocess.PIPE,
+                                           cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
+                                           close_fds=True)
+    except OSError:
+      self.spinner_proc = None
+
+  def __enter__(self):
+    return self
+
+  def update(self, spinner_text: str):
+    if self.spinner_proc is not None:
+      self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
+      try:
+        self.spinner_proc.stdin.flush()
+      except BrokenPipeError:
+        pass
+
+  def update_progress(self, cur: float, total: float):
+    self.update(str(round(100 * cur / total)))
+
+  def close(self):
+    if self.spinner_proc is not None:
+      self.spinner_proc.kill()
+      try:
+        self.spinner_proc.communicate(timeout=2.)
+      except subprocess.TimeoutExpired:
+        print("WARNING: failed to kill spinner")
+      self.spinner_proc = None
+
+  def __del__(self):
+    self.close()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+if __name__ == "__main__":
+  import time
+  with Spinner() as s:
+    s.update("Spinner text")
+    time.sleep(5.0)
+  print("gone")
+  time.sleep(5.0)

common/swaglog.cc

@@ -11,11 +11,9 @@
 
 #include <zmq.h>
 #include <stdarg.h>
-#include "json11/json11.hpp"
+#include "third_party/json11/json11.hpp"
 #include "common/version.h"
-#include "common/hardware/hw.h"
-
-#include "sunnypilot/common/version.h"
+#include "system/hardware/hw.h"
 
 class SwaglogState {
 public:
@@ -28,9 +26,6 @@ public:
     zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
     zmq_connect(sock, Path::swaglog_ipc().c_str());
 
-    // workaround for https://github.com/dropbox/json11/issues/38
-    setlocale(LC_NUMERIC, "C");
-
     print_level = CLOUDLOG_WARNING;
     if (const char* print_lvl = getenv("LOGPRINT")) {
       if (strcmp(print_lvl, "debug") == 0) {
@@ -58,7 +53,7 @@ public:
     if (char* daemon_name = getenv("MANAGER_DAEMON")) {
       ctx_j["daemon"] = daemon_name;
     }
-    ctx_j["version"] = SUNNYPILOT_VERSION;
+    ctx_j["version"] = COMMA_VERSION;
     ctx_j["dirty"] = !getenv("CLEAN");
     ctx_j["device"] = Hardware::get_name();
   }

common/swaglog.py

@@ -8,7 +8,7 @@ from logging.handlers import BaseRotatingHandler
 import zmq
 
 from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
-from openpilot.common.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
 
 
 def get_file_handler():

common/text_window.py

@@ -8,9 +8,9 @@ from openpilot.common.basedir import BASEDIR
 class TextWindow:
   def __init__(self, text):
     try:
-      self.text_proc = subprocess.Popen(["./text.py", text],
+      self.text_proc = subprocess.Popen(["./text", text],
                                         stdin=subprocess.PIPE,
-                                        cwd=os.path.join(BASEDIR, "openpilot/system", "ui"),
+                                        cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
                                         close_fds=True)
     except OSError:
       self.text_proc = None

common/time.py

@@ -0,0 +1,15 @@
+import datetime
+from pathlib import Path
+
+_MIN_DATE = datetime.datetime(year=2024, month=8, day=26)
+
+def min_date():
+  # on systemd systems, the default time is the systemd build time
+  systemd_path = Path("/lib/systemd/systemd")
+  if systemd_path.exists():
+    d = datetime.datetime.fromtimestamp(systemd_path.stat().st_mtime)
+    return d + datetime.timedelta(days=1)
+  return _MIN_DATE
+
+def system_time_valid():
+  return datetime.datetime.now() > min_date()

common/transformations/.gitignore

@@ -0,0 +1,2 @@
+transformations
+transformations.cpp

common/transformations/SConscript

@@ -0,0 +1,5 @@
+Import('env', 'envCython')
+
+transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc'])
+transformations_python = envCython.Program('transformations.so', 'transformations.pyx')
+Export('transformations', 'transformations_python')

common/transformations/coordinates.cc

@@ -1,6 +1,6 @@
 #define _USE_MATH_DEFINES
 
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 #include <iostream>
 #include <cmath>

common/transformations/model.py

@@ -0,0 +1,63 @@
+import numpy as np
+
+from openpilot.common.transformations.orientation import rot_from_euler
+from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame
+
+# segnet
+SEGNET_SIZE = (512, 384)
+
+# MED model
+MEDMODEL_INPUT_SIZE = (512, 256)
+MEDMODEL_YUV_SIZE = (MEDMODEL_INPUT_SIZE[0], MEDMODEL_INPUT_SIZE[1] * 3 // 2)
+MEDMODEL_CY = 47.6
+
+medmodel_fl = 910.0
+medmodel_intrinsics = np.array([
+  [medmodel_fl,  0.0,  0.5 * MEDMODEL_INPUT_SIZE[0]],
+  [0.0,  medmodel_fl,                   MEDMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
+
+
+# BIG model
+BIGMODEL_INPUT_SIZE = (1024, 512)
+BIGMODEL_YUV_SIZE = (BIGMODEL_INPUT_SIZE[0], BIGMODEL_INPUT_SIZE[1] * 3 // 2)
+
+bigmodel_fl = 910.0
+bigmodel_intrinsics = np.array([
+  [bigmodel_fl,  0.0,  0.5 * BIGMODEL_INPUT_SIZE[0]],
+  [0.0,  bigmodel_fl,             256 + MEDMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
+
+
+# SBIG model (big model with the size of small model)
+SBIGMODEL_INPUT_SIZE = (512, 256)
+SBIGMODEL_YUV_SIZE = (SBIGMODEL_INPUT_SIZE[0], SBIGMODEL_INPUT_SIZE[1] * 3 // 2)
+
+sbigmodel_fl = 455.0
+sbigmodel_intrinsics = np.array([
+  [sbigmodel_fl,  0.0,  0.5 * SBIGMODEL_INPUT_SIZE[0]],
+  [0.0,  sbigmodel_fl,      0.5 * (256 + MEDMODEL_CY)],
+  [0.0,  0.0,                                     1.0]])
+
+bigmodel_frame_from_calib_frame = np.dot(bigmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
+
+sbigmodel_frame_from_calib_frame = np.dot(sbigmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
+medmodel_frame_from_calib_frame = np.dot(medmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
+medmodel_frame_from_bigmodel_frame = np.dot(medmodel_intrinsics, np.linalg.inv(bigmodel_intrinsics))
+
+calib_from_medmodel = np.linalg.inv(medmodel_frame_from_calib_frame[:, :3])
+calib_from_sbigmodel = np.linalg.inv(sbigmodel_frame_from_calib_frame[:, :3])
+
+# This function is verified to give similar results to xx.uncommon.utils.transform_img
+def get_warp_matrix(device_from_calib_euler: np.ndarray, intrinsics: np.ndarray, bigmodel_frame: bool = False) -> np.ndarray:
+  calib_from_model = calib_from_sbigmodel if bigmodel_frame else calib_from_medmodel
+  device_from_calib = rot_from_euler(device_from_calib_euler)
+  camera_from_calib = intrinsics @ view_frame_from_device_frame @ device_from_calib
+  warp_matrix: np.ndarray = camera_from_calib @ calib_from_model
+  return warp_matrix

common/transformations/orientation.cc

@@ -4,8 +4,8 @@
 #include <cmath>
 #include <eigen3/Eigen/Dense>
 
-#include "sunnypilot/common/transformations/orientation.hpp"
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/orientation.hpp"
+#include "common/transformations/coordinates.hpp"
 
 Eigen::Quaterniond ensure_unique(const Eigen::Quaterniond &quat) {
   if (quat.w() > 0){
@@ -141,3 +141,4 @@ Eigen::Vector3d ned_euler_from_ecef(const ECEF &ecef_init, const Eigen::Vector3d
 
   return {phi, theta, psi};
 }
+

common/transformations/orientation.hpp

@@ -1,6 +1,6 @@
 #pragma once
 #include <eigen3/Eigen/Dense>
-#include "sunnypilot/common/transformations/coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 
 Eigen::Quaterniond ensure_unique(const Eigen::Quaterniond &quat);

common/transformations/orientation.py

@@ -25,10 +25,11 @@ def numpy_wrap(function, input_shape, output_shape) -> Callable[..., np.ndarray]
 
     # Add empty dimension if inputs is not a list
     if len(shape) == len(input_shape):
-      inp = inp.reshape((1, ) + inp.shape)
+      inp.shape = (1, ) + inp.shape
 
     result = np.asarray([function(*args, i) for i in inp])
-    return result.reshape(out_shape)
+    result.shape = out_shape
+    return result
   return f
 
 

common/transformations/tests/test_coordinates.py

@@ -102,36 +102,3 @@ class TestNED:
     np.testing.assert_allclose(converter.ned2ecef(ned_offsets_batch),
                                                            ecef_positions_offset_batch,
                                                            rtol=1e-9, atol=1e-7)
-
-  def test_errors(self):
-    # Test wrong shape/type for geodetic2ecef
-    # numpy_wrap raises IndexError for scalar input
-    with np.testing.assert_raises(IndexError):
-      coord.geodetic2ecef(1.0)
-
-    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
-      coord.geodetic2ecef([0, 0])
-
-    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
-      coord.geodetic2ecef([0, 0, 0, 0])
-
-    with np.testing.assert_raises(TypeError):
-      coord.geodetic2ecef(['a', 'b', 'c'])
-
-    # Test LocalCoord constructor errors
-    with np.testing.assert_raises(ValueError):
-      coord.LocalCoord.from_geodetic([0, 0])
-
-    with np.testing.assert_raises(ValueError):
-      coord.LocalCoord.from_geodetic(1)
-
-    with np.testing.assert_raises(TypeError):
-      coord.LocalCoord.from_geodetic(['a', 'b', 'c'])
-
-    # Test wrong shape/type for ecef2geodetic
-    with np.testing.assert_raises(ValueError):
-      coord.ecef2geodetic([1, 2])
-    with np.testing.assert_raises(ValueError):
-      coord.ecef2geodetic([1, 2, 3, 4])
-    with np.testing.assert_raises(IndexError):
-      coord.ecef2geodetic(1.0)

common/transformations/tests/test_orientation.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pytest
 
 from openpilot.common.transformations.orientation import euler2quat, quat2euler, euler2rot, rot2euler, \
                                                rot2quat, quat2rot, \
@@ -60,32 +59,3 @@ class TestOrientation:
       np.testing.assert_allclose(ned_eulers[i], ned_euler_from_ecef(ecef_positions[i], eulers[i]), rtol=1e-7)
       #np.testing.assert_allclose(eulers[i], ecef_euler_from_ned(ecef_positions[i], ned_eulers[i]), rtol=1e-7)
     # np.testing.assert_allclose(ned_eulers, ned_euler_from_ecef(ecef_positions, eulers), rtol=1e-7)
-
-  def test_inputs(self):
-    with pytest.raises(ValueError):
-      euler2quat([1, 2])
-
-    with pytest.raises(ValueError):
-      quat2rot([1, 2, 3])
-
-    with pytest.raises(IndexError):
-      rot2quat(np.zeros((2, 2)))
-
-  def test_euler_rot_consistency(self):
-    rpy = [0.1, 0.2, 0.3]
-    R = euler2rot(rpy)
-
-    # R -> q -> R
-    q = rot2quat(R)
-    R_new = quat2rot(q)
-    np.testing.assert_allclose(R, R_new, atol=1e-15)
-
-    # q -> R -> Euler (quat2euler) -> R
-    rpy_new = quat2euler(q)
-    R_new2 = euler2rot(rpy_new)
-    np.testing.assert_allclose(R, R_new2, atol=1e-15)
-
-    # R -> Euler (rot2euler) -> R
-    rpy_from_rot = rot2euler(R)
-    R_new3 = euler2rot(rpy_from_rot)
-    np.testing.assert_allclose(R, R_new3, atol=1e-15)

common/transformations/transformations.pxd

@@ -0,0 +1,72 @@
+# cython: language_level=3
+from libcpp cimport bool
+
+cdef extern from "orientation.cc":
+  pass
+
+cdef extern from "orientation.hpp":
+  cdef cppclass Quaternion "Eigen::Quaterniond":
+    Quaternion()
+    Quaternion(double, double, double, double)
+    double w()
+    double x()
+    double y()
+    double z()
+
+  cdef cppclass Vector3 "Eigen::Vector3d":
+    Vector3()
+    Vector3(double, double, double)
+    double operator()(int)
+
+  cdef cppclass Matrix3 "Eigen::Matrix3d":
+    Matrix3()
+    Matrix3(double*)
+
+    double operator()(int, int)
+
+  Quaternion euler2quat(const Vector3 &)
+  Vector3 quat2euler(const Quaternion &)
+  Matrix3 quat2rot(const Quaternion &)
+  Quaternion rot2quat(const Matrix3 &)
+  Vector3 rot2euler(const Matrix3 &)
+  Matrix3 euler2rot(const Vector3 &)
+  Matrix3 rot_matrix(double, double, double)
+  Vector3 ecef_euler_from_ned(const ECEF &, const Vector3 &)
+  Vector3 ned_euler_from_ecef(const ECEF &, const Vector3 &)
+
+
+cdef extern from "coordinates.cc":
+  cdef struct ECEF:
+    double x
+    double y
+    double z
+
+  cdef struct NED:
+    double n
+    double e
+    double d
+
+  cdef struct Geodetic:
+    double lat
+    double lon
+    double alt
+    bool radians
+
+  ECEF geodetic2ecef(const Geodetic &)
+  Geodetic ecef2geodetic(const ECEF &)
+
+  cdef cppclass LocalCoord_c "LocalCoord":
+    Matrix3 ned2ecef_matrix
+    Matrix3 ecef2ned_matrix
+
+    LocalCoord_c(const Geodetic &, const ECEF &)
+    LocalCoord_c(const Geodetic &)
+    LocalCoord_c(const ECEF &)
+
+    NED ecef2ned(const ECEF &)
+    ECEF ned2ecef(const NED &)
+    NED geodetic2ned(const Geodetic &)
+    Geodetic ned2geodetic(const NED &)
+
+cdef extern from "coordinates.hpp":
+  pass

common/transformations/transformations.pyx

@@ -0,0 +1,173 @@
+# distutils: language = c++
+# cython: language_level = 3
+from openpilot.common.transformations.transformations cimport Matrix3, Vector3, Quaternion
+from openpilot.common.transformations.transformations cimport ECEF, NED, Geodetic
+
+from openpilot.common.transformations.transformations cimport euler2quat as euler2quat_c
+from openpilot.common.transformations.transformations cimport quat2euler as quat2euler_c
+from openpilot.common.transformations.transformations cimport quat2rot as quat2rot_c
+from openpilot.common.transformations.transformations cimport rot2quat as rot2quat_c
+from openpilot.common.transformations.transformations cimport euler2rot as euler2rot_c
+from openpilot.common.transformations.transformations cimport rot2euler as rot2euler_c
+from openpilot.common.transformations.transformations cimport rot_matrix as rot_matrix_c
+from openpilot.common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from openpilot.common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from openpilot.common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
+from openpilot.common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
+from openpilot.common.transformations.transformations cimport LocalCoord_c
+
+
+import numpy as np
+cimport numpy as np
+
+cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m):
+    return np.array([
+        [m(0, 0), m(0, 1), m(0, 2)],
+        [m(1, 0), m(1, 1), m(1, 2)],
+        [m(2, 0), m(2, 1), m(2, 2)],
+    ])
+
+cdef Matrix3 numpy2matrix(np.ndarray[double, ndim=2, mode="fortran"] m):
+    assert m.shape[0] == 3
+    assert m.shape[1] == 3
+    return Matrix3(<double*>m.data)
+
+cdef ECEF list2ecef(ecef):
+    cdef ECEF e
+    e.x = ecef[0]
+    e.y = ecef[1]
+    e.z = ecef[2]
+    return e
+
+cdef NED list2ned(ned):
+    cdef NED n
+    n.n = ned[0]
+    n.e = ned[1]
+    n.d = ned[2]
+    return n
+
+cdef Geodetic list2geodetic(geodetic):
+    cdef Geodetic g
+    g.lat = geodetic[0]
+    g.lon = geodetic[1]
+    g.alt = geodetic[2]
+    return g
+
+def euler2quat_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Quaternion q = euler2quat_c(e)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def quat2euler_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Vector3 e = quat2euler_c(q)
+    return [e(0), e(1), e(2)]
+
+def quat2rot_single(quat):
+    cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
+    cdef Matrix3 r = quat2rot_c(q)
+    return matrix2numpy(r)
+
+def rot2quat_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Quaternion q = rot2quat_c(r)
+    return [q.w(), q.x(), q.y(), q.z()]
+
+def euler2rot_single(euler):
+    cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
+    cdef Matrix3 r = euler2rot_c(e)
+    return matrix2numpy(r)
+
+def rot2euler_single(rot):
+    cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
+    cdef Vector3 e = rot2euler_c(r)
+    return [e(0), e(1), e(2)]
+
+def rot_matrix(roll, pitch, yaw):
+    return matrix2numpy(rot_matrix_c(roll, pitch, yaw))
+
+def ecef_euler_from_ned_single(ecef_init, ned_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ned_pose[0], ned_pose[1], ned_pose[2])
+
+    cdef Vector3 e = ecef_euler_from_ned_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def ned_euler_from_ecef_single(ecef_init, ecef_pose):
+    cdef ECEF init = list2ecef(ecef_init)
+    cdef Vector3 pose = Vector3(ecef_pose[0], ecef_pose[1], ecef_pose[2])
+
+    cdef Vector3 e = ned_euler_from_ecef_c(init, pose)
+    return [e(0), e(1), e(2)]
+
+def geodetic2ecef_single(geodetic):
+    cdef Geodetic g = list2geodetic(geodetic)
+    cdef ECEF e = geodetic2ecef_c(g)
+    return [e.x, e.y, e.z]
+
+def ecef2geodetic_single(ecef):
+    cdef ECEF e = list2ecef(ecef)
+    cdef Geodetic g = ecef2geodetic_c(e)
+    return [g.lat, g.lon, g.alt]
+
+
+cdef class LocalCoord:
+    cdef LocalCoord_c * lc
+
+    def __init__(self, geodetic=None, ecef=None):
+        assert (geodetic is not None) or (ecef is not None)
+        if geodetic is not None:
+            self.lc = new LocalCoord_c(list2geodetic(geodetic))
+        elif ecef is not None:
+            self.lc = new LocalCoord_c(list2ecef(ecef))
+
+    @property
+    def ned2ecef_matrix(self):
+        return matrix2numpy(self.lc.ned2ecef_matrix)
+
+    @property
+    def ecef2ned_matrix(self):
+        return matrix2numpy(self.lc.ecef2ned_matrix)
+
+    @property
+    def ned_from_ecef_matrix(self):
+        return self.ecef2ned_matrix
+
+    @property
+    def ecef_from_ned_matrix(self):
+        return self.ned2ecef_matrix
+
+    @classmethod
+    def from_geodetic(cls, geodetic):
+        return cls(geodetic=geodetic)
+
+    @classmethod
+    def from_ecef(cls, ecef):
+        return cls(ecef=ecef)
+
+    def ecef2ned_single(self, ecef):
+        assert self.lc
+        cdef ECEF e = list2ecef(ecef)
+        cdef NED n = self.lc.ecef2ned(e)
+        return [n.n, n.e, n.d]
+
+    def ned2ecef_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef ECEF e = self.lc.ned2ecef(n)
+        return [e.x, e.y, e.z]
+
+    def geodetic2ned_single(self, geodetic):
+        assert self.lc
+        cdef Geodetic g = list2geodetic(geodetic)
+        cdef NED n = self.lc.geodetic2ned(g)
+        return [n.n, n.e, n.d]
+
+    def ned2geodetic_single(self, ned):
+        assert self.lc
+        cdef NED n = list2ned(ned)
+        cdef Geodetic g = self.lc.ned2geodetic(n)
+        return [g.lat, g.lon, g.alt]
+
+    def __dealloc__(self):
+        del self.lc

common/util.cc

@@ -0,0 +1,274 @@
+#include "common/util.h"
+
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/resource.h>
+
+#include <cassert>
+#include <cerrno>
+#include <cstring>
+#include <dirent.h>
+#include <fstream>
+#include <iomanip>
+#include <random>
+#include <sstream>
+
+#ifdef __linux__
+#include <sys/prctl.h>
+#include <sys/syscall.h>
+#ifndef __USE_GNU
+#define __USE_GNU
+#endif
+#include <sched.h>
+#endif  // __linux__
+
+namespace util {
+
+void set_thread_name(const char* name) {
+#ifdef __linux__
+  // pthread_setname_np is dumb (fails instead of truncates)
+  prctl(PR_SET_NAME, (unsigned long)name, 0, 0, 0);
+#endif
+}
+
+int set_realtime_priority(int level) {
+#ifdef __linux__
+  long tid = syscall(SYS_gettid);
+
+  // should match python using chrt
+  struct sched_param sa;
+  memset(&sa, 0, sizeof(sa));
+  sa.sched_priority = level;
+  return sched_setscheduler(tid, SCHED_FIFO, &sa);
+#else
+  return -1;
+#endif
+}
+
+int set_core_affinity(std::vector<int> cores) {
+#ifdef __linux__
+  long tid = syscall(SYS_gettid);
+  cpu_set_t cpu;
+
+  CPU_ZERO(&cpu);
+  for (const int n : cores) {
+    CPU_SET(n, &cpu);
+  }
+  return sched_setaffinity(tid, sizeof(cpu), &cpu);
+#else
+  return -1;
+#endif
+}
+
+int set_file_descriptor_limit(uint64_t limit_val) {
+  struct rlimit limit;
+  int status;
+
+  if ((status = getrlimit(RLIMIT_NOFILE, &limit)) < 0)
+    return status;
+
+  limit.rlim_cur = limit_val;
+  if ((status = setrlimit(RLIMIT_NOFILE, &limit)) < 0)
+    return status;
+
+  return 0;
+}
+
+std::string read_file(const std::string& fn) {
+  std::ifstream f(fn, std::ios::binary | std::ios::in);
+  if (f.is_open()) {
+    f.seekg(0, std::ios::end);
+    int size = f.tellg();
+    if (f.good() && size > 0) {
+      std::string result(size, '\0');
+      f.seekg(0, std::ios::beg);
+      f.read(result.data(), size);
+      // return either good() or has reached end-of-file (e.g. /sys/power/wakeup_count)
+      if (f.good() || f.eof()) {
+        result.resize(f.gcount());
+        return result;
+      }
+    }
+    // fallback for files created on read, e.g. procfs
+    std::stringstream buffer;
+    buffer << f.rdbuf();
+    return buffer.str();
+  }
+  return std::string();
+}
+
+std::map<std::string, std::string> read_files_in_dir(const std::string &path) {
+  std::map<std::string, std::string> ret;
+  DIR *d = opendir(path.c_str());
+  if (!d) return ret;
+
+  struct dirent *de = NULL;
+  while ((de = readdir(d))) {
+    if (de->d_type != DT_DIR) {
+      ret[de->d_name] = util::read_file(path + "/" + de->d_name);
+    }
+  }
+
+  closedir(d);
+  return ret;
+}
+
+int write_file(const char* path, const void* data, size_t size, int flags, mode_t mode) {
+  int fd = HANDLE_EINTR(open(path, flags, mode));
+  if (fd == -1) {
+    return -1;
+  }
+  ssize_t n = HANDLE_EINTR(write(fd, data, size));
+  close(fd);
+  return (n >= 0 && (size_t)n == size) ? 0 : -1;
+}
+
+FILE* safe_fopen(const char* filename, const char* mode) {
+  FILE* fp = NULL;
+  do {
+    fp = fopen(filename, mode);
+  } while ((nullptr == fp) && (errno == EINTR));
+  return fp;
+}
+
+size_t safe_fwrite(const void* ptr, size_t size, size_t count, FILE* stream) {
+  size_t written = 0;
+  do {
+    size_t ret = ::fwrite((void*)((char *)ptr + written * size), size, count - written, stream);
+    if (ret == 0 && errno != EINTR) break;
+    written += ret;
+  } while (written != count);
+  return written;
+}
+
+int safe_fflush(FILE *stream) {
+  int ret = EOF;
+  do {
+    ret = fflush(stream);
+  } while ((EOF == ret) && (errno == EINTR));
+  return ret;
+}
+
+int safe_ioctl(int fd, unsigned long request, void *argp) {
+  int ret;
+  do {
+    ret = ioctl(fd, request, argp);
+  } while ((ret == -1) && (errno == EINTR));
+  return ret;
+}
+
+std::string readlink(const std::string &path) {
+  char buff[4096];
+  ssize_t len = ::readlink(path.c_str(), buff, sizeof(buff)-1);
+  if (len != -1) {
+    buff[len] = '\0';
+    return std::string(buff);
+  }
+  return "";
+}
+
+bool file_exists(const std::string& fn) {
+  struct stat st = {};
+  return stat(fn.c_str(), &st) != -1;
+}
+
+static bool createDirectory(std::string dir, mode_t mode) {
+  auto verify_dir = [](const std::string& dir) -> bool {
+    struct stat st = {};
+    return (stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR);
+  };
+  // remove trailing /'s
+  while (dir.size() > 1 && dir.back() == '/') {
+    dir.pop_back();
+  }
+  // try to mkdir this directory
+  if (mkdir(dir.c_str(), mode) == 0) return true;
+  if (errno == EEXIST) return verify_dir(dir);
+  if (errno != ENOENT) return false;
+
+  // mkdir failed because the parent dir doesn't exist, so try to create it
+  size_t slash = dir.rfind('/');
+  if ((slash == std::string::npos || slash < 1) ||
+      !createDirectory(dir.substr(0, slash), mode)) {
+    return false;
+  }
+
+  // try again
+  if (mkdir(dir.c_str(), mode) == 0) return true;
+  return errno == EEXIST && verify_dir(dir);
+}
+
+bool create_directories(const std::string& dir, mode_t mode) {
+  if (dir.empty()) return false;
+  return createDirectory(dir, mode);
+}
+
+std::string getenv(const char* key, std::string default_val) {
+  const char* val = ::getenv(key);
+  return val ? val : default_val;
+}
+
+int getenv(const char* key, int default_val) {
+  const char* val = ::getenv(key);
+  return val ? atoi(val) : default_val;
+}
+
+float getenv(const char* key, float default_val) {
+  const char* val = ::getenv(key);
+  return val ? atof(val) : default_val;
+}
+
+std::string hexdump(const uint8_t* in, const size_t size) {
+  std::stringstream ss;
+  ss << std::hex << std::setfill('0');
+  for (size_t i = 0; i < size; i++) {
+    ss << std::setw(2) << static_cast<unsigned int>(in[i]);
+  }
+  return ss.str();
+}
+
+int random_int(int min, int max) {
+  std::random_device dev;
+  std::mt19937 rng(dev());
+  std::uniform_int_distribution<std::mt19937::result_type> dist(min, max);
+  return dist(rng);
+}
+
+std::string random_string(std::string::size_type length) {
+  const std::string chrs = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  std::mt19937 rg{std::random_device{}()};
+  std::uniform_int_distribution<std::string::size_type> pick(0, chrs.length() - 1);
+  std::string s;
+  s.reserve(length);
+  while (length--) {
+    s += chrs[pick(rg)];
+  }
+  return s;
+}
+
+bool starts_with(const std::string &s1, const std::string &s2) {
+  return strncmp(s1.c_str(), s2.c_str(), s2.size()) == 0;
+}
+
+bool ends_with(const std::string& s, const std::string& suffix) {
+  return s.size() >= suffix.size() &&
+         strcmp(s.c_str() + (s.size() - suffix.size()), suffix.c_str()) == 0;
+}
+
+std::string check_output(const std::string& command) {
+  char buffer[128];
+  std::string result;
+  std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(command.c_str(), "r"), pclose);
+
+  if (!pipe) {
+    return "";
+  }
+
+  while (fgets(buffer, std::size(buffer), pipe.get()) != nullptr) {
+    result += std::string(buffer);
+  }
+
+  return result;
+}
+
+}  // namespace util

common/util.h

@@ -0,0 +1,184 @@
+#pragma once
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <csignal>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+// keep trying if x gets interrupted by a signal
+#define HANDLE_EINTR(x)                                        \
+  ({                                                           \
+    decltype(x) ret_;                                          \
+    int try_cnt = 0;                                           \
+    do {                                                       \
+      ret_ = (x);                                              \
+    } while (ret_ == -1 && errno == EINTR && try_cnt++ < 100); \
+    ret_;                                                       \
+  })
+
+#ifndef sighandler_t
+typedef void (*sighandler_t)(int sig);
+#endif
+
+const double MILE_TO_KM = 1.609344;
+const double KM_TO_MILE = 1. / MILE_TO_KM;
+const double MS_TO_KPH = 3.6;
+const double MS_TO_MPH = MS_TO_KPH * KM_TO_MILE;
+const double METER_TO_MILE = KM_TO_MILE / 1000.0;
+const double METER_TO_FOOT = 3.28084;
+
+#define ALIGNED_SIZE(x, align) (((x) + (align)-1) & ~((align)-1))
+
+namespace util {
+
+void set_thread_name(const char* name);
+int set_realtime_priority(int level);
+int set_core_affinity(std::vector<int> cores);
+int set_file_descriptor_limit(uint64_t limit);
+
+// ***** math helpers *****
+
+// map x from [a1, a2] to [b1, b2]
+template <typename T>
+T map_val(T x, T a1, T a2, T b1, T b2) {
+  x = std::clamp(x, a1, a2);
+  T ra = a2 - a1;
+  T rb = b2 - b1;
+  return (x - a1) * rb / ra + b1;
+}
+
+// ***** string helpers *****
+
+template <typename... Args>
+std::string string_format(const std::string& format, Args... args) {
+  size_t size = snprintf(nullptr, 0, format.c_str(), args...) + 1;
+  std::unique_ptr<char[]> buf(new char[size]);
+  snprintf(buf.get(), size, format.c_str(), args...);
+  return std::string(buf.get(), buf.get() + size - 1);
+}
+
+std::string getenv(const char* key, std::string default_val = "");
+int getenv(const char* key, int default_val);
+float getenv(const char* key, float default_val);
+
+std::string hexdump(const uint8_t* in, const size_t size);
+bool starts_with(const std::string &s1, const std::string &s2);
+bool ends_with(const std::string &s, const std::string &suffix);
+
+// ***** random helpers *****
+int random_int(int min, int max);
+std::string random_string(std::string::size_type length);
+
+// **** file helpers *****
+std::string read_file(const std::string& fn);
+std::map<std::string, std::string> read_files_in_dir(const std::string& path);
+int write_file(const char* path, const void* data, size_t size, int flags = O_WRONLY, mode_t mode = 0664);
+
+FILE* safe_fopen(const char* filename, const char* mode);
+size_t safe_fwrite(const void * ptr, size_t size, size_t count, FILE * stream);
+int safe_fflush(FILE *stream);
+int safe_ioctl(int fd, unsigned long request, void *argp);
+
+std::string readlink(const std::string& path);
+bool file_exists(const std::string& fn);
+bool create_directories(const std::string &dir, mode_t mode);
+
+std::string check_output(const std::string& command);
+
+inline void sleep_for(const int milliseconds) {
+  if (milliseconds > 0) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
+  }
+}
+
+}  // namespace util
+
+class ExitHandler {
+public:
+  ExitHandler() {
+    std::signal(SIGINT, (sighandler_t)set_do_exit);
+    std::signal(SIGTERM, (sighandler_t)set_do_exit);
+
+#ifndef __APPLE__
+    std::signal(SIGPWR, (sighandler_t)set_do_exit);
+#endif
+  }
+  inline static std::atomic<bool> power_failure = false;
+  inline static std::atomic<int> signal = 0;
+  inline operator bool() { return do_exit; }
+  inline ExitHandler& operator=(bool v) {
+    signal = 0;
+    do_exit = v;
+    return *this;
+  }
+private:
+  static void set_do_exit(int sig) {
+#ifndef __APPLE__
+    power_failure = (sig == SIGPWR);
+#endif
+    signal = sig;
+    do_exit = true;
+  }
+  inline static std::atomic<bool> do_exit = false;
+};
+
+struct unique_fd {
+  unique_fd(int fd = -1) : fd_(fd) {}
+  unique_fd& operator=(unique_fd&& uf) {
+    fd_ = uf.fd_;
+    uf.fd_ = -1;
+    return *this;
+  }
+  ~unique_fd() {
+    if (fd_ != -1) close(fd_);
+  }
+  operator int() const { return fd_; }
+  int fd_;
+};
+
+class FirstOrderFilter {
+public:
+  FirstOrderFilter(float x0, float ts, float dt, bool initialized = true) {
+    k_ = (dt / ts) / (1.0 + dt / ts);
+    x_ = x0;
+    initialized_ = initialized;
+  }
+  inline float update(float x) {
+    if (initialized_) {
+      x_ = (1. - k_) * x_ + k_ * x;
+    } else {
+      initialized_ = true;
+      x_ = x;
+    }
+    return x_;
+  }
+  inline void reset(float x) { x_ = x; }
+  inline float x(){ return x_; }
+
+private:
+  float x_, k_;
+  bool initialized_;
+};
+
+template<typename T>
+void update_max_atomic(std::atomic<T>& max, T const& value) {
+  T prev = max;
+  while (prev < value && !max.compare_exchange_weak(prev, value)) {}
+}
+
+typedef struct Rect {
+  int x;
+  int y;
+  int w;
+  int h;
+} Rect;