Tools: Update setup command for macOS native setup

.github/PULL_REQUEST_TEMPLATE/bugfix.md

@@ -0,0 +1,15 @@
+---
+name: Bug fix
+about: For openpilot bug fixes
+title: ''
+labels: 'bugfix'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of the bug and the fix. Also link the issue if it exists. -->
+
+**Verification**
+
+<!-- Explain how you tested this bug fix. -->

.github/PULL_REQUEST_TEMPLATE/car_bugfix.md

@@ -0,0 +1,19 @@
+---
+name: Car Bug fix
+about: For vehicle/brand specific bug fixes
+title: ''
+labels: 'car bug fix'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of the bug and the fix. Also link the issue if it exists. -->
+
+**Verification**
+
+<!-- Explain how you tested this bug fix. -->
+
+**Route**
+
+Route: [a route with the bug fix]

.github/PULL_REQUEST_TEMPLATE/car_port.md

@@ -0,0 +1,15 @@
+---
+name: Car port
+about: For new car ports
+title: ''
+labels: 'car port'
+assignees: ''
+---
+
+**Checklist**
+
+- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
+- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
+- [ ] route with openpilot:
+- [ ] route with stock system:
+- [ ] car harness used (if comma doesn't sell it, put N/A):

.github/PULL_REQUEST_TEMPLATE/fingerprint.md

@@ -0,0 +1,13 @@
+---
+name: Fingerprint
+about: For adding fingerprints to existing cars
+title: ''
+labels: 'fingerprint'
+assignees: ''
+---
+
+**Car**
+Which car (make, model, year) this fingerprint is for
+
+**Route**
+A route with the fingerprint

.github/PULL_REQUEST_TEMPLATE/refactor.md

@@ -0,0 +1,15 @@
+---
+name: Refactor
+about: For code refactors
+title: ''
+labels: 'refactor'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of the refactor, including the goals it accomplishes. -->
+
+**Verification**
+
+<!-- Explain how you tested the refactor for regressions. -->

.github/PULL_REQUEST_TEMPLATE/tuning.md

@@ -0,0 +1,31 @@
+---
+name: Tuning
+about: For openpilot tuning changes
+title: ''
+labels: 'tuning'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of what is wrong with the current tuning and how the PR addresses this. -->
+
+**Verification**
+
+<!-- To verify tuning, capture the following scenarios (broadly, not exactly), with current tune and this tune.
+Use the PlotJuggler tuning layout to compare planned versus actual behavior.
+
+Run ./juggle.py <route> --layout layouts/tuning.xml , screenshot the full tab of interest, and paste into this PR.
+
+Longitudinal:
+* Maintaining speed at 25, 40, 65mph
+* Driving up and down hills
+* Accelerating from a stop
+* Decelerating to a stop
+* Following large changes in set speed
+* Coming to a stop behind a lead car
+
+Lateral:
+* Straight driving at ~25, ~45 and ~65mph
+* Turns driving at ~25, ~45 and ~65mph
+-->

.github/build.py

@@ -0,0 +1,30 @@
+import pathlib
+
+GITHUB_FOLDER = pathlib.Path(__file__).parent
+
+PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
+
+order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
+
+def create_pull_request_template():
+  with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
+    f.write("<!-- Please copy and paste the relevant template -->\n\n")
+
+    for t in order:
+      template = PULL_REQUEST_TEMPLATES / f"{t}.md"
+      text = template.read_text()
+
+      # Remove metadata for GitHub
+      start = text.find("---")
+      end = text.find("---", start+1)
+      text = text[end + 4:]
+
+      # Remove comments
+      text = text.replace("<!-- ", "").replace("-->", "")
+
+      f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
+      f.write(text)
+      f.write("\n\n")
+      f.write("-->\n\n")
+
+create_pull_request_template()

.github/workflows/docs.yaml

@@ -18,7 +18,7 @@ concurrency:
 jobs:
   docs:
     name: build docs
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     steps:
     - uses: commaai/timeout@v1
 

.github/workflows/selfdrive_tests.yaml

@@ -64,7 +64,15 @@ jobs:
       timeout-minutes: 1
       run: |
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "release/check-dirty.sh"
+        ${{ env.RUN }} "release/check-dirty.sh && \
+                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car system/manager"
+    - name: Static analysis
+      timeout-minutes: 1
+      run: |
+        cd $GITHUB_WORKSPACE
+        cp pyproject.toml $STRIPPED_DIR
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "scripts/lint/lint.sh --skip check_added_large_files"
 
   build:
     runs-on:
@@ -223,7 +231,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: .ci_cache/comma_download_cache
-        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }}
+        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
@@ -301,7 +309,6 @@ jobs:
     runs-on:
       - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
       - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
-    if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
     steps:
     - uses: actions/checkout@v4
       with:
@@ -315,7 +322,7 @@ jobs:
       run: |
         ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                         source selfdrive/test/setup_vsound.sh && \
-                        CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
+                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
 
   create_ui_report:
     # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml

.github/workflows/ui_preview.yaml

@@ -84,7 +84,7 @@ jobs:
         run: >-
           sudo apt-get install -y imagemagick
 
-          scenes="homescreen settings_device settings_software settings_toggles settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard keyboard_uppercase"
+          scenes="homescreen settings_device settings_software settings_toggles settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard"
           A=($scenes)
 
           DIFF=""

Dockerfile.openpilot

@@ -1,9 +1,9 @@
 FROM ghcr.io/commaai/openpilot-base:latest
 
-ENV PYTHONUNBUFFERED=1
+ENV PYTHONUNBUFFERED 1
 
-ENV OPENPILOT_PATH=/home/batman/openpilot
-ENV PYTHONPATH=${OPENPILOT_PATH}:${PYTHONPATH}
+ENV OPENPILOT_PATH /home/batman/openpilot
+ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
 
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}

Dockerfile.openpilot_base

@@ -1,16 +1,16 @@
 FROM ubuntu:24.04
 
-ENV PYTHONUNBUFFERED=1
+ENV PYTHONUNBUFFERED 1
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
     rm -rf /var/lib/apt/lists/*
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
 RUN /tmp/tools/install_ubuntu_dependencies.sh && \
@@ -55,9 +55,9 @@ RUN mkdir -p /tmp/opencl-driver-intel && \
     cd / && \
     rm -rf /tmp/opencl-driver-intel
 
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX=1
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX 1
 
 RUN dbus-uuidgen > /etc/machine-id
 

README.md

@@ -38,8 +38,7 @@ Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
   </tr>
 </table>
 
-
-Using openpilot in a car
+To start using openpilot in a car
 ------
 
 To use openpilot in a car, you need four things:
@@ -50,14 +49,6 @@ To use openpilot in a car, you need four things:
 
 We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
 
-### Branches
-| branch           | URL                                    | description                                                                         |
-|------------------|----------------------------------------|-------------------------------------------------------------------------------------|
-| `release3`         | openpilot.comma.ai                      | This is openpilot's release branch.                                                 |
-| `release3-staging` | openpilot-test.comma.ai                | This is the staging branch for releases. Use it to get new releases slightly early. |
-| `nightly`          | openpilot-nightly.comma.ai             | This is the bleeding edge development branch. Do not expect this to be stable.      |
-| `nightly-dev`      | installer.comma.ai/commaai/nightly-dev | Same as nightly, but includes experimental development features for some cars.      |
-
 To start developing openpilot
 ------
 

RELEASES.md

@@ -1,23 +1,16 @@
-Version 0.9.9 (2025-03-30)
-========================
-* Coming soon
-  * Rivian support
-  * F-150 & Mach-E support
-  * Tesla Model 3 support
-
-Version 0.9.8 (2025-01-30)
+Version 0.9.8 (2024-XX-XX)
 ========================
+* New driving model
+  * Trained in brand new ML simulator
+  * Model now gates applying positive accel in Chill mode
 * New driving monitoring model
   * Reduced false positives related to passengers
 * Image processing pipeline moved to the ISP
   * More GPU time for driving models
-  * Power draw reduced 0.5W, which means your device runs cooler
+  * Power draw reduced 0.5W, which means your device runs cooler 
 * Added toggle to enable driver monitoring even when openpilot is not engaged
 * Enable openpilot longitudinal control for Ford Q3 vehicles
 * New Toyota TSS2 longitudinal tune
-* Coming soon
-  * New driving model with gas gating
-  * Training data upload mode
 
 Version 0.9.7 (2024-06-13)
 ========================

SConstruct

@@ -49,6 +49,10 @@ AddOption('--ccflags',
           default='',
           help='pass arbitrary flags over the command line')
 
+AddOption('--snpe',
+          action='store_true',
+          help='use SNPE on PC')
+
 AddOption('--external-sconscript',
           action='store',
           metavar='FILE',
@@ -107,6 +111,7 @@ if arch == "larch64":
   ]
 
   libpath += [
+    "#third_party/snpe/larch64",
     "#third_party/libyuv/larch64/lib",
     "/usr/lib/aarch64-linux-gnu"
   ]
@@ -145,6 +150,14 @@ else:
       "/usr/local/lib",
     ]
 
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
+
 if GetOption('asan'):
   ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
   ldflags = ["-fsanitize=address"]
@@ -188,6 +201,7 @@ env = Environment(
     "#third_party/libyuv/include",
     "#third_party/json11",
     "#third_party/linux/include",
+    "#third_party/snpe/include",
     "#third_party",
     "#msgq",
   ],
@@ -340,7 +354,7 @@ SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
 SConscript(['cereal/SConscript'])
 
 Import('socketmaster', 'msgq')
-messaging = [socketmaster, msgq, 'capnp', 'kj',]
+messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
 Export('messaging')
 
 

cereal/README.md

@@ -29,50 +29,6 @@ then means breaking backwards-compatibility with all old logs of your fork. So w
 [custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
-An example of compatible changes:
-```diff
-diff --git a/cereal/custom.capnp b/cereal/custom.capnp
-index 3348e859e..3365c7b98 100644
---- a/cereal/custom.capnp
-+++ b/cereal/custom.capnp
-@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
- # DO rename the structs
- # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-
--struct CustomReserved0 @0x81c2f05a394cf4af {
-+struct SteeringInfo @0x81c2f05a394cf4af {
-+  active @0 :Bool;
-+  steeringAngleDeg @1 :Float32;
-+  steeringRateDeg @2 :Float32;
-+  steeringAccelDeg @3 :Float32;
- }
-
- struct CustomReserved1 @0xaedffd8f31e7b55d {
-diff --git a/cereal/log.capnp b/cereal/log.capnp
-index 1209f3fd9..b189f58b6 100644
---- a/cereal/log.capnp
-+++ b/cereal/log.capnp
-@@ -2558,14 +2558,14 @@ struct Event {
-
-     # DO change the name of the field
-     # DON'T change anything after the "@"
--    customReservedRawData0 @124 :Data;
-+    rawCanData @124 :Data;
-     customReservedRawData1 @125 :Data;
-     customReservedRawData2 @126 :Data;
-
-     # DO change the name of the field and struct
-     # DON'T change the ID (e.g. @107)
-     # DON'T change which struct it points to
--    customReserved0 @107 :Custom.CustomReserved0;
-+    steeringInfo @107 :Custom.SteeringInfo;
-     customReserved1 @108 :Custom.CustomReserved1;
-     customReserved2 @109 :Custom.CustomReserved2;
-     customReserved3 @110 :Custom.CustomReserved3;
-```
-
----
-
 Example
 ---
 ```python

cereal/custom.capnp

@@ -7,9 +7,7 @@ $Cxx.namespace("cereal");
 # These structs are guaranteed to remain reserved and empty in mainline
 # cereal, so use these if you want custom events in your fork.
 
-# DO rename the structs
-# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-
+# you can rename the struct, but don't change the identifier
 struct CustomReserved0 @0x81c2f05a394cf4af {
 }
 
@@ -39,33 +37,3 @@ struct CustomReserved8 @0xf416ec09499d9d19 {
 
 struct CustomReserved9 @0xa1680744031fdb2d {
 }
-
-struct CustomReserved10 @0xcb9fd56c7057593a {
-}
-
-struct CustomReserved11 @0xc2243c65e0340384 {
-}
-
-struct CustomReserved12 @0x9ccdc8676701b412 {
-}
-
-struct CustomReserved13 @0xcd96dafb67a082d0 {
-}
-
-struct CustomReserved14 @0xb057204d7deadf3f {
-}
-
-struct CustomReserved15 @0xbd443b539493bc68 {
-}
-
-struct CustomReserved16 @0xfc6241ed8877b611 {
-}
-
-struct CustomReserved17 @0xa30662f84033036c {
-}
-
-struct CustomReserved18 @0xc86a3d38d13eb3ef {
-}
-
-struct CustomReserved19 @0xa4f1eb3323f5f582 {
-}

cereal/log.capnp

@@ -409,7 +409,6 @@ struct GpsLocationData {
   speedAccuracy @12 :Float32;
 
   hasFix @13 :Bool;
-  satelliteCount @14 :Int8;
 
   enum SensorSource {
     android @0;
@@ -2441,14 +2440,6 @@ struct Microphone {
   filteredSoundPressureWeightedDb @2 :Float32;
 }
 
-struct Touch {
-  sec @0 :Int64;
-  usec @1 :Int64;
-  type @2 :UInt8;
-  code @3 :Int32;
-  value @4 :Int32;
-}
-
 struct Event {
   logMonoTime @0 :UInt64;  # nanoseconds
   valid @67 :Bool = true;
@@ -2529,9 +2520,6 @@ struct Event {
     logMessage @18 :Text;
     errorLogMessage @85 :Text;
 
-    # touch frame
-    touch @135 :List(Touch);
-
     # navigation
     navInstruction @82 :NavInstruction;
     navRoute @83 :NavRoute;
@@ -2554,17 +2542,11 @@ struct Event {
     livestreamWideRoadEncodeData @121 :EncodeData;
     livestreamDriverEncodeData @122 :EncodeData;
 
-    # *********** Custom: reserved for forks ***********
-
-    # DO change the name of the field
-    # DON'T change anything after the "@"
     customReservedRawData0 @124 :Data;
     customReservedRawData1 @125 :Data;
     customReservedRawData2 @126 :Data;
 
-    # DO change the name of the field and struct
-    # DON'T change the ID (e.g. @107)
-    # DON'T change which struct it points to 
+    # *********** Custom: reserved for forks ***********
     customReserved0 @107 :Custom.CustomReserved0;
     customReserved1 @108 :Custom.CustomReserved1;
     customReserved2 @109 :Custom.CustomReserved2;
@@ -2575,16 +2557,6 @@ struct Event {
     customReserved7 @114 :Custom.CustomReserved7;
     customReserved8 @115 :Custom.CustomReserved8;
     customReserved9 @116 :Custom.CustomReserved9;
-    customReserved10 @136 :Custom.CustomReserved10;
-    customReserved11 @137 :Custom.CustomReserved11;
-    customReserved12 @138 :Custom.CustomReserved12;
-    customReserved13 @139 :Custom.CustomReserved13;
-    customReserved14 @140 :Custom.CustomReserved14;
-    customReserved15 @141 :Custom.CustomReserved15;
-    customReserved16 @142 :Custom.CustomReserved16;
-    customReserved17 @143 :Custom.CustomReserved17;
-    customReserved18 @144 :Custom.CustomReserved18;
-    customReserved19 @145 :Custom.CustomReserved19;
 
     # *********** legacy + deprecated ***********
     model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated

cereal/maptile.capnp

@@ -0,0 +1,49 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0xa086df597ef5d7a0;
+
+# Geometry
+struct Point {
+  x @0: Float64;
+  y @1: Float64;
+  z @2: Float64;
+}
+
+struct PolyLine {
+  points @0: List(Point);
+}
+
+# Map features
+struct Lane {
+  id @0 :Text;
+
+  leftBoundary @1 :LaneBoundary;
+  rightBoundary @2 :LaneBoundary;
+
+  leftAdjacentId @3 :Text;
+  rightAdjacentId @4 :Text;
+
+  inboundIds @5 :List(Text);
+  outboundIds @6 :List(Text);
+
+  struct LaneBoundary {
+    polyLine @0 :PolyLine;
+    startHeading @1 :Float32; # WRT north
+  }
+}
+
+# Map tiles
+struct TileSummary {
+  version @0 :Text;
+  updatedAt @1 :UInt64; # Millis since epoch
+
+  level @2 :UInt8;
+  x @3 :UInt16;
+  y @4 :UInt16;
+}
+
+struct MapTile {
+  summary @0 :TileSummary;
+  lanes @1 :List(Lane);
+}

cereal/messaging/bridge.cc

@@ -6,12 +6,12 @@
 
 ExitHandler do_exit;
 
-static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
+static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
   std::vector<std::string> service_list;
   for (const auto& it : services) {
     std::string name = it.second.name;
     bool in_whitelist = whitelist_str.find(name) != std::string::npos;
-    if (zmq_to_msgq && !in_whitelist) {
+    if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
       continue;
     }
     service_list.push_back(name);

cereal/services.py

@@ -22,7 +22,6 @@ _services: dict[str, tuple] = {
   "temperatureSensor2": (True, 2., 200),
   "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "touch": (True, 20., 1),
   "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
   "controlsState": (True, 100., 10),
   "selfdriveState": (True, 100., 10),
@@ -53,7 +52,7 @@ _services: dict[str, tuple] = {
   "livePose": (True, 20., 4),
   "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 10),
-  "thumbnail": (True, 1 / 60., 1),
+  "thumbnail": (True, 0.2, 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
   "roadCameraState": (True, 20., 20),

common/clutil.cc

@@ -79,10 +79,6 @@ cl_context cl_create_context(cl_device_id device_id) {
   return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
 }
 
-void cl_release_context(cl_context context) {
-  clReleaseContext(context);
-}
-
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
   return cl_program_from_source(ctx, device_id, util::read_file(path), args);
 }

common/clutil.h

@@ -23,7 +23,6 @@
 
 cl_device_id cl_get_device_id(cl_device_type device_type);
 cl_context cl_create_context(cl_device_id device_id);
-void cl_release_context(cl_context context);
 cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
 cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/file_helpers.py

@@ -1,10 +1,6 @@
-import io
 import os
 import tempfile
 import contextlib
-import zstandard as zstd
-
-LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change
 
 
 class CallbackReader:
@@ -39,20 +35,3 @@ def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encodin
     yield tmp_file
     tmp_file_name = tmp_file.name
   os.replace(tmp_file_name, path)
-
-
-def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
-  if not should_compress:
-    file_size = os.path.getsize(filepath)
-    file_stream = open(filepath, "rb")
-    return file_stream, file_size
-
-  # Compress the file on the fly
-  compressed_stream = io.BytesIO()
-  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
-
-  with open(filepath, "rb") as f:
-    compressor.copy_stream(f, compressed_stream)
-    compressed_size = compressed_stream.tell()
-    compressed_stream.seek(0)
-    return compressed_stream, compressed_size

common/logging_extra.py

@@ -8,7 +8,6 @@ import uuid
 import socket
 import logging
 import traceback
-import numpy as np
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
@@ -16,8 +15,6 @@ from contextlib import contextmanager
 LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
 
 def json_handler(obj):
-  if isinstance(obj, np.bool_):
-    return bool(obj)
   # if isinstance(obj, (datetime.date, datetime.time)):
   #   return obj.isoformat()
   return repr(obj)

common/params.cc

@@ -89,7 +89,6 @@ private:
 
 std::unordered_map<std::string, uint32_t> keys = {
     {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
-    {"AdbEnabled", PERSISTENT},
     {"AlwaysOnDM", PERSISTENT},
     {"ApiCache_Device", PERSISTENT},
     {"AssistNowToken", PERSISTENT},
@@ -120,7 +119,6 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
     {"ExperimentalMode", PERSISTENT},
     {"ExperimentalModeConfirmed", PERSISTENT},
-    {"FirehoseMode", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"ForcePowerDown", PERSISTENT},
     {"GitBranch", PERSISTENT},
@@ -135,6 +133,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"GsmRoaming", PERSISTENT},
     {"HardwareSerial", PERSISTENT},
     {"HasAcceptedTerms", PERSISTENT},
+    {"IMEI", PERSISTENT},
     {"InstallDate", PERSISTENT},
     {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
     {"IsEngaged", PERSISTENT},

common/pid.py

@@ -1,6 +1,9 @@
 import numpy as np
 from numbers import Number
 
+from openpilot.common.numpy_fast import clip, interp
+
+
 class PIDController:
   def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
     self._k_p = k_p
@@ -25,15 +28,15 @@ class PIDController:
 
   @property
   def k_p(self):
-    return np.interp(self.speed, self._k_p[0], self._k_p[1])
+    return interp(self.speed, self._k_p[0], self._k_p[1])
 
   @property
   def k_i(self):
-    return np.interp(self.speed, self._k_i[0], self._k_i[1])
+    return interp(self.speed, self._k_i[0], self._k_i[1])
 
   @property
   def k_d(self):
-    return np.interp(self.speed, self._k_d[0], self._k_d[1])
+    return interp(self.speed, self._k_d[0], self._k_d[1])
 
   @property
   def error_integral(self):
@@ -61,10 +64,10 @@ class PIDController:
 
         # Clip i to prevent exceeding control limits
         control_no_i = self.p + self.d + self.f
-        control_no_i = np.clip(control_no_i, self.neg_limit, self.pos_limit)
-        self.i = np.clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
+        control_no_i = clip(control_no_i, self.neg_limit, self.pos_limit)
+        self.i = clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
 
     control = self.p + self.i + self.d + self.f
 
-    self.control = np.clip(control, self.neg_limit, self.pos_limit)
+    self.control = clip(control, self.neg_limit, self.pos_limit)
     return self.control

common/realtime.py

@@ -27,6 +27,11 @@ class Priority:
   CTRL_HIGH = 53
 
 
+def set_realtime_priority(level: int) -> None:
+  if not PC:
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
+
+
 def set_core_affinity(cores: list[int]) -> None:
   if not PC:
     os.sched_setaffinity(0, cores)
@@ -34,8 +39,7 @@ def set_core_affinity(cores: list[int]) -> None:
 
 def config_realtime_process(cores: int | list[int], priority: int) -> None:
   gc.disable()
-  if not PC:
-    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(priority))
+  set_realtime_priority(priority)
   c = cores if isinstance(cores, list) else [cores, ]
   set_core_affinity(c)
 

common/swaglog.cc

@@ -26,9 +26,6 @@ public:
     zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
     zmq_connect(sock, Path::swaglog_ipc().c_str());
 
-    // workaround for https://github.com/dropbox/json11/issues/38
-    setlocale(LC_NUMERIC, "C");
-
     print_level = CLOUDLOG_WARNING;
     if (const char* print_lvl = getenv("LOGPRINT")) {
       if (strcmp(print_lvl, "debug") == 0) {

common/tests/test_numpy_fast.py

@@ -0,0 +1,21 @@
+import numpy as np
+
+from openpilot.common.numpy_fast import interp
+
+
+class TestInterp:
+  def test_correctness_controls(self):
+    _A_CRUISE_MIN_BP = np.asarray([0., 5., 10., 20., 40.])
+    _A_CRUISE_MIN_V = np.asarray([-1.0, -.8, -.67, -.5, -.30])
+    v_ego_arr = [-1, -1e-12, 0, 4, 5, 6, 7, 10, 11, 15.2, 20, 21, 39,
+                 39.999999, 40, 41]
+
+    expected = np.interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
+    actual = interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
+
+    np.testing.assert_equal(actual, expected)
+
+    for v_ego in v_ego_arr:
+      expected = np.interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
+      actual = interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
+      np.testing.assert_equal(actual, expected)

common/transformations/model.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from openpilot.common.transformations.orientation import rot_from_euler
-from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame, _ar_ox_fisheye
+from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame
 
 # segnet
 SEGNET_SIZE = (512, 384)
@@ -39,13 +39,6 @@ sbigmodel_intrinsics = np.array([
   [0.0,  sbigmodel_fl,      0.5 * (256 + MEDMODEL_CY)],
   [0.0,  0.0,                                     1.0]])
 
-DM_INPUT_SIZE = (1440, 960)
-dmonitoringmodel_fl = _ar_ox_fisheye.focal_length
-dmonitoringmodel_intrinsics = np.array([
-  [dmonitoringmodel_fl,  0.0, DM_INPUT_SIZE[0]/2],
-  [0.0, dmonitoringmodel_fl, DM_INPUT_SIZE[1]/2 - (_ar_ox_fisheye.height - DM_INPUT_SIZE[1])/2],
-  [0.0,  0.0, 1.0]])
-
 bigmodel_frame_from_calib_frame = np.dot(bigmodel_intrinsics,
   get_view_frame_from_calib_frame(0, 0, 0, 0))
 

common/util.cc

@@ -255,29 +255,6 @@ bool ends_with(const std::string& s, const std::string& suffix) {
          strcmp(s.c_str() + (s.size() - suffix.size()), suffix.c_str()) == 0;
 }
 
-std::string strip(const std::string &str) {
-  auto should_trim = [](unsigned char ch) {
-    // trim whitespace or a null character
-    return std::isspace(ch) || ch == '\0';
-  };
-
-  size_t start = 0;
-  while (start < str.size() && should_trim(static_cast<unsigned char>(str[start]))) {
-    start++;
-  }
-
-  if (start == str.size()) {
-    return "";
-  }
-
-  size_t end = str.size() - 1;
-  while (end > 0 && should_trim(static_cast<unsigned char>(str[end]))) {
-    end--;
-  }
-
-  return str.substr(start, end - start + 1);
-}
-
 std::string check_output(const std::string& command) {
   char buffer[128];
   std::string result;
@@ -294,17 +271,4 @@ std::string check_output(const std::string& command) {
   return result;
 }
 
-bool system_time_valid() {
-  // Default to August 26, 2024
-  tm min_tm = {.tm_year = 2024 - 1900, .tm_mon = 7, .tm_mday = 26};
-  time_t min_date = mktime(&min_tm);
-
-  struct stat st;
-  if (stat("/lib/systemd/systemd", &st) == 0) {
-    min_date = std::max(min_date, st.st_mtime + 86400);  // Add 1 day (86400 seconds)
-  }
-
-  return time(nullptr) > min_date;
-}
-
 }  // namespace util

common/util.h

@@ -74,7 +74,6 @@ float getenv(const char* key, float default_val);
 std::string hexdump(const uint8_t* in, const size_t size);
 bool starts_with(const std::string &s1, const std::string &s2);
 bool ends_with(const std::string &s, const std::string &suffix);
-std::string strip(const std::string &str);
 
 // ***** random helpers *****
 int random_int(int min, int max);
@@ -96,8 +95,6 @@ bool create_directories(const std::string &dir, mode_t mode);
 
 std::string check_output(const std::string& command);
 
-bool system_time_valid();
-
 inline void sleep_for(const int milliseconds) {
   if (milliseconds > 0) {
     std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));

docs/CARS.md

@@ -4,13 +4,13 @@
 
 A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
 
-# 290 Supported Cars
+# 289 Supported Cars
 
 |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br>&nbsp;|Video|
 |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
 |Acura|ILX 2016-19|AcuraWatch Plus|openpilot|26 mph|25 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Acura&model=ILX 2016-19">Buy Here</a></sub></details>||
 |Acura|RDX 2016-18|AcuraWatch Plus|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Acura&model=RDX 2016-18">Buy Here</a></sub></details>||
-|Acura|RDX 2019-21|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|3 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Acura&model=RDX 2019-21">Buy Here</a></sub></details>||
+|Acura|RDX 2019-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|3 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Acura&model=RDX 2019-22">Buy Here</a></sub></details>||
 |Audi|A3 2014-19|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Audi&model=A3 2014-19">Buy Here</a></sub></details>||
 |Audi|A3 Sportback e-tron 2017-18|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Audi&model=A3 Sportback e-tron 2017-18">Buy Here</a></sub></details>||
 |Audi|Q2 2018|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Audi&model=Q2 2018">Buy Here</a></sub></details>||
@@ -56,7 +56,6 @@ A supported vehicle is one that just works when you install a comma device. All
 |Genesis|GV60 (Performance Trim) 2022-23[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV60 (Performance Trim) 2022-23">Buy Here</a></sub></details>||
 |Genesis|GV70 (2.5T Trim, without HDA II) 2022-23[<sup>5</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 (2.5T Trim, without HDA II) 2022-23">Buy Here</a></sub></details>||
 |Genesis|GV70 (3.5T Trim, without HDA II) 2022-23[<sup>5</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai M connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 (3.5T Trim, without HDA II) 2022-23">Buy Here</a></sub></details>||
-|Genesis|GV70 Electrified (Australia Only) 2022[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 Electrified (Australia Only) 2022">Buy Here</a></sub></details>||
 |Genesis|GV70 Electrified (with HDA II) 2023[<sup>5</sup>](#footnotes)|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 Electrified (with HDA II) 2023">Buy Here</a></sub></details>||
 |Genesis|GV80 2023[<sup>5</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai M connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV80 2023">Buy Here</a></sub></details>||
 |GMC|Sierra 1500 2020-21|Driver Alert Package II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|6 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 GM connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=GMC&model=Sierra 1500 2020-21">Buy Here</a></sub></details>|<a href="https://youtu.be/5HbNoBLzRwE" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
@@ -80,7 +79,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Honda|Odyssey 2018-20|Honda Sensing|openpilot|26 mph|0 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Odyssey 2018-20">Buy Here</a></sub></details>||
 |Honda|Passport 2019-23|All|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Passport 2019-23">Buy Here</a></sub></details>||
 |Honda|Pilot 2016-22|Honda Sensing|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Pilot 2016-22">Buy Here</a></sub></details>||
-|Honda|Ridgeline 2017-25|Honda Sensing|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Ridgeline 2017-25">Buy Here</a></sub></details>||
+|Honda|Ridgeline 2017-24|Honda Sensing|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Ridgeline 2017-24">Buy Here</a></sub></details>||
 |Hyundai|Azera 2022|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Azera 2022">Buy Here</a></sub></details>||
 |Hyundai|Azera Hybrid 2019|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Azera Hybrid 2019">Buy Here</a></sub></details>||
 |Hyundai|Azera Hybrid 2020|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Azera Hybrid 2020">Buy Here</a></sub></details>||
@@ -88,11 +87,11 @@ A supported vehicle is one that just works when you install a comma device. All
 |Hyundai|Elantra 2017-18|Smart Cruise Control (SCC)|Stock|19 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra 2017-18">Buy Here</a></sub></details>||
 |Hyundai|Elantra 2019|Smart Cruise Control (SCC)|Stock|19 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra 2019">Buy Here</a></sub></details>||
 |Hyundai|Elantra 2021-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra 2021-23">Buy Here</a></sub></details>|<a href="https://youtu.be/_EdYQtV52-c" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-|Hyundai|Elantra GT 2017-20|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra GT 2017-20">Buy Here</a></sub></details>||
+|Hyundai|Elantra GT 2017-19|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra GT 2017-19">Buy Here</a></sub></details>||
 |Hyundai|Elantra Hybrid 2021-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra Hybrid 2021-23">Buy Here</a></sub></details>|<a href="https://youtu.be/_EdYQtV52-c" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Hyundai|Genesis 2015-16|Smart Cruise Control (SCC)|Stock|19 mph|37 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai J connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Genesis 2015-16">Buy Here</a></sub></details>||
 |Hyundai|i30 2017-19|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=i30 2017-19">Buy Here</a></sub></details>||
-|Hyundai|Ioniq 5 (Southeast Asia and Europe only) 2022-24[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (Southeast Asia and Europe only) 2022-24">Buy Here</a></sub></details>||
+|Hyundai|Ioniq 5 (Non-US only) 2022-24[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (Non-US only) 2022-24">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 5 (with HDA II) 2022-24[<sup>5</sup>](#footnotes)|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (with HDA II) 2022-24">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 5 (without HDA II) 2022-24[<sup>5</sup>](#footnotes)|Highway Driving Assist|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (without HDA II) 2022-24">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 6 (with HDA II) 2023-24[<sup>5</sup>](#footnotes)|Highway Driving Assist II|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai P connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 6 (with HDA II) 2023-24">Buy Here</a></sub></details>||
@@ -104,7 +103,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Hyundai|Ioniq Plug-in Hybrid 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Plug-in Hybrid 2020-22">Buy Here</a></sub></details>||
 |Hyundai|Kona 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|6 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona 2020">Buy Here</a></sub></details>||
 |Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2018-21">Buy Here</a></sub></details>||
-|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2022-23">Buy Here</a></sub></details>||
+|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2022-23">Buy Here</a></sub></details>||
 |Hyundai|Kona Electric (with HDA II, Korea only) 2023[<sup>5</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai R connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric (with HDA II, Korea only) 2023">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=U2fOCmcQ8hw" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Hyundai|Kona Hybrid 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai I connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Hybrid 2020">Buy Here</a></sub></details>||
 |Hyundai|Palisade 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Palisade 2020-22">Buy Here</a></sub></details>|<a href="https://youtu.be/TAnDqjF4fDY?t=456" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
@@ -128,7 +127,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Jeep|Grand Cherokee 2019-21|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Jeep&model=Grand Cherokee 2019-21">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=jBe4lWnRSu4" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Carnival 2022-24[<sup>5</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Carnival 2022-24">Buy Here</a></sub></details>||
 |Kia|Carnival (China only) 2023[<sup>5</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Carnival (China only) 2023">Buy Here</a></sub></details>||
-|Kia|Ceed 2019-21|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Ceed 2019-21">Buy Here</a></sub></details>||
+|Kia|Ceed 2019|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Ceed 2019">Buy Here</a></sub></details>||
 |Kia|EV6 (Southeast Asia only) 2022-24[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai P connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=EV6 (Southeast Asia only) 2022-24">Buy Here</a></sub></details>||
 |Kia|EV6 (with HDA II) 2022-24[<sup>5</sup>](#footnotes)|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai P connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=EV6 (with HDA II) 2022-24">Buy Here</a></sub></details>||
 |Kia|EV6 (without HDA II) 2022-24[<sup>5</sup>](#footnotes)|Highway Driving Assist|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=EV6 (without HDA II) 2022-24">Buy Here</a></sub></details>||
@@ -184,7 +183,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Lexus|RX Hybrid 2016|Lexus Safety System+|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=RX Hybrid 2016">Buy Here</a></sub></details>||
 |Lexus|RX Hybrid 2017-19|All|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=RX Hybrid 2017-19">Buy Here</a></sub></details>||
 |Lexus|RX Hybrid 2020-22|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=RX Hybrid 2020-22">Buy Here</a></sub></details>||
-|Lexus|UX Hybrid 2019-24|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=UX Hybrid 2019-24">Buy Here</a></sub></details>||
+|Lexus|UX Hybrid 2019-23|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=UX Hybrid 2019-23">Buy Here</a></sub></details>||
 |Lincoln|Aviator 2020-24|Co-Pilot360 Plus|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Ford Q3 connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lincoln&model=Aviator 2020-24">Buy Here</a></sub></details>||
 |Lincoln|Aviator Plug-in Hybrid 2020-24|Co-Pilot360 Plus|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Ford Q3 connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lincoln&model=Aviator Plug-in Hybrid 2020-24">Buy Here</a></sub></details>||
 |MAN|eTGE 2020-24|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=MAN&model=eTGE 2020-24">Buy Here</a></sub></details>|<a href="https://youtu.be/4100gLeabmo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
@@ -260,7 +259,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Toyota|RAV4 Hybrid 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2017-18">Buy Here</a></sub></details>|<a href="https://youtu.be/LhT5VzJVfNI?t=26" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Toyota|RAV4 Hybrid 2019-21|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2019-21">Buy Here</a></sub></details>||
 |Toyota|RAV4 Hybrid 2022|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2022">Buy Here</a></sub></details>|<a href="https://youtu.be/U0nH9cnrFB0" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-|Toyota|RAV4 Hybrid 2023-25|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2023-25">Buy Here</a></sub></details>|<a href="https://youtu.be/4eIsEq4L4Ng" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
+|Toyota|RAV4 Hybrid 2023-24|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2023-24">Buy Here</a></sub></details>||
 |Toyota|Sienna 2018-20|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=Sienna 2018-20">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=q1UPOo4Sh68" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Volkswagen|Arteon 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Arteon 2018-23">Buy Here</a></sub></details>|<a href="https://youtu.be/FAomFKPFlDA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Volkswagen|Arteon eHybrid 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Arteon eHybrid 2020-23">Buy Here</a></sub></details>|<a href="https://youtu.be/FAomFKPFlDA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
@@ -295,7 +294,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Volkswagen|Teramont 2018-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Teramont 2018-22">Buy Here</a></sub></details>||
 |Volkswagen|Teramont Cross Sport 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Teramont Cross Sport 2021-22">Buy Here</a></sub></details>||
 |Volkswagen|Teramont X 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Teramont X 2021-22">Buy Here</a></sub></details>||
-|Volkswagen|Tiguan 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Tiguan 2018-23">Buy Here</a></sub></details>||
+|Volkswagen|Tiguan 2018-24|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Tiguan 2018-24">Buy Here</a></sub></details>||
 |Volkswagen|Tiguan eHybrid 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Tiguan eHybrid 2021-23">Buy Here</a></sub></details>||
 |Volkswagen|Touran 2016-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Touran 2016-23">Buy Here</a></sub></details>||
 

docs/CONTRIBUTING.md

@@ -63,12 +63,3 @@ A good pull request has all of the following:
 * Connect your device to Wi-Fi regularly, so that we can pull data for training better driving models.
 * Run the `nightly` branch and report issues. This branch is like `master` but it's built just like a release.
 * Annotate images in the [comma10k dataset](https://github.com/commaai/comma10k).
-
-## Contributing Training Data
-
-### A guide for forks
-
-In order for your fork's data to be eligible for the training set:
-* **Your cereal messaging structs must be [compatible](../cereal#custom-forks)**
-* **The definitions of all the stock messaging structs must not change**: Do not change how any of the fields are set, including everything from `selfdriveState.enabled` to `carState.steeringAngleDeg`. Instead, create your own structs and set them however you'd like.
-* **Do not include cars that are not supported in upstream platforms**: Instead, create new opendbc platforms for cars that you'd like to support outside of upstream, even if it's just a trim-level difference.

docs/css/tooltip.css

@@ -1,44 +0,0 @@
-[data-tooltip] {
-  position: relative;
-  display: inline-block;
-  border-bottom: 1px dotted black;
-}
-
-[data-tooltip] .tooltip-content {
-  width: max-content;
-  max-width: 25em;
-  position: absolute;
-  top: 100%;
-  left: 50%;
-  transform: translateX(-50%);
-  background-color: white;
-  color: #404040;
-  box-shadow: 0 4px 14px 0 rgba(0,0,0,.2), 0 0 0 1px rgba(0,0,0,.05);
-  padding: 10px;
-  font: 14px/1.5 Lato, proxima-nova, Helvetica Neue, Arial, sans-serif;
-  text-decoration: none;
-  opacity: 0;
-  visibility: hidden;
-  transition: opacity 0.1s, visibility 0s;
-  z-index: 1000;
-  pointer-events: none; /* Prevent accidental interaction */
-}
-
-[data-tooltip]:hover .tooltip-content {
-  opacity: 1;
-  visibility: visible;
-  pointer-events: auto; /* Allow interaction when visible */
-}
-
-.tooltip-content .tooltip-glossary-link {
-  display: inline-block;
-  margin-top: 8px;
-  font-size: 12px;
-  color: #007bff;
-  text-decoration: none;
-}
-
-.tooltip-content .tooltip-glossary-link:hover {
-  color: #0056b3;
-  text-decoration: underline;
-}

docs/hooks/glossary.py

@@ -1,68 +0,0 @@
-import re
-import tomllib
-
-def load_glossary(file_path="docs/glossary.toml"):
-  with open(file_path, "rb") as f:
-    glossary_data = tomllib.load(f)
-  return glossary_data.get("glossary", {})
-
-def generate_anchor_id(name):
-  return name.replace(" ", "-").replace("_", "-").lower()
-
-def format_markdown_term(name, definition):
-  anchor_id = generate_anchor_id(name)
-  markdown = f"* [**{name.replace('_', ' ').title()}**](#{anchor_id})"
-  if definition.get("abbreviation"):
-    markdown += f" *({definition['abbreviation']})*"
-  if definition.get("description"):
-    markdown += f": {definition['description']}\n"
-  return markdown
-
-def glossary_markdown(vocabulary):
-  markdown = ""
-  for category, terms in vocabulary.items():
-    markdown += f"## {category.replace('_', ' ').title()}\n\n"
-    for name, definition in terms.items():
-      markdown += format_markdown_term(name, definition)
-  return markdown
-
-def format_tooltip_html(term_key, definition, html):
-  display_term = term_key.replace("_", " ").title()
-  clean_description = re.sub(r"\[(.+)]\(.+\)", r"\1", definition["description"])
-  glossary_link = (
-    f"<a href='/concepts/glossary#{term_key}' class='tooltip-glossary-link' title='View in glossary'>Glossary🔗</a>"
-  )
-  return re.sub(
-    re.escape(display_term),
-    lambda
-    match: f"<span data-tooltip>{match.group(0)}<span class='tooltip-content'>{clean_description} {glossary_link}</span></span>",
-    html,
-    flags=re.IGNORECASE,
-  )
-
-def apply_tooltip(_term_key, _definition, pattern, html):
-  return re.sub(
-    pattern,
-    lambda match: format_tooltip_html(_term_key, _definition, match.group(0)),
-    html,
-    flags=re.IGNORECASE,
-  )
-
-def tooltip_html(vocabulary, html):
-  for _category, terms in vocabulary.items():
-    for term_key, definition in terms.items():
-      if definition.get("description"):
-        pattern = rf"(?<!\w){re.escape(term_key.replace('_', ' ').title())}(?![^<]*<\/a>)(?!\([^)]*\))"
-        html = apply_tooltip(term_key, definition, pattern, html)
-  return html
-
-# Page Hooks
-def on_page_markdown(markdown, **kwargs):
-  glossary = load_glossary()
-  return markdown.replace("{{GLOSSARY_DEFINITIONS}}", glossary_markdown(glossary))
-
-def on_page_content(html, **kwargs):
-  if kwargs.get("page").title == "Glossary":
-    return html
-  glossary = load_glossary()
-  return tooltip_html(glossary, html)

docs/how-to/connect-to-comma.md

@@ -29,22 +29,6 @@ Here's an example command for connecting to your device using its tethered conne
 
 For doing development work on device, it's recommended to use [SSH agent forwarding](https://docs.github.com/en/developers/overview/using-ssh-agent-forwarding).
 
-
-## ADB
-
-In order to use ADB on your device, you'll need to enable it in the device's settings.
-
-* Enable ADB in your device's settings
-* Connect to your device
-    * `adb shell` over USB
-    * `adb connect` over WiFi
-    * Here's an example command for connecting to your device using its tethered connection: `adb connect 192.168.43.1:5555`
-
-> [!NOTE]
-> The default port for ADB is 5555 on the comma 3/3X.
-
-For more info on ADB, see the [Android Debug Bridge (ADB) documentation](https://developer.android.com/tools/adb).
-
 ### Notes
 
 The public keys are only fetched from your GitHub account once. In order to update your device's authorized keys, you'll need to re-enter your GitHub username.

launch_env.sh

@@ -7,7 +7,7 @@ export OPENBLAS_NUM_THREADS=1
 export VECLIB_MAXIMUM_THREADS=1
 
 if [ -z "$AGNOS_VERSION" ]; then
-  export AGNOS_VERSION="11.7"
+  export AGNOS_VERSION="11.3"
 fi
 
 export STAGING_ROOT="/data/safe_staging"

mkdocs.yml

@@ -8,10 +8,6 @@ strict: true
 docs_dir: docs
 site_dir: docs_site/
 
-hooks:
-  - docs/hooks/glossary.py
-extra_css:
-  - css/tooltip.css
 theme:
   name: readthedocs
   navigation_depth: 3

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openpilot"
-requires-python = ">= 3.11, < 3.13"
+requires-python = ">= 3.11, <= 3.12"
 license = {text = "MIT License"}
 version = "0.1.0"
 description = "an open source driver assistance system"
@@ -42,7 +42,8 @@ dependencies = [
 
   # modeld
   "onnx >= 1.14.0",
-  "onnxruntime >=1.16.3",
+  "onnxruntime >=1.16.3; platform_system == 'Linux' and platform_machine == 'aarch64'",
+  "onnxruntime-gpu >=1.16.3; platform_system == 'Linux' and platform_machine == 'x86_64'",
 
   # logging
   "pyzmq",
@@ -103,6 +104,7 @@ dev = [
   "lru-dict",
   "matplotlib",
   "parameterized >=0.8, <0.9",
+  #"pprofile",
   "pyautogui",
   "pyopencl; platform_machine != 'aarch64'", # broken on arm64
   "pytools < 2024.1.11; platform_machine != 'aarch64'", # pyopencl use a broken version
@@ -111,12 +113,13 @@ dev = [
   "tabulate",
   "types-requests",
   "types-tabulate",
-  "raylib @ https://github.com/commaai/raylib-python-cffi/releases/download/wheel/raylib-5.5.0.2-cp312-cp312-linux_x86_64.whl ; (platform_machine == 'x86_64' and platform_system == 'Linux' and python_version == '3.12')",
-  "raylib @ https://github.com/commaai/raylib-python-cffi/releases/download/wheel/raylib-5.5.0.2-cp311-cp311-linux_x86_64.whl ; (platform_machine == 'x86_64' and platform_system == 'Linux' and python_version == '3.11')",
+
+  # this is only pinned since 5.15.11 is broken
+  "pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
 ]
 
 tools = [
-  "metadrive-simulator @ https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl ; (platform_machine != 'aarch64')",
+  "metadrive-simulator @ https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl ; (platform_machine != 'aarch64')",
   "rerun-sdk >= 0.18",
 ]
 
@@ -166,7 +169,7 @@ testpaths = [
 [tool.codespell]
 quiet-level = 3
 # if you've got a short variable name that's getting flagged, add it here
-ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead,TGE,abl"
+ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead"
 builtin = "clear,rare,informal,code,names,en-GB_to_en-US"
 skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, *.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*"
 

release/release_files.py

@@ -6,11 +6,40 @@ from pathlib import Path
 HERE = os.path.abspath(os.path.dirname(__file__))
 ROOT = HERE + "/.."
 
+# blacklisting is for two purposes:
+# - minimizing release download size
+# - keeping the diff readable
 blacklist = [
-  ".git/",
+  "panda/drivers/",
+  "panda/examples/",
+  "panda/tests/safety/",
+
+  "opendbc_repo/dbc/.*.dbc$",
+  "opendbc_repo/dbc/generator/",
+
+  "cereal/.*test.*",
+  "^common/tests/",
+
+  # particularly large text files
+  "uv.lock",
+  "third_party/catch2",
+  "selfdrive/car/tests/test_models.*",
+
+  "^tools/",
+  "^tinygrad_repo/",
 
   "matlab.*.md",
 
+  ".git/",
+  ".github/",
+  "Darwin/",
+  ".vscode",
+
+  # common things
+  "LICENSE",
+  "Dockerfile",
+  ".pre-commit",
+
   # no LFS or submodules in release
   ".lfsconfig",
   ".gitattributes",
@@ -19,9 +48,80 @@ blacklist = [
 ]
 
 # gets you through the blacklist
-whitelist: list[str] = [
+whitelist = [
+  "tools/lib/",
+  "tools/bodyteleop/",
+  "tools/joystick/",
+  "tools/longitudinal_maneuvers/",
+
+  "tinygrad_repo/openpilot/compile2.py",
+  "tinygrad_repo/extra/onnx.py",
+  "tinygrad_repo/extra/onnx_ops.py",
+  "tinygrad_repo/extra/thneed.py",
+  "tinygrad_repo/extra/utils.py",
+  "tinygrad_repo/tinygrad/codegen/kernel.py",
+  "tinygrad_repo/tinygrad/codegen/linearizer.py",
+  "tinygrad_repo/tinygrad/features/image.py",
+  "tinygrad_repo/tinygrad/features/search.py",
+  "tinygrad_repo/tinygrad/nn/*",
+  "tinygrad_repo/tinygrad/renderer/cstyle.py",
+  "tinygrad_repo/tinygrad/renderer/opencl.py",
+  "tinygrad_repo/tinygrad/runtime/lib.py",
+  "tinygrad_repo/tinygrad/runtime/ops_cpu.py",
+  "tinygrad_repo/tinygrad/runtime/ops_disk.py",
+  "tinygrad_repo/tinygrad/runtime/ops_gpu.py",
+  "tinygrad_repo/tinygrad/shape/*",
+  "tinygrad_repo/tinygrad/.*.py",
+
+  # TODO: do this automatically
+  "opendbc_repo/dbc/comma_body.dbc",
+  "opendbc_repo/dbc/chrysler_ram_hd_generated.dbc",
+  "opendbc_repo/dbc/chrysler_ram_dt_generated.dbc",
+  "opendbc_repo/dbc/chrysler_pacifica_2017_hybrid_generated.dbc",
+  "opendbc_repo/dbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc",
+  "opendbc_repo/dbc/gm_global_a_powertrain_generated.dbc",
+  "opendbc_repo/dbc/gm_global_a_object.dbc",
+  "opendbc_repo/dbc/gm_global_a_chassis.dbc",
+  "opendbc_repo/dbc/FORD_CADS.dbc",
+  "opendbc_repo/dbc/ford_fusion_2018_adas.dbc",
+  "opendbc_repo/dbc/ford_lincoln_base_pt.dbc",
+  "opendbc_repo/dbc/honda_accord_2018_can_generated.dbc",
+  "opendbc_repo/dbc/acura_ilx_2016_can_generated.dbc",
+  "opendbc_repo/dbc/acura_rdx_2018_can_generated.dbc",
+  "opendbc_repo/dbc/acura_rdx_2020_can_generated.dbc",
+  "opendbc_repo/dbc/honda_civic_touring_2016_can_generated.dbc",
+  "opendbc_repo/dbc/honda_civic_hatchback_ex_2017_can_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_touring_2016_can_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_ex_2017_can_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_ex_2017_body_generated.dbc",
+  "opendbc_repo/dbc/honda_crv_executive_2016_can_generated.dbc",
+  "opendbc_repo/dbc/honda_fit_ex_2018_can_generated.dbc",
+  "opendbc_repo/dbc/honda_odyssey_exl_2018_generated.dbc",
+  "opendbc_repo/dbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc",
+  "opendbc_repo/dbc/honda_insight_ex_2019_can_generated.dbc",
+  "opendbc_repo/dbc/acura_ilx_2016_nidec.dbc",
+  "opendbc_repo/dbc/honda_civic_ex_2022_can_generated.dbc",
+  "opendbc_repo/dbc/hyundai_canfd.dbc",
+  "opendbc_repo/dbc/hyundai_kia_generic.dbc",
+  "opendbc_repo/dbc/hyundai_kia_mando_front_radar_generated.dbc",
+  "opendbc_repo/dbc/mazda_2017.dbc",
+  "opendbc_repo/dbc/nissan_x_trail_2017_generated.dbc",
+  "opendbc_repo/dbc/nissan_leaf_2018_generated.dbc",
+  "opendbc_repo/dbc/subaru_global_2017_generated.dbc",
+  "opendbc_repo/dbc/subaru_global_2020_hybrid_generated.dbc",
+  "opendbc_repo/dbc/subaru_outback_2015_generated.dbc",
+  "opendbc_repo/dbc/subaru_outback_2019_generated.dbc",
+  "opendbc_repo/dbc/subaru_forester_2017_generated.dbc",
+  "opendbc_repo/dbc/toyota_tnga_k_pt_generated.dbc",
+  "opendbc_repo/dbc/toyota_new_mc_pt_generated.dbc",
+  "opendbc_repo/dbc/toyota_nodsu_pt_generated.dbc",
+  "opendbc_repo/dbc/toyota_adas.dbc",
+  "opendbc_repo/dbc/toyota_tss2_adas.dbc",
+  "opendbc_repo/dbc/vw_golf_mk4.dbc",
+  "opendbc_repo/dbc/vw_mqb_2010.dbc",
 ]
 
+
 if __name__ == "__main__":
   for f in Path(ROOT).rglob("**/*"):
     if not (f.is_file() or f.is_symlink()):

scripts/build_small.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -ex
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+# git clone --mirror
+SRC=/tmp/openpilot.git/
+OUT=/tmp/smallpilot/
+
+echo "starting size $(du -hs .git/)"
+
+rm -rf $OUT
+
+cd $SRC
+git remote update
+
+# copy contents
+#rsync -a --exclude='.git/' $DIR $OUT
+
+cp -r $SRC $OUT
+
+cd $OUT
+
+# remove all tags
+git tag -l | xargs git tag -d
+
+# remove non-master branches
+BRANCHES="release2 release3 devel master-ci nightly"
+for branch in $BRANCHES; do
+  git branch -D $branch
+  git branch -D ${branch}-staging || true
+done
+
+#git gc
+git reflog expire --expire=now --all
+git gc --prune=now
+git gc --aggressive --prune=now
+echo "new one is $(du -hs .)"

scripts/code_stats.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import os
+import ast
+import stat
+import subprocess
+
+fouts = {x.decode('utf-8') for x in subprocess.check_output(['git', 'ls-files']).strip().split()}
+
+pyf = []
+for d in ["cereal", "common", "scripts", "selfdrive", "tools"]:
+  for root, _, files in os.walk(d):
+    for f in files:
+      if f.endswith(".py"):
+        pyf.append(os.path.join(root, f))
+
+imps: set[str] = set()
+
+class Analyzer(ast.NodeVisitor):
+  def visit_Import(self, node):
+    for alias in node.names:
+      imps.add(alias.name)
+    self.generic_visit(node)
+
+  def visit_ImportFrom(self, node):
+    imps.add(node.module)
+    self.generic_visit(node)
+
+tlns = 0
+carlns = 0
+scriptlns = 0
+testlns = 0
+for f in sorted(pyf):
+  if f not in fouts:
+    continue
+  xbit = bool(os.stat(f)[stat.ST_MODE] & stat.S_IXUSR)
+  src = open(f).read()
+  lns = len(src.split("\n"))
+  tree = ast.parse(src)
+  Analyzer().visit(tree)
+  print(f"{lns:5d} {f} {xbit}")
+  if 'test' in f:
+    testlns += lns
+  elif f.startswith(('tools/', 'scripts/', 'selfdrive/debug')):
+    scriptlns += lns
+  elif f.startswith('selfdrive/car'):
+    carlns += lns
+  else:
+    tlns += lns
+
+print(f"{tlns} lines of openpilot python")
+print(f"{carlns} lines of car ports")
+print(f"{scriptlns} lines of tools/scripts/debug")
+print(f"{testlns} lines of tests")
+#print(sorted(list(imps)))

scripts/count_cars.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+from collections import Counter
+from pprint import pprint
+
+from opendbc.car.docs import get_all_car_docs
+
+if __name__ == "__main__":
+  cars = get_all_car_docs()
+  make_count = Counter(l.make for l in cars)
+  print("\n", "*" * 20, len(cars), "total", "*" * 20, "\n")
+  pprint(make_count)

scripts/git_rewrite/rewrite-git-history.sh

@@ -0,0 +1,391 @@
+#!/usr/bin/env bash
+set -e
+
+SRC=/tmp/openpilot/
+SRC_CLONE=/tmp/openpilot-clone/
+OUT=/tmp/openpilot-tiny/
+
+REWRITE_IGNORE_BRANCHES=(
+  dashcam3
+  devel
+  master-ci
+  nightly
+  release2
+  release3
+  release3-staging
+)
+
+VALIDATE_IGNORE_FILES=(
+  ".github/ISSUE_TEMPLATE/bug_report.md"
+  ".github/pull_request_template.md"
+)
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+LOGS_DIR=$DIR/git-rewrite-$(date +"%Y-%m-%dT%H:%M:%S%z")
+mkdir -p $LOGS_DIR
+
+GIT_REWRITE_LOG=$LOGS_DIR/git-rewrite-log.txt
+BRANCH_DIFF_LOG=$LOGS_DIR/branch-diff-log.txt
+COMMIT_DIFF_LOG=$LOGS_DIR/commit-diff-log.txt
+
+START_TIME=$(date +%s)
+exec > >(while IFS= read -r line; do
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  echo "[${ELAPSED_TIME}s] $line"
+done | tee -a "$GIT_REWRITE_LOG") 2>&1
+
+# INSTALL git-filter-repo
+if [ ! -f /tmp/git-filter-repo ]; then
+  echo "Installing git-filter-repo..."
+  curl -sSo /tmp/git-filter-repo https://raw.githubusercontent.com/newren/git-filter-repo/main/git-filter-repo
+  chmod +x /tmp/git-filter-repo
+fi
+
+# MIRROR openpilot
+if [ ! -d $SRC ]; then
+  echo "Mirroring openpilot..."
+  git clone --mirror https://github.com/commaai/openpilot.git $SRC # 4.18 GiB (488034 objects)
+
+  cd $SRC
+
+  echo "Starting size $(du -sh .)"
+
+  git remote update
+
+  # the git-filter-repo analysis is bliss - can be found in the repo root/filter-repo/analysis
+  echo "Analyzing with git-filter-repo..."
+  /tmp/git-filter-repo --force --analyze
+
+  echo "Pushing to openpilot-archive..."
+  # push to archive repo - in smaller parts because the 2 GB push limit - https://docs.github.com/en/get-started/using-git/troubleshooting-the-2-gb-push-limit
+  ARCHIVE_REPO=git@github.com:commaai/openpilot-archive.git
+  git push --prune $ARCHIVE_REPO +refs/heads/master:refs/heads/master # push master first so it's the default branch (when openpilot-archive is an empty repo)
+  git push --prune $ARCHIVE_REPO +refs/heads/*:refs/heads/* # 956.39 MiB (110725 objects)
+  git push --prune $ARCHIVE_REPO +refs/tags/*:refs/tags/* # 1.75 GiB (21694 objects)
+  # git push --mirror $ARCHIVE_REPO || true # fails to push refs/pull/* (deny updating a hidden ref) for pull requests
+  # we fail and continue - more reading: https://stackoverflow.com/a/34266401/639708 and https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
+fi
+
+# REWRITE master and tags
+if [ ! -d $SRC_CLONE ]; then
+  echo "Cloning $SRC..."
+  GIT_LFS_SKIP_SMUDGE=1 git clone $SRC $SRC_CLONE
+
+  cd $SRC_CLONE
+
+  echo "Checking out old history..."
+
+  git checkout tags/v0.7.1 > /dev/null 2>&1
+  # checkout as main, since we need master ref later
+  git checkout -b main
+
+  echo "Creating setup commits..."
+
+  # rm these so we don't get conflicts later
+  git rm -r cereal opendbc panda selfdrive/ui/ui > /dev/null
+  git commit -m "removed conflicting files" > /dev/null
+
+  # skip-smudge to get rid of some lfs errors that it can't find the reference of some lfs files
+  # we don't care about fetching/pushing lfs right now
+  git lfs install --skip-smudge --local
+
+  # squash initial setup commits
+  git cherry-pick -n -X theirs 6c33a5c..59b3d06 > /dev/null
+  git commit -m "switching to master" > /dev/null
+
+  # squash the two commits
+  git reset --soft HEAD~2
+  git commit -m "switching to master" -m "$(git log --reverse --format=%B 6c33a5c..59b3d06)" -m "removed conflicting files" > /dev/null
+
+  # get commits we want to cherry-pick
+  # will start with the next commit after #59b3d06 tools is local now
+  COMMITS=$(git rev-list --reverse 59b3d06..master)
+
+  # we need this for logging
+  TOTAL_COMMITS=$(echo $COMMITS | wc -w | xargs)
+  CURRENT_COMMIT_NUMBER=0
+
+  # empty this file
+  > commit-map.txt
+
+  echo "Rewriting master commits..."
+
+  for COMMIT in $COMMITS; do
+      CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
+      # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"\\r
+      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"
+
+      # set environment variables to preserve author/committer and dates
+      export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
+      export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
+      export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
+      export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
+      export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
+      export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
+
+      # cherry-pick the commit
+      if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
+        # check if the failure is because of an empty commit
+        if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
+          echo "Empty commit detected. Skipping commit $COMMIT"
+          git cherry-pick --skip
+          # log it was empty to the mapping file
+          echo "$COMMIT EMPTY" >> commit-map.txt
+        else
+          # handle other errors or conflicts
+          echo "Cherry-pick failed. Handling error..."
+          echo "$GIT_OUTPUT"
+          exit 1
+        fi
+      else
+        # capture the new commit hash
+        NEW_COMMIT=$(git rev-parse HEAD)
+
+        # save the old and new commit hashes to the mapping file
+        echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
+
+        # append the old commit ID to the commit message
+        git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
+      fi
+
+    # prune every 3000 commits to avoid gc errors
+    if [ $((CURRENT_COMMIT_NUMBER % 3000)) -eq 0 ]; then
+      echo "Pruning repo..."
+      git gc
+    fi
+  done
+
+  echo "Rewriting tags..."
+
+  # remove all old tags
+  git tag -l | xargs git tag -d
+
+  # read each line from the tag-commit-map.txt
+  while IFS=' ' read -r TAG OLD_COMMIT; do
+    # search for the new commit in commit-map.txt corresponding to the old commit
+    NEW_COMMIT=$(grep "^$OLD_COMMIT " "commit-map.txt" | awk '{print $2}')
+
+    # check if this is a rebased commit
+    if [ -z "$NEW_COMMIT" ]; then
+      # if not, then just use old commit hash
+      NEW_COMMIT=$OLD_COMMIT
+    fi
+
+    echo "Rewriting tag $TAG from commit $NEW_COMMIT"
+    git tag -f "$TAG" "$NEW_COMMIT"
+  done < "$DIR/tag-commit-map.txt"
+
+  # uninstall lfs since we don't want to touch (push to) lfs right now
+  # git push will also push lfs, if we don't uninstall (--local so just for this repo)
+  git lfs uninstall --local
+
+  # force push new master
+  git push --force origin main:master
+
+  # force push new tags
+  git push --force --tags
+fi
+
+# REWRITE branches based on master
+if [ ! -f "$SRC_CLONE/rewrite-branches-done" ]; then
+  cd $SRC_CLONE
+  > rewrite-branches-done
+
+  # empty file
+  > $BRANCH_DIFF_LOG
+
+  echo "Rewriting branches based on master..."
+
+  # will store raw diffs here, if exist
+  mkdir -p differences
+
+  # get a list of all branches except master and REWRITE_IGNORE_BRANCHES
+  BRANCHES=$(git branch -r | grep -v ' -> ' | sed 's/.*origin\///' | grep -v '^master$' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n'))
+
+  for BRANCH in $BRANCHES; do
+    # check if the branch is based on master history
+    MERGE_BASE=$(git merge-base master origin/$BRANCH) || true
+    if [ -n "$MERGE_BASE" ]; then
+      echo "Rewriting branch: $BRANCH"
+
+      # create a new branch based on the new master
+      NEW_MERGE_BASE=$(grep "^$MERGE_BASE " "commit-map.txt" | awk '{print $2}')
+      if [ -z "$NEW_MERGE_BASE" ]; then
+        echo "Error: could not find new merge base for branch $BRANCH" >> $BRANCH_DIFF_LOG
+        continue
+      fi
+      git checkout -b ${BRANCH}_new $NEW_MERGE_BASE
+
+      # get the range of commits unique to this branch
+      COMMITS=$(git rev-list --reverse $MERGE_BASE..origin/${BRANCH})
+
+      HAS_ERROR=0
+
+      # simple delimiter
+      echo "BRANCH ${BRANCH}" >> commit-map.txt
+
+      for COMMIT in $COMMITS; do
+        # set environment variables to preserve author/committer and dates
+        export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
+        export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
+        export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
+        export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
+        export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
+        export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
+
+        # cherry-pick the commit
+        if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
+          # check if the failure is because of an empty commit
+          if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
+            echo "Empty commit detected. Skipping commit $COMMIT"
+            git cherry-pick --skip
+            # log it was empty to the mapping file
+            echo "$COMMIT EMPTY" >> commit-map.txt
+          else
+            # handle other errors or conflicts
+            echo "Cherry-pick of ${BRANCH} branch failed. Removing branch upstream..." >> $BRANCH_DIFF_LOG
+            echo "$GIT_OUTPUT" > "$LOGS_DIR/branch-${BRANCH}"
+            git cherry-pick --abort
+            git push --delete origin ${BRANCH}
+            HAS_ERROR=1
+            break
+          fi
+        else
+          # capture the new commit hash
+          NEW_COMMIT=$(git rev-parse HEAD)
+
+          # save the old and new commit hashes to the mapping file
+          echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
+
+          # append the old commit ID to the commit message
+          git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
+        fi
+      done
+
+      # force push the new branch
+      if [ $HAS_ERROR -eq 0 ]; then
+        # git lfs goes haywire here, so we need to install and uninstall
+        # git lfs install --skip-smudge --local
+        git lfs uninstall --local > /dev/null
+        git push -f origin ${BRANCH}_new:${BRANCH}
+      fi
+
+      # clean up local branch
+      git checkout master > /dev/null
+      git branch -D ${BRANCH}_new > /dev/null
+    else
+      echo "Deleting branch $BRANCH as it's not based on master history" >> $BRANCH_DIFF_LOG
+      git push --delete origin ${BRANCH}
+    fi
+  done
+fi
+
+# VALIDATE cherry-pick
+if [ ! -f "$SRC_CLONE/validation-done" ]; then
+  cd $SRC_CLONE
+  > validation-done
+
+  TOTAL_COMMITS=$(grep -cve '^\s*$' commit-map.txt)
+  CURRENT_COMMIT_NUMBER=0
+  COUNT_SAME=0
+  COUNT_DIFF=0
+
+  # empty file
+  > $COMMIT_DIFF_LOG
+
+  echo "Validating commits..."
+
+  # will store raw diffs here, if exist
+  mkdir -p differences
+
+  # read each line from commit-map.txt
+  while IFS=' ' read -r OLD_COMMIT NEW_COMMIT; do
+    if [ "$NEW_COMMIT" == "EMPTY" ]; then
+      continue
+    fi
+    if [ "$OLD_COMMIT" == "BRANCH" ]; then
+      echo "Branch ${NEW_COMMIT} below:" >> $COMMIT_DIFF_LOG
+      continue
+    fi
+    CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
+    # retrieve short hashes and dates for the old and new commits
+    OLD_COMMIT_SHORT=$(git rev-parse --short $OLD_COMMIT)
+    NEW_COMMIT_SHORT=$(git rev-parse --short $NEW_COMMIT)
+    OLD_DATE=$(git show -s --format='%cd' $OLD_COMMIT)
+    NEW_DATE=$(git show -s --format='%cd' $NEW_COMMIT)
+
+    # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"\\r
+    echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"
+
+    # generate lists of files and their hashes for the old and new commits, excluding ignored files
+    OLD_FILES=$(git ls-tree -r $OLD_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
+    NEW_FILES=$(git ls-tree -r $NEW_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
+
+    # Compare the diffs
+    if diff <(echo "$OLD_FILES") <(echo "$NEW_FILES") > /dev/null; then
+      # echo "Old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT are equivalent."
+      COUNT_SAME=$((COUNT_SAME + 1))
+    else
+      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Difference found between old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT" >> $COMMIT_DIFF_LOG
+      COUNT_DIFF=$((COUNT_DIFF + 1))
+      set +e
+      diff -u <(echo "$OLD_FILES") <(echo "$NEW_FILES") > "$LOGS_DIR/commit-$CURRENT_COMMIT_NUMBER-$OLD_COMMIT_SHORT-$NEW_COMMIT_SHORT"
+      set -e
+    fi
+  done < "commit-map.txt"
+
+  echo "Summary:" >> $COMMIT_DIFF_LOG
+  echo "Equivalent commits: $COUNT_SAME" >> $COMMIT_DIFF_LOG
+  echo "Different commits: $COUNT_DIFF" >> $COMMIT_DIFF_LOG
+fi
+
+if [ ! -d $OUT ]; then
+  cp -r $SRC $OUT
+
+  cd $OUT
+
+  # remove all non-master branches
+  # git branch | grep -v "^  master$" | grep -v "\*" | xargs git branch -D
+
+  # echo "cleaning up refs"
+  # delete pull request refs since we can't alter them anyway (https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#error-failed-to-push-some-refs)
+  # git for-each-ref --format='%(refname)' | grep '^refs/pull/' | xargs -I {} git update-ref -d {}
+
+  echo "importing new lfs files"
+  # import "almost" everything to lfs
+  BRANCHES=$(git for-each-ref --format='%(refname)' refs/heads/ | sed 's%refs/heads/%%g' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n') | tr '\n' ' ')
+  git lfs migrate import --include="*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,selfdrive/car/tests/test_models_segs.txt,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,flask/**/*,panda/**/*,board/**/*,messaging/**/*,opendbc/**/*,tools/cabana/chartswidget.cc,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,selfdrive/ui/paint.cc,werkzeug/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,selfdrive/locationd/laikad.py,selfdrive/locationd/test/test_laikad.py,tools/gpstest/test_laikad.py,selfdrive/locationd/laikad_helpers.py,tools/nui/**/*,jsonrpc/**/*,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,selfdrive/camerad/cameras/camera_qcom.cc,selfdrive/manager.py,selfdrive/modeld/models/driving.cc,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,gunicorn/**/*,*.qm,jinja2/**/*,click/**/*,dbcs/**/*,websocket/**/*" $BRANCHES
+
+  echo "reflog and gc"
+  # this is needed after lfs import
+  git reflog expire --expire=now --all
+  git gc --prune=now --aggressive
+
+  # check the git-filter-repo analysis again - can be found in the repo root/filter-repo/analysis
+  echo "Analyzing with git-filter-repo..."
+  /tmp/git-filter-repo --force --analyze
+
+  echo "New size is $(du -sh .)"
+fi
+
+cd $OUT
+
+# fetch all lfs files from https://github.com/commaai/openpilot.git
+# some lfs files are missing on gitlab, but they can be found on github
+git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
+git config lfs.pushurl ssh://git@github.com/commaai/openpilot.git
+git lfs fetch --all || true
+
+# also fetch all lfs files from https://gitlab.com/commaai/openpilot-lfs.git
+git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
+git lfs fetch --all || true
+
+# final push - will also push lfs
+# TODO: switch to git@github.com:commaai/openpilot.git when ready
+# git push --mirror git@github.com:commaai/openpilot-tiny.git
+# using this instead to ignore refs/pull/* - since this is also what --mirror does - https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
+git push --prune git@github.com:commaai/openpilot-tiny.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/rewrite.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+git clone --bare https://github.com/commaai/openpilot
+cp -r openpilot.git openpilot_backup
+cd openpilot.git
+
+# backup old repo
+git push git@github.com:commaai/openpilot-archive.git +refs/heads/master:refs/heads/master
+git push git@github.com:commaai/openpilot-archive.git +refs/heads/*:refs/heads/*
+git push git@github.com:commaai/openpilot-archive.git +refs/tags/*:refs/tags/*
+git push --mirror git@github.com:commaai/openpilot-archive.git
+
+# ignore all release branches
+git for-each-ref --format='delete %(refname)' | grep 'dashcam3\|devel\|master-ci\|nightly\|release2\|release3\|release3-staging' | git update-ref --stdin
+
+# re-tag old releases on master
+declare -A TAGS=( ["f8cb04e4a8b032b72a909f68b808a50936184bee"]="v0.9.7" ["0b4d08fab8e35a264bc7383e878538f8083c33e5"]="v0.9.6" ["3b1e9017c560499786d8a0e46aaaeea65037acac"]="v0.9.5" ["fa310d9e2542cf497d92f007baec8fd751ffa99c"]="v0.9.4" ["8704c1ff952b5c85a44f50143bbd1a4f7b4887e2"]="v0.9.3" ["c7d3b28b93faa6c955fb24bc64031512ee985ee9"]="v0.9.2" ["89f68bf0cbf53a81b0553d3816fdbe522f941fa1"]="v0.9.1" ["58b84fb401a804967aa0dd5ee66fafa90194fd30"]="v0.9.0" ["f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e"]="v0.8.16" ["5a7c2f90361e72e9c35e88abd2e11acdc4aba354"]="v0.8.15" ["71901c94dbbaa2f9f156a80c14cc7ea65219fc7c"]="v0.8.14" ["95da47079510afc91665263619e5939126da637c"]="v0.8.13" ["472177e2a8a1d002e56f9096326fd2dff62e54f9"]="v0.8.12" ["08078acbd0b4f7da469c7dff6159000e358974a9"]="v0.8.11" ["687925c775c375495f9827946138a724bde00b9d"]="v0.8.10" ["204e5a090735a059d69c29145a4cee49450da07e"]="v0.8.9" ["4be956f8861ecbb521ef9503a3c87b07c9d36721"]="v0.8.8" ["589f82c76627d634761a31a34b2488403556eb0b"]="v0.8.7" ["507cfc8910f74ddb8810039d68b880b426ff9ff9"]="v0.8.6" ["d47b00b45a866bef088f51d1ff31de5885ab04e9"]="v0.8.5" ["553e7d1cce314e7eb0587186b1764c3ff43bed62"]="v0.8.4" ["9896438d1511602a1ff87f7c4eb3c7172b30104a"]="v0.8.3" ["280192ed1443f112463417c2d815ea8ee2762fbd"]="v0.8.2" ["8039361567e4659eae2a084e6f39f34acadf4cac"]="v0.8.1" ["d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc"]="v0.8" ["3d456e5d0fbf0c9887d0499dee812f2b029edf6d"]="v0.7.10" ["81763a18b5d0e379b749e090ecce36a91fca7c43"]="v0.7.9" ["9bc0b350fd273bbb2deb3dcaef0312944e4f6cfd"]="v0.7.8" ["ede5b632b58c55e4ff003f948efae07fe03c2280"]="v0.7.7" ["775acd11ba2e0a8c2f5a5655338718d796491b36"]="v0.7.6.1" ["302417b4cf0dcf00d45e4995b5410e543ad121d1"]="v0.7.5" ["12ff088b42221dd17d9d97decb1fc61a7cb0a861"]="v0.7.4" ["9563f7730252451fdcba9bc3d9fe36dab9c86a26"]="v0.7.3" ["8321cf283abbc2ca3fda7e0c7a069a77a492fe0c"]="v0.7.2" ["1e1de64a1e59476b7b3d3558b92149246d5c3292"]="v0.7.1" ["a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3"]="v0.7" ["d4eb5a6eafdd4803d09e6f3963918216cca5a81f"]="v0.6.6" ["70d17cd69b80e7627dcad8fd5b6438f2309ac307"]="v0.6.5" ["58f376002e0c654fbc2de127765fa297cf694a33"]="v0.6.4" ["d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3"]="v0.6.3" ["095ef5f9f60fca1b269aabcc3cfd322b17b9e674"]="v0.6.2" ["cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22"]="v0.6.1" ["60a20537c5f3fcc7f11946d81aebc8f90c08c117"]="v0.6" ["dd34ccfe288ebda8e2568cf550994ae890379f45"]="v0.5.13" ["3f9059fea886f1fa3b0c19a62a981d891dcc84eb"]="v0.5.12" ["2f92d577f995ff6ae1945ef6b89df3cb69b92999"]="v0.5.11" ["5a9d89ed42ddcd209d001a10d7eb828ef0e6d9de"]="v0.5.10" ["0207a970400ee28d3e366f2e8f5c551281accf02"]="v0.5.9" ["b967da5fc1f7a07e3561db072dd714d325e857b0"]="v0.5.8" ["210db686bb89f8696aa040e6e16de65424b808c9"]="v0.5.7" ["860a48765d1016ba226fb2c64aea35a45fe40e4a"]="v0.5.6" ["8f3539a27b28851153454eb737da9624cccaed2d"]="v0.5.5" ["a422246dc30bce11e970514f13f7c110f4470cc3"]="v0.5.4" ["285c52eb693265a0a530543e9ca0aeb593a2a55e"]="v0.5.3" ["0129a8a4ff8da5314e8e4d4d3336e89667ff6d54"]="v0.5.2" ["6f3d10a4c475c4c4509f0b370805419acd13912d"]="v0.5.1" ["de33bc46452b1046387ee2b3a03191b2c71135fb"]="v0.5" ["ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9"]="v0.4.7" ["c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1"]="v0.4.6" ["37285038d3f91fa1b49159c4a35a8383168e644f"]="v0.4.5" ["9a9ff839a9b70cb2601d7696af743f5652395389"]="v0.4.4" ["28c0797d30175043bbfa31307b63aab4197cf996"]="v0.4.2" ["4474b9b3718653aeb0aee26422caefb90460cc0e"]="v0.4.1" ["da52d065a4c4f52d6017a537f3a80326f5af8bdc"]="v0.4.0.2" ["9d3963559ae7b15193057937ff3e72481899f40d"]="v0.3.5"  ["1b8c44b5067525a5d266b6e99799d8097da76a29"]="v0.3.4" ["5cf91d0496688fed4f2a6c7021349b1fc0e057a2"]="v0.3.3" ["7fe46f1e1df5dec08a940451ba0feefd5c039165"]="v0.3.2" ["41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24"]="v0.3.1" ["c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8"]="v0.3.0" ["693bcb0f83478f2651db6bac9be5ca5ad60d03f3"]="v0.2.9" ["95a349abcc050712c50d4d85a1c8a804eee7f6c2"]="v0.2.8" ["c6ba5dc5391d3ca6cda479bf1923b88ce45509a0"]="v0.2.7" ["6c3afeec0fb439070b2912978b8dbb659033b1d9"]="v0.2.6" ["29c58b45882ac79595356caf98580c1d2a626011"]="v0.2.5" ["ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0"]="v0.2.4" ["adaa4ed350acda4067fc0b455ad15b54cdf4c768"]="v0.2.3" ["a64b9aa9b8cb5863c917b6926516291a63c02fe5"]="v0.2.2" ["17d9becd3c673091b22f09aa02559a9ed9230f50"]="v0.2.1" ["449b482cc3236ccf31829830b4f6a44b2dcc06c2"]="v0.2" ["e94a30bec07e719c5a7b037ca1f4db8312702cce"]="v0.1" )
+for tag in "${!TAGS[@]}"; do git tag -f "${TAGS[$tag]}" "$tag" ; done
+
+# get master root commit
+ROOT_COMMIT=$(git rev-list --max-parents=0 HEAD | tail -n 1)
+
+# link master and devel
+git replace --graft $ROOT_COMMIT v0.7.1
+git-filter-repo --prune-empty never --force --commit-callback 'h=commit.original_id.decode("utf-8");m=commit.message.decode("utf-8");commit.message=str.encode(m + "\n" + "old-commit-hash: " + h)'
+
+# delete replace refs
+git for-each-ref --format='delete %(refname)' refs/replace | git update-ref --stdin
+
+# machine validation
+tail -n +2 "filter-repo/commit-map" | tr ' ' '\n' | xargs -P $(nproc) -n 2 bash -c 'H1=$(cd ../openpilot_backup && git ls-tree -r $0 | sha1sum) && H2=$(git ls-tree -r $1 | sha1sum) && echo "$H1 $H2" >> /tmp/GIT_HASHES && diff <(echo $H1) <(echo $H2) || exit 255'
+# human validation
+less /tmp/GIT_HASH
+
+# cleanup
+git reflog expire --expire=now --all
+git gc --prune=now --aggressive
+
+# get all lfs files
+set +e
+git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
+git lfs fetch --all
+git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+git lfs fetch --all
+set -e
+
+# add new files to lfs
+git lfs migrate import --everything --include="*.ico,*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,pyextra/**/*,panda/board/**/inc/*.h,panda/board/obj/*.elf,board/inc/*.h,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,*.pro,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,poetry.lock,*.qm"
+
+# set new lfs endpoint
+git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
+
+# push all branch+tag (scary stuff...)
+git push -f --set-upstream git@github.com:commaai/openpilot.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/tag-commit-map.txt

@@ -0,0 +1,82 @@
+v0.1 e94a30bec07e719c5a7b037ca1f4db8312702cce
+v0.2 449b482cc3236ccf31829830b4f6a44b2dcc06c2
+v0.2.1 17d9becd3c673091b22f09aa02559a9ed9230f50
+v0.2.2 a64b9aa9b8cb5863c917b6926516291a63c02fe5
+v0.2.3 adaa4ed350acda4067fc0b455ad15b54cdf4c768
+v0.2.4 ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0
+v0.2.5 29c58b45882ac79595356caf98580c1d2a626011
+v0.2.6 6c3afeec0fb439070b2912978b8dbb659033b1d9
+v0.2.7 c6ba5dc5391d3ca6cda479bf1923b88ce45509a0
+v0.2.8 95a349abcc050712c50d4d85a1c8a804eee7f6c2
+v0.2.9 693bcb0f83478f2651db6bac9be5ca5ad60d03f3
+v0.3.0 c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8
+v0.3.1 41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24
+v0.3.2 7fe46f1e1df5dec08a940451ba0feefd5c039165
+v0.3.3 5cf91d0496688fed4f2a6c7021349b1fc0e057a2
+v0.3.4 1b8c44b5067525a5d266b6e99799d8097da76a29
+v0.3.5 b111277f464cf66fa34b67819a83ea683e0f64df
+v0.4.0.2 da52d065a4c4f52d6017a537f3a80326f5af8bdc
+v0.4.1 4474b9b3718653aeb0aee26422caefb90460cc0e
+v0.4.2 28c0797d30175043bbfa31307b63aab4197cf996
+v0.4.4 9a9ff839a9b70cb2601d7696af743f5652395389
+v0.4.5 37285038d3f91fa1b49159c4a35a8383168e644f
+v0.4.6 c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1
+v0.4.7 ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9
+v0.5 de33bc46452b1046387ee2b3a03191b2c71135fb
+v0.5.1 8f22f52235c48eada586795ac57edb22688e4d08
+v0.5.2 0129a8a4ff8da5314e8e4d4d3336e89667ff6d54
+v0.5.3 285c52eb693265a0a530543e9ca0aeb593a2a55e
+v0.5.4 a422246dc30bce11e970514f13f7c110f4470cc3
+v0.5.5 8f3539a27b28851153454eb737da9624cccaed2d
+v0.5.6 860a48765d1016ba226fb2c64aea35a45fe40e4a
+v0.5.7 9ce3045f139ee29bf0eea5ec59dfe7df9c3d2c51
+v0.5.8 2cee2e05ba0f3824fdbb8b957958800fa99071a1
+v0.5.9 ad145da3bcded0fe75306df02061d07a633963c3
+v0.5.10 ff4c1557d8358f158f4358788ff18ef93d2470ef
+v0.5.11 d1866845df423c6855e2b365ff230cf7d89a420b
+v0.5.12 f6e8ef27546e9a406724841e75f8df71cc4c2c97
+v0.5.13 dd34ccfe288ebda8e2568cf550994ae890379f45
+v0.6 60a20537c5f3fcc7f11946d81aebc8f90c08c117
+v0.6.1 cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22
+v0.6.2 095ef5f9f60fca1b269aabcc3cfd322b17b9e674
+v0.6.3 d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3
+v0.6.4 58f376002e0c654fbc2de127765fa297cf694a33
+v0.6.5 70d17cd69b80e7627dcad8fd5b6438f2309ac307
+v0.6.6 d4eb5a6eafdd4803d09e6f3963918216cca5a81f
+v0.7 a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3
+v0.7.1 1e1de64a1e59476b7b3d3558b92149246d5c3292
+v0.7.2 59bd58c940673b4c4a6a86f299022614bcf42b22
+v0.7.3 d7acd8b68f8131e0e714400cf124a3e228638643
+v0.7.4 e93649882c5e914eec4a8b8b593dc0587e497033
+v0.7.5 8abc0afe464626a461d2c7e192c912eeebeccc65
+v0.7.6 69aacd9d179fe6dd3110253a099c38b34cff7899
+v0.7.7 f1caed7299cdba5e45635d8377da6cc1e5fd7072
+v0.7.8 2189fe8741b635d8394d55dee28959425cfd5ad0
+v0.7.9 86dc54b836a973f132ed26db9f5a60b29f9b25b2
+v0.7.10 47a42ff432db8a2494e922ca5e767e58020f0446
+v0.7.11 f46ed718ba8d6bb4d42cd7b0f0150c406017c373
+v0.8 d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc
+v0.8.1 cd6f26664cb8d32a13847d6648567c47c580e248
+v0.8.2 7cc0999aebfe63b6bb6dd83c1dff62c3915c4820
+v0.8.3 986500fe2f10870018f1fba1e5465476b8915977
+v0.8.4 f0d0b82b8d6f5f450952113e234d0a5a49e80c48
+v0.8.5 f5d9ddc6c2a2802a61e5ce590c6b6688bf736a69
+v0.8.6 75904ed7452c6cbfb2a70cd379a899d8a75b97c2
+v0.8.7 4f9e568019492126e236da85b5ca0a059f292900
+v0.8.8 a949a49d5efaaf2d881143d23e9fb5ff9e28e88c
+v0.8.9 a034926264cd1025c69d6ceb3fe444965f960b75
+v0.8.10 59accdd814398b884167c0f41dbf46dcccf0c29c
+v0.8.11 d630ec9092f039cb5e51c5dd6d92fc47b91407e4
+v0.8.12 57871c99031cf597ffa0d819057ac1401e129f32
+v0.8.13 e43e6e876513450d235124fcb711f1724ed9814c
+v0.8.14 71901c94dbbaa2f9f156a80c14cc7ea65219fc7c
+v0.8.15 5a7c2f90361e72e9c35e88abd2e11acdc4aba354
+v0.8.16 f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e
+v0.9.0 58b84fb401a804967aa0dd5ee66fafa90194fd30
+v0.9.1 89f68bf0cbf53a81b0553d3816fdbe522f941fa1
+v0.9.2 c7d3b28b93faa6c955fb24bc64031512ee985ee9
+v0.9.3 8704c1ff952b5c85a44f50143bbd1a4f7b4887e2
+v0.9.4 fa310d9e2542cf497d92f007baec8fd751ffa99c
+v0.9.5 3b1e9017c560499786d8a0e46aaaeea65037acac
+v0.9.6 0b4d08fab8e35a264bc7383e878538f8083c33e5
+v0.9.7 f8cb04e4a8b032b72a909f68b808a50936184bee

scripts/pyqt_demo.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+from PyQt5.QtWidgets import QApplication, QLabel
+from openpilot.selfdrive.ui.qt.python_helpers import set_main_window
+
+
+if __name__ == "__main__":
+  app = QApplication([])
+  label = QLabel('Hello World!')
+
+  # Set full screen and rotate
+  set_main_window(label)
+
+  app.exec_()

selfdrive/assets/offroad/icon_plus.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa3f1c39a4e82adfb52d43fc0ad6773a70dbaa4fc79109a7d6b6c1f73b298eac
+size 2833

selfdrive/car/car_specific.py

@@ -42,16 +42,19 @@ class CarSpecificEvents:
     self.cruise_buttons: deque = deque([], maxlen=HYUNDAI_PREV_BUTTON_SAMPLES)
 
   def update(self, CS: car.CarState, CS_prev: car.CarState, CC: car.CarControl):
-    if self.CP.brand in ('body', 'mock'):
+    if self.CP.carName in ('body', 'mock'):
       events = Events()
 
-    elif self.CP.brand == 'ford':
+    elif self.CP.carName in ('subaru', 'mazda'):
+      events = self.create_common_events(CS, CS_prev)
+
+    elif self.CP.carName == 'ford':
       events = self.create_common_events(CS, CS_prev, extra_gears=[GearShifter.manumatic])
 
-    elif self.CP.brand == 'nissan':
+    elif self.CP.carName == 'nissan':
       events = self.create_common_events(CS, CS_prev, extra_gears=[GearShifter.brake])
 
-    elif self.CP.brand == 'chrysler':
+    elif self.CP.carName == 'chrysler':
       events = self.create_common_events(CS, CS_prev, extra_gears=[GearShifter.low])
 
       # Low speed steer alert hysteresis logic
@@ -62,7 +65,7 @@ class CarSpecificEvents:
       if self.low_speed_alert:
         events.add(EventName.belowSteerSpeed)
 
-    elif self.CP.brand == 'honda':
+    elif self.CP.carName == 'honda':
       events = self.create_common_events(CS, CS_prev, pcm_enable=False)
 
       if self.CP.pcmCruise and CS.vEgo < self.CP.minEnableSpeed:
@@ -83,7 +86,7 @@ class CarSpecificEvents:
       if self.CP.minEnableSpeed > 0 and CS.vEgo < 0.001:
         events.add(EventName.manualRestart)
 
-    elif self.CP.brand == 'toyota':
+    elif self.CP.carName == 'toyota':
       events = self.create_common_events(CS, CS_prev)
 
       if self.CP.openpilotLongitudinalControl:
@@ -98,10 +101,14 @@ class CarSpecificEvents:
             # while in standstill, send a user alert
             events.add(EventName.manualRestart)
 
-    elif self.CP.brand == 'gm':
+    elif self.CP.carName == 'gm':
+      # The ECM allows enabling on falling edge of set, but only rising edge of resume
       events = self.create_common_events(CS, CS_prev, extra_gears=[GearShifter.sport, GearShifter.low,
                                                                    GearShifter.eco, GearShifter.manumatic],
-                                         pcm_enable=self.CP.pcmCruise)
+                                         pcm_enable=self.CP.pcmCruise, enable_buttons=(ButtonType.decelCruise,))
+      if not self.CP.pcmCruise:
+        if any(b.type == ButtonType.accelCruise and b.pressed for b in CS.buttonEvents):
+          events.add(EventName.buttonEnable)
 
       # Enabling at a standstill with brake is allowed
       # TODO: verify 17 Volt can enable for the first time at a stop and allow for all GMs
@@ -113,9 +120,10 @@ class CarSpecificEvents:
       if CS.vEgo < self.CP.minSteerSpeed:
         events.add(EventName.belowSteerSpeed)
 
-    elif self.CP.brand == 'volkswagen':
+    elif self.CP.carName == 'volkswagen':
       events = self.create_common_events(CS, CS_prev, extra_gears=[GearShifter.eco, GearShifter.sport, GearShifter.manumatic],
-                                         pcm_enable=self.CP.pcmCruise)
+                                         pcm_enable=not self.CP.openpilotLongitudinalControl,
+                                         enable_buttons=(ButtonType.setCruise, ButtonType.resumeCruise))
 
       # Low speed steer alert hysteresis logic
       if (self.CP.minSteerSpeed - 1e-3) > VWCarControllerParams.DEFAULT_MIN_STEER_SPEED and CS.vEgo < (self.CP.minSteerSpeed + 1.):
@@ -135,13 +143,12 @@ class CarSpecificEvents:
       # if CC.eps_timer_soft_disable_alert:  # type: ignore[attr-defined]
       #   events.add(EventName.steerTimeLimit)
 
-    elif self.CP.brand == 'hyundai':
+    elif self.CP.carName == 'hyundai':
       # On some newer model years, the CANCEL button acts as a pause/resume button based on the PCM state
       # To avoid re-engaging when openpilot cancels, check user engagement intention via buttons
       # Main button also can trigger an engagement on these cars
       self.cruise_buttons.append(any(ev.type in HYUNDAI_ENABLE_BUTTONS for ev in CS.buttonEvents))
-      events = self.create_common_events(CS, CS_prev, extra_gears=(GearShifter.sport, GearShifter.manumatic),
-                                         pcm_enable=self.CP.pcmCruise, allow_enable=any(self.cruise_buttons), allow_button_cancel=False)
+      events = self.create_common_events(CS, CS_prev, pcm_enable=self.CP.pcmCruise, allow_enable=any(self.cruise_buttons))
 
       # low speed steer alert hysteresis logic (only for cars with steer cut off above 10 m/s)
       if CS.vEgo < (self.CP.minSteerSpeed + 2.) and self.CP.minSteerSpeed > 10.:
@@ -152,12 +159,12 @@ class CarSpecificEvents:
         events.add(EventName.belowSteerSpeed)
 
     else:
-      events = self.create_common_events(CS, CS_prev)
+      raise ValueError(f"Unsupported car: {self.CP.carName}")
 
     return events
 
   def create_common_events(self, CS: structs.CarState, CS_prev: car.CarState, extra_gears=None, pcm_enable=True,
-                           allow_enable=True, allow_button_cancel=True):
+                           allow_enable=True, enable_buttons=(ButtonType.accelCruise, ButtonType.decelCruise)):
     events = Events()
 
     if CS.doorOpen:
@@ -201,14 +208,14 @@ class CarSpecificEvents:
       events.add(EventName.invalidLkasSetting)
     if CS.lowSpeedAlert:
       events.add(EventName.belowSteerSpeed)
-    if CS.buttonEnable:
-      events.add(EventName.buttonEnable)
 
-    # Handle cancel button presses
+    # Handle button presses
     for b in CS.buttonEvents:
+      # Enable OP long on falling edge of enable buttons (defaults to accelCruise and decelCruise, overridable per-port)
+      if not self.CP.pcmCruise and (b.type in enable_buttons and not b.pressed):
+        events.add(EventName.buttonEnable)
       # Disable on rising and falling edge of cancel for both stock and OP long
-      # TODO: only check the cancel button with openpilot longitudinal on all brands to match panda safety
-      if b.type == ButtonType.cancel and (allow_button_cancel or not self.CP.pcmCruise):
+      if b.type == ButtonType.cancel:
         events.add(EventName.buttonCancel)
 
     # Handle permanent and temporary steering faults

selfdrive/car/card.py

@@ -7,17 +7,17 @@ import cereal.messaging as messaging
 
 from cereal import car, log
 
+from panda import ALTERNATIVE_EXPERIENCE
+
 from openpilot.common.params import Params
 from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper
 from openpilot.common.swaglog import cloudlog, ForwardingHandler
 
-from opendbc.car import DT_CTRL, structs
+from opendbc.car import DT_CTRL, carlog, structs
 from opendbc.car.can_definitions import CanData, CanRecvCallable, CanSendCallable
-from opendbc.car.carlog import carlog
 from opendbc.car.fw_versions import ObdCallback
 from opendbc.car.car_helpers import get_car, get_radar_interface
 from opendbc.car.interfaces import CarInterfaceBase, RadarInterfaceBase
-from opendbc.safety import ALTERNATIVE_EXPERIENCE
 from openpilot.selfdrive.pandad import can_capnp_to_list, can_list_to_can_capnp
 from openpilot.selfdrive.car.cruise import VCruiseHelper
 from openpilot.selfdrive.car.car_specific import MockCarState
@@ -72,7 +72,6 @@ class Car:
     self.can_rcv_cum_timeout_counter = 0
 
     self.CC_prev = car.CarControl.new_message()
-    self.CS_prev = car.CarState.new_message()
     self.initialized_prev = False
 
     self.last_actuators_output = structs.CarControl.Actuators()
@@ -125,15 +124,6 @@ class Car:
       self.CP.safetyConfigs = [safety_config]
 
     if self.CP.secOcRequired and not self.params.get_bool("IsReleaseBranch"):
-      # Copy user key if available
-      try:
-        with open("/cache/params/SecOCKey") as f:
-          user_key = f.readline().strip()
-          if len(user_key) == 32:
-            self.params.put("SecOCKey", user_key)
-      except Exception:
-        pass
-
       secoc_key = self.params.get("SecOCKey", encoding='utf8')
       if secoc_key is not None:
         saved_secoc_key = bytes.fromhex(secoc_key.strip())
@@ -173,7 +163,7 @@ class Car:
 
     # Update carState from CAN
     CS = self.CI.update(can_list)
-    if self.CP.brand == 'mock':
+    if self.CP.carName == 'mock':
       CS = self.mock_carstate.update(CS)
 
     # Update radar tracks from CAN
@@ -190,12 +180,8 @@ class Car:
     if can_rcv_valid and REPLAY:
       self.can_log_mono_time = messaging.log_from_bytes(can_strs[0]).logMonoTime
 
-    self.v_cruise_helper.update_v_cruise(CS, self.sm['carControl'].enabled, self.is_metric)
-    if self.sm['carControl'].enabled and not self.CC_prev.enabled:
-      # Use CarState w/ buttons from the step selfdrived enables on
-      self.v_cruise_helper.initialize_v_cruise(self.CS_prev, self.experimental_mode)
-
     # TODO: mirror the carState.cruiseState struct?
+    self.v_cruise_helper.update_v_cruise(CS, self.sm['carControl'].enabled, self.is_metric)
     CS.vCruise = float(self.v_cruise_helper.v_cruise_kph)
     CS.vCruiseCluster = float(self.v_cruise_helper.v_cruise_cluster_kph)
 
@@ -252,6 +238,9 @@ class Car:
   def step(self):
     CS, RD = self.state_update()
 
+    if self.sm['carControl'].enabled and not self.CC_prev.enabled:
+      self.v_cruise_helper.initialize_v_cruise(CS, self.experimental_mode)
+
     self.state_publish(CS, RD)
 
     initialized = (not any(e.name == EventName.selfdriveInitializing for e in self.sm['onroadEvents']) and
@@ -260,7 +249,6 @@ class Car:
       self.controls_update(CS, self.sm['carControl'])
 
     self.initialized_prev = initialized
-    self.CS_prev = CS
 
   def params_thread(self, evt):
     while not evt.is_set():

selfdrive/car/cruise.py

@@ -1,8 +1,8 @@
 import math
-import numpy as np
 
 from cereal import car
 from openpilot.common.conversions import Conversions as CV
+from openpilot.common.numpy_fast import clip
 
 
 # WARNING: this value was determined based on the model's training distribution,
@@ -106,7 +106,7 @@ class VCruiseHelper:
     if CS.gasPressed and button_type in (ButtonType.decelCruise, ButtonType.setCruise):
       self.v_cruise_kph = max(self.v_cruise_kph, CS.vEgo * CV.MS_TO_KPH)
 
-    self.v_cruise_kph = np.clip(round(self.v_cruise_kph, 1), V_CRUISE_MIN, V_CRUISE_MAX)
+    self.v_cruise_kph = clip(round(self.v_cruise_kph, 1), V_CRUISE_MIN, V_CRUISE_MAX)
 
   def update_button_timers(self, CS, enabled):
     # increment timer for buttons still pressed
@@ -130,6 +130,6 @@ class VCruiseHelper:
     if any(b.type in (ButtonType.accelCruise, ButtonType.resumeCruise) for b in CS.buttonEvents) and self.v_cruise_initialized:
       self.v_cruise_kph = self.v_cruise_kph_last
     else:
-      self.v_cruise_kph = int(round(np.clip(CS.vEgo * CV.MS_TO_KPH, initial, V_CRUISE_MAX)))
+      self.v_cruise_kph = int(round(clip(CS.vEgo * CV.MS_TO_KPH, initial, V_CRUISE_MAX)))
 
     self.v_cruise_cluster_kph = self.v_cruise_kph

selfdrive/car/tests/test_models.py

@@ -1,4 +1,3 @@
-import time
 import capnp
 import os
 import pytest
@@ -10,15 +9,17 @@ import hypothesis.strategies as st
 from hypothesis import Phase, given, settings
 from parameterized import parameterized_class
 
-from opendbc.car import DT_CTRL, gen_empty_fingerprint, structs
-from opendbc.car.can_definitions import CanData
-from opendbc.car.car_helpers import FRAME_FINGERPRINT, interfaces
-from opendbc.car.fingerprints import all_known_cars, MIGRATION
-from opendbc.car.honda.values import CAR as HONDA, HondaFlags
-from opendbc.car.structs import car
-from opendbc.car.tests.routes import non_tested_cars, routes, CarTestRoute
-from opendbc.car.values import Platform
+from cereal import messaging, log, car
 from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+from opendbc.car import DT_CTRL, gen_empty_fingerprint, structs
+from opendbc.car.fingerprints import all_known_cars, MIGRATION
+from opendbc.car.car_helpers import FRAME_FINGERPRINT, interfaces
+from opendbc.car.honda.values import CAR as HONDA, HondaFlags
+from opendbc.car.values import Platform
+from opendbc.car.tests.routes import non_tested_cars, routes, CarTestRoute
+from openpilot.selfdrive.selfdrived.events import ET
+from openpilot.selfdrive.selfdrived.selfdrived import SelfdriveD
 from openpilot.selfdrive.pandad import can_capnp_to_list
 from openpilot.selfdrive.test.helpers import read_segment_list
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
@@ -28,6 +29,8 @@ from openpilot.tools.lib.route import SegmentName
 
 from panda.tests.libpanda import libpanda_py
 
+EventName = log.OnroadEvent.EventName
+PandaType = log.PandaState.PandaType
 SafetyModel = car.CarParams.SafetyModel
 
 NUM_JOBS = int(os.environ.get("NUM_JOBS", "1"))
@@ -168,6 +171,8 @@ class TestCarModelBase(unittest.TestCase):
     self.CI = self.CarInterface(self.CP.copy(), self.CarController, self.CarState)
     assert self.CI
 
+    Params().put_bool("OpenpilotEnabledToggle", self.openpilot_enabled)
+
     # TODO: check safetyModel is in release panda build
     self.safety = libpanda_py.libpanda
 
@@ -334,8 +339,10 @@ class TestCarModelBase(unittest.TestCase):
       to_send = libpanda_py.make_CANPacket(address, bus, dat)
       self.safety.safety_rx_hook(to_send)
 
-      can = [(int(time.monotonic() * 1e9), [CanData(address=address, dat=dat, src=bus)])]
-      CS = self.CI.update(can)
+      can = messaging.new_message('can', 1)
+      can.can = [log.CanData(address=address, dat=dat, src=bus)]
+
+      CS = self.CI.update(can_capnp_to_list((can.to_bytes(),)))
 
       if self.safety.get_gas_pressed_prev() != prev_panda_gas:
         self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev())
@@ -355,11 +362,11 @@ class TestCarModelBase(unittest.TestCase):
       if self.safety.get_vehicle_moving() != prev_panda_vehicle_moving:
         self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving())
 
-      if not (self.CP.brand == "honda" and not (self.CP.flags & HondaFlags.BOSCH)):
+      if not (self.CP.carName == "honda" and not (self.CP.flags & HondaFlags.BOSCH)):
         if self.safety.get_cruise_engaged_prev() != prev_panda_cruise_engaged:
           self.assertEqual(CS.cruiseState.enabled, self.safety.get_cruise_engaged_prev())
 
-      if self.CP.brand == "honda":
+      if self.CP.carName == "honda":
         if self.safety.get_acc_main_on() != prev_panda_acc_main_on:
           self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on())
 
@@ -380,6 +387,8 @@ class TestCarModelBase(unittest.TestCase):
     controls_allowed_prev = False
     CS_prev = car.CarState.new_message()
     checks = defaultdict(int)
+    selfdrived = SelfdriveD(CP=self.CP)
+    selfdrived.initialized = True
     for idx, can in enumerate(self.can_msgs):
       CS = self.CI.update(can_capnp_to_list((can.as_builder().to_bytes(), ))).as_reader()
       for msg in filter(lambda m: m.src in range(64), can.can):
@@ -412,7 +421,7 @@ class TestCarModelBase(unittest.TestCase):
         # On most pcmCruise cars, openpilot's state is always tied to the PCM's cruise state.
         # On Honda Nidec, we always engage on the rising edge of the PCM cruise state, but
         # openpilot brakes to zero even if the min ACC speed is non-zero (i.e. the PCM disengages).
-        if self.CP.brand == "honda" and not (self.CP.flags & HondaFlags.BOSCH):
+        if self.CP.carName == "honda" and not (self.CP.flags & HondaFlags.BOSCH):
           # only the rising edges are expected to match
           if CS.cruiseState.enabled and not CS_prev.cruiseState.enabled:
             checks['controlsAllowed'] += not self.safety.get_controls_allowed()
@@ -423,15 +432,18 @@ class TestCarModelBase(unittest.TestCase):
         if not self.CP.notCar:
           checks['cruiseState'] += CS.cruiseState.enabled != self.safety.get_cruise_engaged_prev()
       else:
-        # Check for user button enable on rising edge of controls allowed
-        button_enable = CS.buttonEnable and (not CS.brakePressed or CS.standstill)
+        # Check for enable events on rising edge of controls allowed
+        selfdrived.update_events(CS)
+        selfdrived.CS_prev = CS
+        button_enable = (selfdrived.events.contains(ET.ENABLE) and
+                         EventName.pedalPressed not in selfdrived.events.names)
         mismatch = button_enable != (self.safety.get_controls_allowed() and not controls_allowed_prev)
         checks['controlsAllowed'] += mismatch
         controls_allowed_prev = self.safety.get_controls_allowed()
         if button_enable and not mismatch:
           self.safety.set_controls_allowed(False)
 
-      if self.CP.brand == "honda":
+      if self.CP.carName == "honda":
         checks['mainOn'] += CS.cruiseState.available != self.safety.get_acc_main_on()
 
       CS_prev = CS

selfdrive/controls/controlsd.py

@@ -106,16 +106,15 @@ class Controls:
 
     # accel PID loop
     pid_accel_limits = self.CI.get_pid_accel_limits(self.CP, CS.vEgo, CS.vCruise * CV.KPH_TO_MS)
-    actuators.accel = float(self.LoC.update(CC.longActive, CS, long_plan.aTarget, long_plan.shouldStop, pid_accel_limits))
+    actuators.accel = self.LoC.update(CC.longActive, CS, long_plan.aTarget, long_plan.shouldStop, pid_accel_limits)
 
     # Steering PID loop and lateral MPC
     self.desired_curvature = clip_curvature(CS.vEgo, self.desired_curvature, model_v2.action.desiredCurvature)
-    actuators.curvature = float(self.desired_curvature)
-    steer, steeringAngleDeg, lac_log = self.LaC.update(CC.latActive, CS, self.VM, lp,
+    actuators.curvature = self.desired_curvature
+    actuators.steer, actuators.steeringAngleDeg, lac_log = self.LaC.update(CC.latActive, CS, self.VM, lp,
                                                                             self.steer_limited, self.desired_curvature,
                                                                             self.calibrated_pose) # TODO what if not available
-    actuators.steer = float(steer)
-    actuators.steeringAngleDeg = float(steeringAngleDeg)
+
     # Ensure no NaNs/Infs
     for p in ACTUATOR_FIELDS:
       attr = getattr(actuators, p)
@@ -180,7 +179,7 @@ class Controls:
 
     cs.longitudinalPlanMonoTime = self.sm.logMonoTime['longitudinalPlan']
     cs.lateralPlanMonoTime = self.sm.logMonoTime['modelV2']
-    cs.desiredCurvature = float(self.desired_curvature)
+    cs.desiredCurvature = self.desired_curvature
     cs.longControlState = self.LoC.long_control_state
     cs.upAccelCmd = float(self.LoC.pid.p)
     cs.uiAccelCmd = float(self.LoC.pid.i)

selfdrive/controls/lib/drive_helpers.py

@@ -1,22 +1,19 @@
-import numpy as np
 from cereal import log
+from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_CTRL
 
 MIN_SPEED = 1.0
 CONTROL_N = 17
 CAR_ROTATION_RADIUS = 0.0
-# This is a turn radius smaller than most cars can achieve
-MAX_CURVATURE = 0.2
 
 # EU guidelines
 MAX_LATERAL_JERK = 5.0
 MAX_VEL_ERR = 5.0
 
 def clip_curvature(v_ego, prev_curvature, new_curvature):
-  new_curvature = np.clip(new_curvature, -MAX_CURVATURE, MAX_CURVATURE)
   v_ego = max(MIN_SPEED, v_ego)
   max_curvature_rate = MAX_LATERAL_JERK / (v_ego**2) # inexact calculation, check https://github.com/commaai/openpilot/pull/24755
-  safe_desired_curvature = np.clip(new_curvature,
+  safe_desired_curvature = clip(new_curvature,
                                 prev_curvature - max_curvature_rate * DT_CTRL,
                                 prev_curvature + max_curvature_rate * DT_CTRL)
 
@@ -26,6 +23,6 @@ def clip_curvature(v_ego, prev_curvature, new_curvature):
 def get_speed_error(modelV2: log.ModelDataV2, v_ego: float) -> float:
   # ToDo: Try relative error, and absolute speed
   if len(modelV2.temporalPose.trans):
-    vel_err = np.clip(modelV2.temporalPose.trans[0] - v_ego, -MAX_VEL_ERR, MAX_VEL_ERR)
+    vel_err = clip(modelV2.temporalPose.trans[0] - v_ego, -MAX_VEL_ERR, MAX_VEL_ERR)
     return float(vel_err)
   return 0.0

selfdrive/controls/lib/latcontrol.py

@@ -1,6 +1,6 @@
-import numpy as np
 from abc import abstractmethod, ABC
 
+from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_CTRL
 
 MIN_LATERAL_CONTROL_SPEED = 0.3  # m/s
@@ -28,5 +28,5 @@ class LatControl(ABC):
       self.sat_count += self.sat_count_rate
     else:
       self.sat_count -= self.sat_count_rate
-    self.sat_count = np.clip(self.sat_count, 0.0, self.sat_limit)
+    self.sat_count = clip(self.sat_count, 0.0, self.sat_limit)
     return self.sat_count > (self.sat_limit - 1e-3)

selfdrive/controls/lib/latcontrol_angle.py

@@ -23,7 +23,7 @@ class LatControlAngle(LatControl):
       angle_steers_des += params.angleOffsetDeg
 
     angle_control_saturated = abs(angle_steers_des - CS.steeringAngleDeg) > STEER_ANGLE_SATURATION_THRESHOLD
-    angle_log.saturated = bool(self._check_saturation(angle_control_saturated, CS, False))
+    angle_log.saturated = self._check_saturation(angle_control_saturated, CS, False)
     angle_log.steeringAngleDeg = float(CS.steeringAngleDeg)
     angle_log.steeringAngleDesiredDeg = angle_steers_des
     return 0, float(angle_steers_des), angle_log

selfdrive/controls/lib/latcontrol_pid.py

@@ -39,10 +39,10 @@ class LatControlPID(LatControl):
       output_steer = self.pid.update(error, override=CS.steeringPressed,
                                      feedforward=steer_feedforward, speed=CS.vEgo)
       pid_log.active = True
-      pid_log.p = float(self.pid.p)
-      pid_log.i = float(self.pid.i)
-      pid_log.f = float(self.pid.f)
-      pid_log.output = float(output_steer)
-      pid_log.saturated = bool(self._check_saturation(self.steer_max - abs(output_steer) < 1e-3, CS, steer_limited))
+      pid_log.p = self.pid.p
+      pid_log.i = self.pid.i
+      pid_log.f = self.pid.f
+      pid_log.output = output_steer
+      pid_log.saturated = self._check_saturation(self.steer_max - abs(output_steer) < 1e-3, CS, steer_limited)
 
     return output_steer, angle_steers_des, pid_log

selfdrive/controls/lib/latcontrol_torque.py

@@ -1,8 +1,8 @@
 import math
-import numpy as np
 
 from cereal import log
 from opendbc.car.interfaces import LatControlInputs
+from openpilot.common.numpy_fast import interp
 from openpilot.selfdrive.controls.lib.latcontrol import LatControl
 from openpilot.common.pid import PIDController
 from openpilot.selfdrive.controls.lib.vehicle_model import ACCELERATION_DUE_TO_GRAVITY
@@ -51,7 +51,7 @@ class LatControlTorque(LatControl):
       else:
         assert calibrated_pose is not None
         actual_curvature_pose = calibrated_pose.angular_velocity.yaw / CS.vEgo
-        actual_curvature = np.interp(CS.vEgo, [2.0, 5.0], [actual_curvature_vm, actual_curvature_pose])
+        actual_curvature = interp(CS.vEgo, [2.0, 5.0], [actual_curvature_vm, actual_curvature_pose])
         curvature_deadzone = 0.0
       desired_lateral_accel = desired_curvature * CS.vEgo ** 2
 
@@ -60,7 +60,7 @@ class LatControlTorque(LatControl):
       actual_lateral_accel = actual_curvature * CS.vEgo ** 2
       lateral_accel_deadzone = curvature_deadzone * CS.vEgo ** 2
 
-      low_speed_factor = np.interp(CS.vEgo, LOW_SPEED_X, LOW_SPEED_Y)**2
+      low_speed_factor = interp(CS.vEgo, LOW_SPEED_X, LOW_SPEED_Y)**2
       setpoint = desired_lateral_accel + low_speed_factor * desired_curvature
       measurement = actual_lateral_accel + low_speed_factor * actual_curvature
       gravity_adjusted_lateral_accel = desired_lateral_accel - roll_compensation
@@ -68,7 +68,7 @@ class LatControlTorque(LatControl):
                                                             setpoint, lateral_accel_deadzone, friction_compensation=False, gravity_adjusted=False)
       torque_from_measurement = self.torque_from_lateral_accel(LatControlInputs(measurement, roll_compensation, CS.vEgo, CS.aEgo), self.torque_params,
                                                                measurement, lateral_accel_deadzone, friction_compensation=False, gravity_adjusted=False)
-      pid_log.error = float(torque_from_setpoint - torque_from_measurement)
+      pid_log.error = torque_from_setpoint - torque_from_measurement
       ff = self.torque_from_lateral_accel(LatControlInputs(gravity_adjusted_lateral_accel, roll_compensation, CS.vEgo, CS.aEgo), self.torque_params,
                                           desired_lateral_accel - actual_lateral_accel, lateral_accel_deadzone, friction_compensation=True,
                                           gravity_adjusted=True)
@@ -80,14 +80,14 @@ class LatControlTorque(LatControl):
                                       freeze_integrator=freeze_integrator)
 
       pid_log.active = True
-      pid_log.p = float(self.pid.p)
-      pid_log.i = float(self.pid.i)
-      pid_log.d = float(self.pid.d)
-      pid_log.f = float(self.pid.f)
-      pid_log.output = float(-output_torque)
-      pid_log.actualLateralAccel = float(actual_lateral_accel)
-      pid_log.desiredLateralAccel = float(desired_lateral_accel)
-      pid_log.saturated = bool(self._check_saturation(self.steer_max - abs(output_torque) < 1e-3, CS, steer_limited))
+      pid_log.p = self.pid.p
+      pid_log.i = self.pid.i
+      pid_log.d = self.pid.d
+      pid_log.f = self.pid.f
+      pid_log.output = -output_torque
+      pid_log.actualLateralAccel = actual_lateral_accel
+      pid_log.desiredLateralAccel = desired_lateral_accel
+      pid_log.saturated = self._check_saturation(self.steer_max - abs(output_torque) < 1e-3, CS, steer_limited)
 
     # TODO left is positive in this convention
     return -output_torque, 0.0, pid_log

selfdrive/controls/lib/longcontrol.py

@@ -1,5 +1,5 @@
-import numpy as np
 from cereal import car
+from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_CTRL
 from openpilot.selfdrive.controls.lib.drive_helpers import CONTROL_N
 from openpilot.common.pid import PIDController
@@ -84,5 +84,5 @@ class LongControl:
       output_accel = self.pid.update(error, speed=CS.vEgo,
                                      feedforward=a_target)
 
-    self.last_output_accel = np.clip(output_accel, accel_limits[0], accel_limits[1])
+    self.last_output_accel = clip(output_accel, accel_limits[0], accel_limits[1])
     return self.last_output_accel

selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py

@@ -4,6 +4,7 @@ import time
 import numpy as np
 from cereal import log
 from opendbc.car.interfaces import ACCEL_MIN
+from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_MDL
 from openpilot.common.swaglog import cloudlog
 # WARNING: imports outside of constants will not trigger a rebuild
@@ -319,9 +320,9 @@ class LongitudinalMpc:
     # MPC will not converge if immediate crash is expected
     # Clip lead distance to what is still possible to brake for
     min_x_lead = ((v_ego + v_lead)/2) * (v_ego - v_lead) / (-ACCEL_MIN * 2)
-    x_lead = np.clip(x_lead, min_x_lead, 1e8)
-    v_lead = np.clip(v_lead, 0.0, 1e8)
-    a_lead = np.clip(a_lead, -10., 5.)
+    x_lead = clip(x_lead, min_x_lead, 1e8)
+    v_lead = clip(v_lead, 0.0, 1e8)
+    a_lead = clip(a_lead, -10., 5.)
     lead_xv = self.extrapolate_lead(x_lead, v_lead, a_lead, a_lead_tau)
     return lead_xv
 

selfdrive/controls/lib/longitudinal_planner.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import math
 import numpy as np
+from openpilot.common.numpy_fast import clip, interp
 
 import cereal.messaging as messaging
 from opendbc.car.interfaces import ACCEL_MIN, ACCEL_MAX
@@ -29,7 +30,7 @@ _A_TOTAL_MAX_BP = [20., 40.]
 
 
 def get_max_accel(v_ego):
-  return np.interp(v_ego, A_CRUISE_MAX_BP, A_CRUISE_MAX_VALS)
+  return interp(v_ego, A_CRUISE_MAX_BP, A_CRUISE_MAX_VALS)
 
 def get_coast_accel(pitch):
   return np.sin(pitch) * -5.65 - 0.3  # fitted from data using xx/projects/allow_throttle/compute_coast_accel.py
@@ -42,27 +43,31 @@ def limit_accel_in_turns(v_ego, angle_steers, a_target, CP):
   """
   # FIXME: This function to calculate lateral accel is incorrect and should use the VehicleModel
   # The lookup table for turns should also be updated if we do this
-  a_total_max = np.interp(v_ego, _A_TOTAL_MAX_BP, _A_TOTAL_MAX_V)
+  a_total_max = interp(v_ego, _A_TOTAL_MAX_BP, _A_TOTAL_MAX_V)
   a_y = v_ego ** 2 * angle_steers * CV.DEG_TO_RAD / (CP.steerRatio * CP.wheelbase)
   a_x_allowed = math.sqrt(max(a_total_max ** 2 - a_y ** 2, 0.))
 
   return [a_target[0], min(a_target[1], a_x_allowed)]
 
 
-def get_accel_from_plan(speeds, accels, action_t=DT_MDL, vEgoStopping=0.05):
+def get_accel_from_plan(CP, speeds, accels):
   if len(speeds) == CONTROL_N:
-    v_now = speeds[0]
-    a_now = accels[0]
+    v_target_now = interp(DT_MDL, CONTROL_N_T_IDX, speeds)
+    a_target_now = interp(DT_MDL, CONTROL_N_T_IDX, accels)
 
-    v_target = np.interp(action_t, CONTROL_N_T_IDX, speeds)
-    a_target = 2 * (v_target - v_now) / (action_t) - a_now
-    v_target_1sec = np.interp(action_t + 1.0, CONTROL_N_T_IDX, speeds)
+    v_target = interp(CP.longitudinalActuatorDelay + DT_MDL, CONTROL_N_T_IDX, speeds)
+    if v_target != v_target_now:
+      a_target = 2 * (v_target - v_target_now) / CP.longitudinalActuatorDelay - a_target_now
+    else:
+      a_target = a_target_now
+
+    v_target_1sec = interp(CP.longitudinalActuatorDelay + DT_MDL + 1.0, CONTROL_N_T_IDX, speeds)
   else:
     v_target = 0.0
     v_target_1sec = 0.0
     a_target = 0.0
-  should_stop = (v_target < vEgoStopping and
-                 v_target_1sec < vEgoStopping)
+  should_stop = (v_target < CP.vEgoStopping and
+                 v_target_1sec < CP.vEgoStopping)
   return a_target, should_stop
 
 
@@ -138,7 +143,7 @@ class LongitudinalPlanner:
     if reset_state:
       self.v_desired_filter.x = v_ego
       # Clip aEgo to cruise limits to prevent large accelerations when becoming active
-      self.a_desired = np.clip(sm['carState'].aEgo, accel_limits[0], accel_limits[1])
+      self.a_desired = clip(sm['carState'].aEgo, accel_limits[0], accel_limits[1])
 
     # Prevent divergence, smooth in current v_ego
     self.v_desired_filter.x = max(0.0, self.v_desired_filter.update(v_ego))
@@ -150,7 +155,7 @@ class LongitudinalPlanner:
 
     if not self.allow_throttle:
       clipped_accel_coast = max(accel_coast, accel_limits_turns[0])
-      clipped_accel_coast_interp = np.interp(v_ego, [MIN_ALLOW_THROTTLE_SPEED, MIN_ALLOW_THROTTLE_SPEED*2], [accel_limits_turns[1], clipped_accel_coast])
+      clipped_accel_coast_interp = interp(v_ego, [MIN_ALLOW_THROTTLE_SPEED, MIN_ALLOW_THROTTLE_SPEED*2], [accel_limits_turns[1], clipped_accel_coast])
       accel_limits_turns[1] = min(accel_limits_turns[1], clipped_accel_coast_interp)
 
     if force_slow_decel:
@@ -175,7 +180,7 @@ class LongitudinalPlanner:
 
     # Interpolate 0.05 seconds and save as starting point for next iteration
     a_prev = self.a_desired
-    self.a_desired = float(np.interp(self.dt, CONTROL_N_T_IDX, self.a_desired_trajectory))
+    self.a_desired = float(interp(self.dt, CONTROL_N_T_IDX, self.a_desired_trajectory))
     self.v_desired_filter.x = self.v_desired_filter.x + self.dt * (self.a_desired + a_prev) / 2.0
 
   def publish(self, sm, pm):
@@ -196,12 +201,10 @@ class LongitudinalPlanner:
     longitudinalPlan.longitudinalPlanSource = self.mpc.source
     longitudinalPlan.fcw = self.fcw
 
-    action_t =  self.CP.longitudinalActuatorDelay + DT_MDL
-    a_target, should_stop = get_accel_from_plan(longitudinalPlan.speeds, longitudinalPlan.accels,
-                                                action_t=action_t, vEgoStopping=self.CP.vEgoStopping)
-    longitudinalPlan.aTarget = float(a_target)
-    longitudinalPlan.shouldStop = bool(should_stop)
+    a_target, should_stop = get_accel_from_plan(self.CP, longitudinalPlan.speeds, longitudinalPlan.accels)
+    longitudinalPlan.aTarget = a_target
+    longitudinalPlan.shouldStop = should_stop
     longitudinalPlan.allowBrake = True
-    longitudinalPlan.allowThrottle = bool(self.allow_throttle)
+    longitudinalPlan.allowThrottle = self.allow_throttle
 
     pm.send('longitudinalPlan', plan_send)

selfdrive/controls/plannerd.py

@@ -14,7 +14,7 @@ def main():
   cloudlog.info("plannerd is waiting for CarParams")
   params = Params()
   CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
-  cloudlog.info("plannerd got CarParams: %s", CP.brand)
+  cloudlog.info("plannerd got CarParams: %s", CP.carName)
 
   ldw = LaneDepartureWarning()
   longitudinal_planner = LongitudinalPlanner(CP)

selfdrive/controls/radard.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 import math
-import numpy as np
 from collections import deque
 from typing import Any
 
 import capnp
 from cereal import messaging, log, car
+from openpilot.common.numpy_fast import interp
 from openpilot.common.params import Params
 from openpilot.common.realtime import DT_MDL, Priority, config_realtime_process
 from openpilot.common.swaglog import cloudlog
@@ -44,7 +44,7 @@ class KalmanParams:
           0.28144091, 0.27958406, 0.27783249, 0.27617149, 0.27458948, 0.27307714,
           0.27162685, 0.27023228, 0.26888809, 0.26758976, 0.26633338, 0.26511557,
           0.26393339, 0.26278425]
-    self.K = [[np.interp(dt, dts, K0)], [np.interp(dt, dts, K1)]]
+    self.K = [[interp(dt, dts, K0)], [interp(dt, dts, K1)]]
 
 
 class Track:

selfdrive/debug/adb.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+PORT=5555
+
+setprop service.adb.tcp.port $PORT
+sudo systemctl start adbd
+
+IP=$(echo $SSH_CONNECTION | awk '{ print $3}')
+echo "then, connect on your computer:"
+echo "adb connect $IP:$PORT"

selfdrive/debug/car/disable_ecu.py

@@ -11,5 +11,5 @@ if __name__ == "__main__":
   time.sleep(1)
 
   # honda bosch radar disable
-  disabled = disable_ecu(*can_callbacks, bus=1, addr=0x18DAB0F1, com_cont_req=b'\x28\x83\x03', timeout=0.5)
+  disabled = disable_ecu(*can_callbacks, bus=1, addr=0x18DAB0F1, com_cont_req=b'\x28\x83\x03', timeout=0.5, debug=False)
   print(f"disabled: {disabled}")

selfdrive/debug/car/ecu_addrs.py

@@ -2,7 +2,6 @@
 import argparse
 import time
 import cereal.messaging as messaging
-from opendbc.car.carlog import carlog
 from opendbc.car.ecu_addrs import get_all_ecu_addrs
 from openpilot.common.params import Params
 from openpilot.selfdrive.car.card import can_comm_callbacks, obd_callback
@@ -16,9 +15,6 @@ if __name__ == "__main__":
   parser.add_argument('--timeout', type=float, default=1.0)
   args = parser.parse_args()
 
-  if args.debug:
-    carlog.setLevel('DEBUG')
-
   logcan = messaging.sub_sock('can')
   sendcan = messaging.pub_sock('sendcan')
   can_callbacks = can_comm_callbacks(logcan, sendcan)
@@ -33,7 +29,7 @@ if __name__ == "__main__":
   obd_callback(params)(not args.no_obd)
 
   print("Getting ECU addresses ...")
-  ecu_addrs = get_all_ecu_addrs(*can_callbacks, args.bus, args.timeout)
+  ecu_addrs = get_all_ecu_addrs(*can_callbacks, args.bus, args.timeout, debug=args.debug)
 
   print()
   print("Found ECUs on rx addresses:")

selfdrive/debug/car/fw_versions.py

@@ -3,7 +3,6 @@ import time
 import argparse
 import cereal.messaging as messaging
 from cereal import car
-from opendbc.car.carlog import carlog
 from opendbc.car.fw_versions import get_fw_versions, match_fw_to_car
 from opendbc.car.vin import get_vin
 from openpilot.common.params import Params
@@ -19,9 +18,6 @@ if __name__ == "__main__":
   parser.add_argument('--brand', help='Only query addresses/with requests for this brand')
   args = parser.parse_args()
 
-  if args.debug:
-    carlog.setLevel('DEBUG')
-
   logcan = messaging.sub_sock('can')
   pandaStates_sock = messaging.sub_sock('pandaStates')
   sendcan = messaging.pub_sock('sendcan')
@@ -50,13 +46,13 @@ if __name__ == "__main__":
   t = time.time()
   print("Getting vin...")
   set_obd_multiplexing(True)
-  vin_rx_addr, vin_rx_bus, vin = get_vin(*can_callbacks, (0, 1))
+  vin_rx_addr, vin_rx_bus, vin = get_vin(*can_callbacks, (0, 1), debug=args.debug)
   print(f'RX: {hex(vin_rx_addr)}, BUS: {vin_rx_bus}, VIN: {vin}')
   print(f"Getting VIN took {time.time() - t:.3f} s")
   print()
 
   t = time.time()
-  fw_vers = get_fw_versions(*can_callbacks, set_obd_multiplexing, query_brand=args.brand, extra=extra, num_pandas=num_pandas, progress=True)
+  fw_vers = get_fw_versions(*can_callbacks, set_obd_multiplexing, query_brand=args.brand, extra=extra, num_pandas=num_pandas, debug=args.debug, progress=True)
   _, candidates = match_fw_to_car(fw_vers, vin)
 
   print()

selfdrive/debug/car/vin.py

@@ -2,7 +2,6 @@
 import argparse
 import time
 import cereal.messaging as messaging
-from opendbc.car.carlog import carlog
 from opendbc.car.vin import get_vin
 from openpilot.selfdrive.car.card import can_comm_callbacks
 
@@ -14,13 +13,10 @@ if __name__ == "__main__":
   parser.add_argument('--retry', type=int, default=5)
   args = parser.parse_args()
 
-  if args.debug:
-    carlog.setLevel('DEBUG')
-
   sendcan = messaging.pub_sock('sendcan')
   logcan = messaging.sub_sock('can')
   can_callbacks = can_comm_callbacks(logcan, sendcan)
   time.sleep(1)
 
-  vin_rx_addr, vin_rx_bus, vin = get_vin(*can_callbacks, (args.bus,), args.timeout, args.retry)
+  vin_rx_addr, vin_rx_bus, vin = get_vin(*can_callbacks, (args.bus,), args.timeout, args.retry, debug=args.debug)
   print(f'RX: {hex(vin_rx_addr)}, BUS: {vin_rx_bus}, VIN: {vin}')

selfdrive/debug/clear_dtc.py

@@ -2,10 +2,8 @@
 import sys
 import argparse
 from subprocess import check_output, CalledProcessError
-from opendbc.car.carlog import carlog
-from opendbc.car.uds import UdsClient, MessageTimeoutError, SESSION_TYPE, DTC_GROUP_TYPE
-from opendbc.safety import Safety
 from panda import Panda
+from panda.python.uds import UdsClient, MessageTimeoutError, SESSION_TYPE, DTC_GROUP_TYPE
 
 parser = argparse.ArgumentParser(description="clear DTC status")
 parser.add_argument("addr", type=lambda x: int(x,0), nargs="?", default=0x7DF) # default is functional (broadcast) address
@@ -13,9 +11,6 @@ parser.add_argument("--bus", type=int, default=0)
 parser.add_argument('--debug', action='store_true')
 args = parser.parse_args()
 
-if args.debug:
-  carlog.setLevel('DEBUG')
-
 try:
   check_output(["pidof", "pandad"])
   print("pandad is running, please kill openpilot before running this script! (aborted)")
@@ -25,8 +20,8 @@ except CalledProcessError as e:
     raise e
 
 panda = Panda()
-panda.set_safety_mode(Safety.SAFETY_ELM327)
-uds_client = UdsClient(panda, args.addr, bus=args.bus)
+panda.set_safety_mode(Panda.SAFETY_ELM327)
+uds_client = UdsClient(panda, args.addr, bus=args.bus, debug=args.debug)
 print("extended diagnostic session ...")
 try:
   uds_client.diagnostic_session_control(SESSION_TYPE.EXTENDED_DIAGNOSTIC)

selfdrive/debug/debug_fw_fingerprinting_offline.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 import argparse
 
-from opendbc.car import uds
 from openpilot.tools.lib.live_logreader import live_logreader
 from openpilot.tools.lib.logreader import LogReader, ReadMode
+from panda.python import uds
 
 
 def main(route: str | None, addrs: list[int]):

selfdrive/debug/hyundai_enable_radar_points.py

@@ -16,10 +16,8 @@ import argparse
 from typing import NamedTuple
 from subprocess import check_output, CalledProcessError
 
-from opendbc.car.carlog import carlog
-from opendbc.car.uds import UdsClient, SESSION_TYPE, DATA_IDENTIFIER_TYPE
-from opendbc.safety import Safety
 from panda.python import Panda
+from panda.python.uds import UdsClient, SESSION_TYPE, DATA_IDENTIFIER_TYPE
 
 class ConfigValues(NamedTuple):
   default_config: bytes
@@ -80,9 +78,6 @@ if __name__ == "__main__":
   parser.add_argument('--bus', type=int, default=0, help='can bus to use (default: 0)')
   args = parser.parse_args()
 
-  if args.debug:
-    carlog.setLevel('DEBUG')
-
   try:
     check_output(["pidof", "pandad"])
     print("pandad is running, please kill openpilot before running this script! (aborted)")
@@ -97,8 +92,8 @@ if __name__ == "__main__":
     sys.exit(0)
 
   panda = Panda()
-  panda.set_safety_mode(Safety.SAFETY_ELM327)
-  uds_client = UdsClient(panda, 0x7D0, bus=args.bus)
+  panda.set_safety_mode(Panda.SAFETY_ELM327)
+  uds_client = UdsClient(panda, 0x7D0, bus=args.bus, debug=args.debug)
 
   print("\n[START DIAGNOSTIC SESSION]")
   session_type : SESSION_TYPE = 0x07 # type: ignore

selfdrive/debug/internal/qlog_size.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 import matplotlib.pyplot as plt
 
 from cereal.services import SERVICE_LIST
-from openpilot.common.file_helpers import LOG_COMPRESSION_LEVEL
+from openpilot.system.loggerd.uploader import LOG_COMPRESSION_LEVEL
 from openpilot.tools.lib.logreader import LogReader
 from tqdm import tqdm
 

selfdrive/debug/live_cpu_and_temp.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 import argparse
-import numpy as np
 import capnp
 from collections import defaultdict
 
 from cereal.messaging import SubMaster
+from openpilot.common.numpy_fast import mean
 
 def cputime_total(ct):
   return ct.user + ct.nice + ct.system + ct.idle + ct.iowait + ct.irq + ct.softirq
@@ -49,7 +49,7 @@ if __name__ == "__main__":
 
     if sm.updated['deviceState']:
       t = sm['deviceState']
-      last_temp = np.mean(t.cpuTempC)
+      last_temp = mean(t.cpuTempC)
       last_mem = t.memoryUsagePercent
 
     if sm.updated['procLog']:
@@ -72,7 +72,7 @@ if __name__ == "__main__":
       total_times = total_times_new[:]
       busy_times = busy_times_new[:]
 
-      print(f"CPU {100.0 * np.mean(cores):.2f}% - RAM: {last_mem:.2f}% - Temp {last_temp:.2f}C")
+      print(f"CPU {100.0 * mean(cores):.2f}% - RAM: {last_mem:.2f}% - Temp {last_temp:.2f}C")
 
       if args.cpu and prev_proclog is not None and prev_proclog_t is not None:
         procs: dict[str, float] = defaultdict(float)

selfdrive/debug/read_dtc_status.py

@@ -2,10 +2,9 @@
 import sys
 import argparse
 from subprocess import check_output, CalledProcessError
-from opendbc.car.carlog import carlog
-from opendbc.car.uds import UdsClient, SESSION_TYPE, DTC_REPORT_TYPE, DTC_STATUS_MASK_TYPE, get_dtc_num_as_str, get_dtc_status_names
-from opendbc.safety import Safety
 from panda import Panda
+from panda.python.uds import UdsClient, SESSION_TYPE, DTC_REPORT_TYPE, DTC_STATUS_MASK_TYPE
+from panda.python.uds import get_dtc_num_as_str, get_dtc_status_names
 
 parser = argparse.ArgumentParser(description="read DTC status")
 parser.add_argument("addr", type=lambda x: int(x,0))
@@ -13,9 +12,6 @@ parser.add_argument("--bus", type=int, default=0)
 parser.add_argument('--debug', action='store_true')
 args = parser.parse_args()
 
-if args.debug:
-  carlog.setLevel('DEBUG')
-
 try:
   check_output(["pidof", "pandad"])
   print("pandad is running, please kill openpilot before running this script! (aborted)")
@@ -25,8 +21,8 @@ except CalledProcessError as e:
     raise e
 
 panda = Panda()
-panda.set_safety_mode(Safety.SAFETY_ELM327)
-uds_client = UdsClient(panda, args.addr, bus=args.bus)
+panda.set_safety_mode(Panda.SAFETY_ELM327)
+uds_client = UdsClient(panda, args.addr, bus=args.bus, debug=args.debug)
 print("extended diagnostic session ...")
 uds_client.diagnostic_session_control(SESSION_TYPE.EXTENDED_DIAGNOSTIC)
 print("read diagnostic codes ...")

selfdrive/debug/touch_replay.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-
-import numpy as np
-import matplotlib.pyplot as plt
-
-from openpilot.tools.lib.logreader import LogReader
-
-if __name__ == '__main__':
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--width', default=2160, type=int)
-  parser.add_argument('--height', default=1080, type=int)
-  parser.add_argument('--route', default='rlog', type=str)
-  args = parser.parse_args()
-
-  w = args.width
-  h = args.height
-  route = args.route
-
-  fingers = [[-1, -1]] * 5
-  touch_points = []
-  current_slot = 0
-
-  lr = list(LogReader(route))
-  for msg in lr:
-    if msg.which() == 'touch':
-      for event in msg.touch:
-        if event.type == 3 and event.code == 47:
-          current_slot = event.value
-        elif event.type == 3 and event.code == 57 and event.value == -1:
-          fingers[current_slot] = [-1, -1]
-        elif event.type == 3 and event.code == 53:
-          fingers[current_slot][1] = event.value
-          if fingers[current_slot][0] != -1:
-            touch_points.append(fingers[current_slot].copy())
-        elif event.type == 3 and event.code == 54:
-          fingers[current_slot][0] = w - event.value
-          if fingers[current_slot][1] != -1:
-            touch_points.append(fingers[current_slot].copy())
-
-  if not touch_points:
-    print(f'No touch events found for {route}')
-    quit()
-
-  unique_points, counts = np.unique(touch_points, axis=0, return_counts=True)
-
-  plt.figure(figsize=(10, 3))
-  plt.scatter(unique_points[:, 0], unique_points[:, 1], c=counts, s=counts * 20, edgecolors='red')
-  plt.colorbar()
-  plt.title(f'Touches for {route}')
-  plt.xlim(0, w)
-  plt.ylim(0, h)
-  plt.grid(True)
-  plt.show()

selfdrive/debug/vw_mqb_config.py

@@ -3,11 +3,9 @@
 import argparse
 import struct
 from enum import IntEnum
-from opendbc.car.carlog import carlog
-from opendbc.car.uds import UdsClient, MessageTimeoutError, NegativeResponseError, SESSION_TYPE,\
-  DATA_IDENTIFIER_TYPE, ACCESS_TYPE
-from opendbc.safety import Safety
 from panda import Panda
+from panda.python.uds import UdsClient, MessageTimeoutError, NegativeResponseError, SESSION_TYPE,\
+  DATA_IDENTIFIER_TYPE, ACCESS_TYPE
 from datetime import date
 
 # TODO: extend UDS library to allow custom/vendor-defined data identifiers without ignoring type checks
@@ -35,13 +33,10 @@ if __name__ == "__main__":
   parser.add_argument("action", choices={"show", "enable", "disable"}, help="show or modify current EPS HCA config")
   args = parser.parse_args()
 
-  if args.debug:
-    carlog.setLevel('DEBUG')
-
   panda = Panda()
-  panda.set_safety_mode(Safety.SAFETY_ELM327)
+  panda.set_safety_mode(Panda.SAFETY_ELM327)
   bus = 1 if panda.has_obd() else 0
-  uds_client = UdsClient(panda, MQB_EPS_CAN_ADDR, MQB_EPS_CAN_ADDR + RX_OFFSET, bus, timeout=0.2)
+  uds_client = UdsClient(panda, MQB_EPS_CAN_ADDR, MQB_EPS_CAN_ADDR + RX_OFFSET, bus, timeout=0.2, debug=args.debug)
 
   try:
     uds_client.diagnostic_session_control(SESSION_TYPE.EXTENDED_DIAGNOSTIC)

selfdrive/locationd/calibrationd.py

@@ -6,6 +6,7 @@ While the roll calibration is a real value that can be estimated, here we assume
 and the image input into the neural network is not corrected for roll.
 '''
 
+import gc
 import os
 import capnp
 import numpy as np
@@ -15,7 +16,7 @@ from cereal import log
 import cereal.messaging as messaging
 from openpilot.common.conversions import Conversions as CV
 from openpilot.common.params import Params
-from openpilot.common.realtime import config_realtime_process
+from openpilot.common.realtime import set_realtime_priority
 from openpilot.common.transformations.orientation import rot_from_euler, euler_from_rot
 from openpilot.common.swaglog import cloudlog
 
@@ -255,7 +256,8 @@ class Calibrator:
 
 
 def main() -> NoReturn:
-  config_realtime_process([0, 1, 2, 3], 5)
+  gc.disable()
+  set_realtime_priority(1)
 
   pm = messaging.PubMaster(['liveCalibration'])
   sm = messaging.SubMaster(['cameraOdometry', 'carState', 'carParams'], poll='cameraOdometry')

selfdrive/locationd/locationd.py

@@ -12,7 +12,6 @@ from cereal.services import SERVICE_LIST
 from openpilot.common.transformations.orientation import rot_from_euler
 from openpilot.common.realtime import config_realtime_process
 from openpilot.common.params import Params
-from openpilot.common.swaglog import cloudlog
 from openpilot.selfdrive.locationd.helpers import rotate_std
 from openpilot.selfdrive.locationd.models.pose_kf import PoseKalman, States
 from openpilot.selfdrive.locationd.models.constants import ObservationKind, GENERATED_DIR
@@ -25,16 +24,14 @@ MIN_STD_SANITY_CHECK = 1e-5  # m or rad
 MAX_FILTER_REWIND_TIME = 0.8  # s
 MAX_SENSOR_TIME_DIFF = 0.1  # s
 YAWRATE_CROSS_ERR_CHECK_FACTOR = 30
-INPUT_INVALID_LIMIT = 2.0 # 1 (camodo) / 9 (sensor) bad input[s] ignored
-INPUT_INVALID_RECOVERY = 10.0 # ~10 secs to resume after exceeding allowed bad inputs by one
+INPUT_INVALID_THRESHOLD = 0.5  # 0 bad inputs ignored
+TIMING_INVALID_THRESHOLD = 2.5  # 2 bad timings ignored
+INPUT_INVALID_DECAY = 0.9993  # ~10 secs to resume after exceeding allowed bad inputs by one (at 100hz)
+TIMING_INVALID_DECAY = 0.9990  # ~2 secs to resume after exceeding allowed bad timings by one (at 100hz)
 POSENET_STD_INITIAL_VALUE = 10.0
 POSENET_STD_HIST_HALF = 20
 
 
-def calculate_invalid_input_decay(invalid_limit, recovery_time, frequency):
-  return (1 - 1 / (2 * invalid_limit)) ** (1 / (recovery_time * frequency))
-
-
 def init_xyz_measurement(measurement: capnp._DynamicStructBuilder, values: np.ndarray, stds: np.ndarray, valid: bool):
   assert len(values) == len(stds) == 3
   measurement.x, measurement.y, measurement.z = map(float, values)
@@ -79,20 +76,20 @@ class LocationEstimator:
     # sensor time and log time should be close
     sensor_time_invalid = abs(sensor_time - t) > MAX_SENSOR_TIME_DIFF
     if sensor_time_invalid:
-      cloudlog.warning("Sensor reading ignored, sensor timestamp more than 100ms off from log time")
+      print("Sensor reading ignored, sensor timestamp more than 100ms off from log time")
     return not sensor_time_invalid
 
   def _validate_timestamp(self, t: float):
     kf_t = self.kf.t
     invalid = not np.isnan(kf_t) and (kf_t - t) > MAX_FILTER_REWIND_TIME
     if invalid:
-      cloudlog.warning("Observation timestamp is older than the max rewind threshold of the filter")
+      print("Observation timestamp is older than the max rewind threshold of the filter")
     return not invalid
 
   def _finite_check(self, t: float, new_x: np.ndarray, new_P: np.ndarray):
     all_finite = np.isfinite(new_x).all() and np.isfinite(new_P).all()
     if not all_finite:
-      cloudlog.error("Non-finite values detected, kalman reset")
+      print("Non-finite values detected, kalman reset")
       self.reset(t)
 
   def handle_log(self, t: float, which: str, msg: capnp._DynamicStructReader) -> HandleLogResult:
@@ -272,11 +269,11 @@ def main():
 
   filter_initialized = False
   critcal_services = ["accelerometer", "gyroscope", "cameraOdometry"]
+  observation_timing_invalid = defaultdict(int)
   observation_input_invalid = defaultdict(int)
 
-  input_invalid_limit = {s: round(INPUT_INVALID_LIMIT * (SERVICE_LIST[s].frequency / 20.)) for s in critcal_services}
-  input_invalid_threshold = {s: input_invalid_limit[s] - 0.5 for s in critcal_services}
-  input_invalid_decay = {s: calculate_invalid_input_decay(input_invalid_limit[s], INPUT_INVALID_RECOVERY, SERVICE_LIST[s].frequency) for s in critcal_services}
+  input_invalid_decay = {s: INPUT_INVALID_DECAY ** (100. / SERVICE_LIST[s].frequency) for s in critcal_services}
+  timing_invalid_decay = {s: TIMING_INVALID_DECAY ** (100. / SERVICE_LIST[s].frequency) for s in critcal_services}
 
   initial_pose = params.get("LocationFilterInitialState")
   if initial_pose is not None:
@@ -309,19 +306,19 @@ def main():
             continue
 
           if res == HandleLogResult.TIMING_INVALID:
-            cloudlog.warning(f"Observation {which} ignored due to failed timing check")
-            observation_input_invalid[which] += 1
+            observation_timing_invalid[which] += 1
           elif res == HandleLogResult.INPUT_INVALID:
-            cloudlog.warning(f"Observation {which} ignored due to failed sanity check")
             observation_input_invalid[which] += 1
-          elif res == HandleLogResult.SUCCESS:
+          else:
             observation_input_invalid[which] *= input_invalid_decay[which]
+            observation_timing_invalid[which] *= timing_invalid_decay[which]
     else:
       filter_initialized = sm.all_checks() and sensor_all_checks(acc_msgs, gyro_msgs, sensor_valid, sensor_recv_time, sensor_alive, SIMULATION)
 
     if sm.updated["cameraOdometry"]:
-      critical_service_inputs_valid = all(observation_input_invalid[s] < input_invalid_threshold[s] for s in critcal_services)
-      inputs_valid = sm.all_valid() and critical_service_inputs_valid
+      critical_service_inputs_valid = all(observation_input_invalid[s] < INPUT_INVALID_THRESHOLD for s in critcal_services)
+      critical_service_timing_valid = all(observation_timing_invalid[s] < TIMING_INVALID_THRESHOLD for s in critcal_services)
+      inputs_valid = sm.all_valid() and critical_service_inputs_valid and critical_service_timing_valid
       sensors_valid = sensor_all_checks(acc_msgs, gyro_msgs, sensor_valid, sensor_recv_time, sensor_alive, SIMULATION)
 
       msg = estimator.get_msg(sensors_valid, inputs_valid, filter_initialized)

selfdrive/locationd/paramsd.py

@@ -8,6 +8,7 @@ import cereal.messaging as messaging
 from cereal import car, log
 from openpilot.common.params import Params
 from openpilot.common.realtime import config_realtime_process, DT_MDL
+from openpilot.common.numpy_fast import clip
 from openpilot.selfdrive.locationd.models.car_kf import CarKalman, ObservationKind, States
 from openpilot.selfdrive.locationd.models.constants import GENERATED_DIR
 from openpilot.selfdrive.locationd.helpers import PoseCalibrator, Pose
@@ -65,7 +66,7 @@ class ParamsLearner:
         # This is done to bound the road roll estimate when localizer values are invalid
         roll = 0.0
         roll_std = np.radians(10.0)
-      self.roll = np.clip(roll, self.roll - ROLL_MAX_DELTA, self.roll + ROLL_MAX_DELTA)
+      self.roll = clip(roll, self.roll - ROLL_MAX_DELTA, self.roll + ROLL_MAX_DELTA)
 
       yaw_rate_valid = msg.angularVelocityDevice.valid and self.calibrator.calib_valid
       yaw_rate_valid = yaw_rate_valid and 0 < self.yaw_rate_std < 10  # rad/s
@@ -202,11 +203,11 @@ def main():
         learner = ParamsLearner(CP, CP.steerRatio, 1.0, 0.0)
         x = learner.kf.x
 
-      angle_offset_average = np.clip(math.degrees(x[States.ANGLE_OFFSET].item()),
+      angle_offset_average = clip(math.degrees(x[States.ANGLE_OFFSET].item()),
                                   angle_offset_average - MAX_ANGLE_OFFSET_DELTA, angle_offset_average + MAX_ANGLE_OFFSET_DELTA)
-      angle_offset = np.clip(math.degrees(x[States.ANGLE_OFFSET].item() + x[States.ANGLE_OFFSET_FAST].item()),
+      angle_offset = clip(math.degrees(x[States.ANGLE_OFFSET].item() + x[States.ANGLE_OFFSET_FAST].item()),
                           angle_offset - MAX_ANGLE_OFFSET_DELTA, angle_offset + MAX_ANGLE_OFFSET_DELTA)
-      roll = np.clip(float(x[States.ROAD_ROLL].item()), roll - ROLL_MAX_DELTA, roll + ROLL_MAX_DELTA)
+      roll = clip(float(x[States.ROAD_ROLL].item()), roll - ROLL_MAX_DELTA, roll + ROLL_MAX_DELTA)
       roll_std = float(P[States.ROAD_ROLL].item())
       if learner.active and learner.speed > LOW_ACTIVE_SPEED:
         # Account for the opposite signs of the yaw rates
@@ -225,9 +226,9 @@ def main():
       liveParameters.sensorValid = sensors_valid
       liveParameters.steerRatio = float(x[States.STEER_RATIO].item())
       liveParameters.stiffnessFactor = float(x[States.STIFFNESS].item())
-      liveParameters.roll = float(roll)
-      liveParameters.angleOffsetAverageDeg = float(angle_offset_average)
-      liveParameters.angleOffsetDeg = float(angle_offset)
+      liveParameters.roll = roll
+      liveParameters.angleOffsetAverageDeg = angle_offset_average
+      liveParameters.angleOffsetDeg = angle_offset
       liveParameters.valid = all((
         avg_offset_valid,
         total_offset_valid,

selfdrive/locationd/test/test_locationd.py

@@ -0,0 +1,56 @@
+import capnp
+
+import cereal.messaging as messaging
+from openpilot.common.params import Params
+
+from openpilot.system.manager.process_config import managed_processes
+
+
+class TestLocationdProc:
+  LLD_MSGS = ['gpsLocationExternal', 'cameraOdometry', 'carState', 'liveCalibration',
+              'accelerometer', 'gyroscope', 'magnetometer']
+
+  def setup_method(self):
+    self.pm = messaging.PubMaster(self.LLD_MSGS)
+
+    self.params = Params()
+    self.params.put_bool("UbloxAvailable", True)
+    managed_processes['locationd'].prepare()
+    managed_processes['locationd'].start()
+
+  def teardown_method(self):
+    managed_processes['locationd'].stop()
+
+  def get_msg(self, name, t):
+    try:
+      msg = messaging.new_message(name)
+    except capnp.lib.capnp.KjException:
+      msg = messaging.new_message(name, 0)
+
+    if name == "gpsLocationExternal":
+      msg.gpsLocationExternal.flags = 1
+      msg.gpsLocationExternal.hasFix = True
+      msg.gpsLocationExternal.verticalAccuracy = 1.0
+      msg.gpsLocationExternal.speedAccuracy = 1.0
+      msg.gpsLocationExternal.bearingAccuracyDeg = 1.0
+      msg.gpsLocationExternal.vNED = [0.0, 0.0, 0.0]
+      msg.gpsLocationExternal.latitude = float(self.lat)
+      msg.gpsLocationExternal.longitude = float(self.lon)
+      msg.gpsLocationExternal.unixTimestampMillis = t * 1e6
+      msg.gpsLocationExternal.altitude = float(self.alt)
+    #if name == "gnssMeasurements":
+    #  msg.gnssMeasurements.measTime = t
+    #  msg.gnssMeasurements.positionECEF.value = [self.x , self.y, self.z]
+    #  msg.gnssMeasurements.positionECEF.std = [0,0,0]
+    #  msg.gnssMeasurements.positionECEF.valid = True
+    #  msg.gnssMeasurements.velocityECEF.value = []
+    #  msg.gnssMeasurements.velocityECEF.std = [0,0,0]
+    #  msg.gnssMeasurements.velocityECEF.valid = True
+    elif name == 'cameraOdometry':
+      msg.cameraOdometry.rot = [0.0, 0.0, 0.0]
+      msg.cameraOdometry.rotStd = [0.0, 0.0, 0.0]
+      msg.cameraOdometry.trans = [0.0, 0.0, 0.0]
+      msg.cameraOdometry.transStd = [0.0, 0.0, 0.0]
+    msg.logMonoTime = t
+    msg.valid = True
+    return msg

selfdrive/locationd/test/test_locationd_scenarios.py

@@ -23,10 +23,8 @@ class Scenario(Enum):
   BASE = 'base'
   GYRO_OFF = 'gyro_off'
   GYRO_SPIKE_MIDWAY = 'gyro_spike_midway'
-  GYRO_CONSISTENT_SPIKES = 'gyro_consistent_spikes'
   ACCEL_OFF = 'accel_off'
   ACCEL_SPIKE_MIDWAY = 'accel_spike_midway'
-  ACCEL_CONSISTENT_SPIKES = 'accel_consistent_spikes'
   SENSOR_TIMING_SPIKE_MIDWAY = 'timing_spikes'
   SENSOR_TIMING_CONSISTENT_SPIKES = 'timing_consistent_spikes'
 
@@ -65,20 +63,18 @@ def run_scenarios(scenario, logs):
   elif scenario == Scenario.GYRO_OFF:
     logs = sorted([x for x in logs if x.which() != 'gyroscope'], key=lambda x: x.logMonoTime)
 
-  elif scenario == Scenario.GYRO_SPIKE_MIDWAY or scenario == Scenario.GYRO_CONSISTENT_SPIKES:
+  elif scenario == Scenario.GYRO_SPIKE_MIDWAY:
     def gyro_spike(msg):
       msg.gyroscope.gyroUncalibrated.v[0] += 3.0
-    count = 1 if scenario == Scenario.GYRO_SPIKE_MIDWAY else CONSISTENT_SPIKES_COUNT
-    logs = modify_logs_midway(logs, 'gyroscope', count, gyro_spike)
+    logs = modify_logs_midway(logs, 'gyroscope', 1, gyro_spike)
 
   elif scenario == Scenario.ACCEL_OFF:
     logs = sorted([x for x in logs if x.which() != 'accelerometer'], key=lambda x: x.logMonoTime)
 
-  elif scenario == Scenario.ACCEL_SPIKE_MIDWAY or scenario == Scenario.ACCEL_CONSISTENT_SPIKES:
+  elif scenario == Scenario.ACCEL_SPIKE_MIDWAY:
     def acc_spike(msg):
-      msg.accelerometer.acceleration.v[0] += 100.0
-    count = 1 if scenario == Scenario.ACCEL_SPIKE_MIDWAY else CONSISTENT_SPIKES_COUNT
-    logs = modify_logs_midway(logs, 'accelerometer', count, acc_spike)
+      msg.accelerometer.acceleration.v[0] += 10.0
+    logs = modify_logs_midway(logs, 'accelerometer', 1, acc_spike)
 
   elif scenario == Scenario.SENSOR_TIMING_SPIKE_MIDWAY or scenario == Scenario.SENSOR_TIMING_CONSISTENT_SPIKES:
     def timing_spike(msg):
@@ -125,7 +121,7 @@ class TestLocationdScenarios:
     assert np.allclose(replayed_data['roll'], 0.0)
     assert np.all(replayed_data['sensors_flag'] == 0.0)
 
-  def test_gyro_spike(self):
+  def test_gyro_spikes(self):
     """
     Test: a gyroscope spike in the middle of the segment
     Expected Result:
@@ -136,17 +132,8 @@ class TestLocationdScenarios:
     orig_data, replayed_data = run_scenarios(Scenario.GYRO_SPIKE_MIDWAY, self.logs)
     assert np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.35))
     assert np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.55))
-    assert np.all(replayed_data['inputs_flag'] == orig_data['inputs_flag'])
-    assert np.all(replayed_data['sensors_flag'] == orig_data['sensors_flag'])
-
-  def test_consistent_gyro_spikes(self):
-    """
-    Test: consistent timing spikes for N gyroscope messages in the middle of the segment
-    Expected Result: inputsOK becomes False after N of bad measurements
-    """
-    orig_data, replayed_data = run_scenarios(Scenario.GYRO_CONSISTENT_SPIKES, self.logs)
-    assert np.diff(replayed_data['inputs_flag'])[501] == -1.0
-    assert np.diff(replayed_data['inputs_flag'])[708] == 1.0
+    assert np.diff(replayed_data['inputs_flag'])[499] == -1.0
+    assert np.diff(replayed_data['inputs_flag'])[704] == 1.0
 
   def test_accel_off(self):
     """
@@ -161,7 +148,7 @@ class TestLocationdScenarios:
     assert np.allclose(replayed_data['roll'], 0.0)
     assert np.all(replayed_data['sensors_flag'] == 0.0)
 
-  def test_accel_spike(self):
+  def test_accel_spikes(self):
     """
     ToDo:
     Test: an accelerometer spike in the middle of the segment
@@ -186,5 +173,5 @@ class TestLocationdScenarios:
     Expected Result: inputsOK becomes False after N of bad measurements
     """
     orig_data, replayed_data = run_scenarios(Scenario.SENSOR_TIMING_CONSISTENT_SPIKES, self.logs)
-    assert np.diff(replayed_data['inputs_flag'])[501] == -1.0
-    assert np.diff(replayed_data['inputs_flag'])[707] == 1.0
+    assert np.diff(replayed_data['inputs_flag'])[500] == -1.0
+    assert np.diff(replayed_data['inputs_flag'])[787] == 1.0

selfdrive/locationd/torqued.py

@@ -71,7 +71,7 @@ class TorqueEstimator(ParameterEstimator):
     self.offline_friction = 0.0
     self.offline_latAccelFactor = 0.0
     self.resets = 0.0
-    self.use_params = CP.brand in ALLOWED_CARS and CP.lateralTuning.which() == 'torque'
+    self.use_params = CP.carName in ALLOWED_CARS and CP.lateralTuning.which() == 'torque'
 
     if CP.lateralTuning.which() == 'torque':
       self.offline_friction = CP.lateralTuning.torque.friction

selfdrive/modeld/SConscript

@@ -13,6 +13,20 @@ common_src = [
   "transforms/transform.cc",
 ]
 
+thneed_src_common = [
+  "thneed/thneed_common.cc",
+  "thneed/serialize.cc",
+]
+
+thneed_src_qcom = thneed_src_common + ["thneed/thneed_qcom2.cc"]
+thneed_src_pc = thneed_src_common + ["thneed/thneed_pc.cc"]
+thneed_src = thneed_src_qcom if arch == "larch64" else thneed_src_pc
+
+# SNPE except on Mac and ARM Linux
+snpe_lib = []
+if arch != "Darwin" and arch != "aarch64":
+  common_src += ['runners/snpemodel.cc']
+  snpe_lib += ['SNPE']
 
 # OpenCL is a framework on Mac
 if arch == "Darwin":
@@ -26,25 +40,39 @@ for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transfor
     xenv['CXXFLAGS'].append(f'-D{pathdef}_PATH=\\"{File(fn).abspath}\\"')
 
 # Compile cython
+snpe_rpath_qcom = "/data/pythonpath/third_party/snpe/larch64"
+snpe_rpath_pc = f"{Dir('#').abspath}/third_party/snpe/x86_64-linux-clang"
+snpe_rpath = lenvCython['RPATH'] + [snpe_rpath_qcom if arch == "larch64" else snpe_rpath_pc]
+
 cython_libs = envCython["LIBS"] + libs
+snpemodel_lib = lenv.Library('snpemodel', ['runners/snpemodel.cc'])
 commonmodel_lib = lenv.Library('commonmodel', common_src)
+
+lenvCython.Program('runners/runmodel_pyx.so', 'runners/runmodel_pyx.pyx', LIBS=cython_libs, FRAMEWORKS=frameworks)
+lenvCython.Program('runners/snpemodel_pyx.so', 'runners/snpemodel_pyx.pyx', LIBS=[snpemodel_lib, snpe_lib, *cython_libs], FRAMEWORKS=frameworks, RPATH=snpe_rpath)
 lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks)
-tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x]
+
+tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath)]
 
 # Get model metadata
 fn = File("models/supercombo").abspath
 cmd = f'python3 {Dir("#selfdrive/modeld").abspath}/get_model_metadata.py {fn}.onnx'
 lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
 
-# Compile tinygrad model
-pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"'
-if arch == 'larch64':
-  device_string = 'QCOM=1'
-else:
-  device_string = 'CLANG=1 IMAGE=0'
+# Build thneed model
+if arch == "larch64" or GetOption('pc_thneed'):
+  tinygrad_opts = []
+  if not GetOption('pc_thneed'):
+    # use FLOAT16 on device for speed + don't cache the CL kernels for space
+    tinygrad_opts += ["FLOAT16=1", "PYOPENCL_NO_CACHE=1"]
+  cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn}.onnx {fn}.thneed"
 
-for model_name in ['supercombo', 'dmonitoring_model']:
-  fn = File(f"models/{model_name}").abspath
-  cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl'
-  lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
+  lenv.Command(fn + ".thneed", [fn + ".onnx"] + tinygrad_files, cmd)
 
+  fn_dm = File("models/dmonitoring_model").abspath
+  cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn_dm}.onnx {fn_dm}.thneed"
+  lenv.Command(fn_dm + ".thneed", [fn_dm + ".onnx"] + tinygrad_files, cmd)
+
+  thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'OpenCL', 'dl'])
+  thneedmodel_lib = env.Library('thneedmodel', ['runners/thneedmodel.cc'])
+  lenvCython.Program('runners/thneedmodel_pyx.so', 'runners/thneedmodel_pyx.pyx', LIBS=envCython["LIBS"]+[thneedmodel_lib, thneed_lib, gpucommon, common, 'dl', 'OpenCL'])

selfdrive/modeld/constants.py

@@ -16,6 +16,7 @@ class ModelConstants:
   MODEL_FREQ = 20
   FEATURE_LEN = 512
   FULL_HISTORY_BUFFER_LEN = 99
+  HISTORY_BUFFER_LEN = 24
   DESIRE_LEN = 8
   TRAFFIC_CONVENTION_LEN = 2
   LAT_PLANNER_STATE_LEN = 4
@@ -31,6 +32,7 @@ class ModelConstants:
   DISENGAGE_WIDTH = 5
   POSE_WIDTH = 6
   WIDE_FROM_DEVICE_WIDTH = 3
+  SIM_POSE_WIDTH = 6
   LEAD_WIDTH = 4
   LANE_LINES_WIDTH = 2
   ROAD_EDGES_WIDTH = 2

selfdrive/modeld/dmonitoringmodeld

@@ -1,4 +1,10 @@
 #!/usr/bin/env bash
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR/../../"
+
+if [ -f "$DIR/libthneed.so" ]; then
+  export LD_PRELOAD="$DIR/libthneed.so"
+fi
+
 exec "$DIR/dmonitoringmodeld.py" "$@"

selfdrive/modeld/dmonitoringmodeld.py

@@ -1,16 +1,8 @@
 #!/usr/bin/env python3
 import os
-from openpilot.system.hardware import TICI
-if TICI:
-  from tinygrad.tensor import Tensor
-  from tinygrad.dtype import dtypes
-  from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address
-  os.environ['QCOM'] = '1'
-else:
-  from openpilot.selfdrive.modeld.runners.ort_helpers import make_onnx_cpu_runner
+import gc
 import math
 import time
-import pickle
 import ctypes
 import numpy as np
 from pathlib import Path
@@ -20,23 +12,22 @@ from cereal import messaging
 from cereal.messaging import PubMaster, SubMaster
 from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.common.swaglog import cloudlog
-from openpilot.common.realtime import config_realtime_process
-from openpilot.common.transformations.model import dmonitoringmodel_intrinsics, DM_INPUT_SIZE
-from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye
-from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame
+from openpilot.common.realtime import set_realtime_priority
+from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
+from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext
 from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid
-from openpilot.system import sentry
 
-MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE
 CALIB_LEN = 3
+MODEL_WIDTH = 1440
+MODEL_HEIGHT = 960
 FEATURE_LEN = 512
 OUTPUT_SIZE = 84 + FEATURE_LEN
 
 PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld"
 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
-MODEL_PATH = Path(__file__).parent / 'models/dmonitoring_model.onnx'
-MODEL_PKL_PATH = Path(__file__).parent / 'models/dmonitoring_model_tinygrad.pkl'
-
+MODEL_PATHS = {
+  ModelRunner.THNEED: Path(__file__).parent / 'models/dmonitoring_model.thneed',
+  ModelRunner.ONNX: Path(__file__).parent / 'models/dmonitoring_model.onnx'}
 
 class DriverStateResult(ctypes.Structure):
   _fields_ = [
@@ -56,7 +47,6 @@ class DriverStateResult(ctypes.Structure):
     ("ready_prob", ctypes.c_float*4),
     ("not_ready_prob", ctypes.c_float*2)]
 
-
 class DMonitoringModelResult(ctypes.Structure):
   _fields_ = [
     ("driver_state_lhd", DriverStateResult),
@@ -65,46 +55,36 @@ class DMonitoringModelResult(ctypes.Structure):
     ("wheel_on_right_prob", ctypes.c_float),
     ("features", ctypes.c_float*FEATURE_LEN)]
 
-
 class ModelState:
   inputs: dict[str, np.ndarray]
   output: np.ndarray
+  model: ModelRunner
 
   def __init__(self, cl_ctx):
     assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float)
+    self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32)
+    self.inputs = {
+      'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8),
+      'calib': np.zeros(CALIB_LEN, dtype=np.float32)}
 
-    self.frame = MonitoringModelFrame(cl_ctx)
-    self.numpy_inputs = {
-      'calib': np.zeros((1, CALIB_LEN), dtype=np.float32),
-    }
+    self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, cl_ctx)
+    self.model.addInput("input_img", None)
+    self.model.addInput("calib", self.inputs['calib'])
 
-    if TICI:
-      self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()}
-      with open(MODEL_PKL_PATH, "rb") as f:
-        self.model_run = pickle.load(f)
-    else:
-      self.onnx_cpu_runner = make_onnx_cpu_runner(MODEL_PATH)
+  def run(self, buf:VisionBuf, calib:np.ndarray) -> tuple[np.ndarray, float]:
+    self.inputs['calib'][:] = calib
 
-  def run(self, buf: VisionBuf, calib: np.ndarray, transform: np.ndarray) -> tuple[np.ndarray, float]:
-    self.numpy_inputs['calib'][0,:] = calib
+    v_offset = buf.height - MODEL_HEIGHT
+    h_offset = (buf.width - MODEL_WIDTH) // 2
+    buf_data = buf.data.reshape(-1, buf.stride)
+    input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH)
+    input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH]
 
+    self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32))
     t1 = time.perf_counter()
-
-    input_img_cl = self.frame.prepare(buf, transform.flatten())
-    if TICI:
-      # The imgs tensors are backed by opencl memory, only need init once
-      if 'input_img' not in self.tensor_inputs:
-        self.tensor_inputs['input_img'] = qcom_tensor_from_opencl_address(input_img_cl.mem_address, (1, MODEL_WIDTH*MODEL_HEIGHT), dtype=dtypes.uint8)
-    else:
-      self.numpy_inputs['input_img'] = self.frame.buffer_from_cl(input_img_cl).reshape((1, MODEL_WIDTH*MODEL_HEIGHT))
-
-    if TICI:
-      output = self.model_run(**self.tensor_inputs).numpy().flatten()
-    else:
-      output = self.onnx_cpu_runner.run(None, self.numpy_inputs)[0].flatten()
-
+    self.model.execute()
     t2 = time.perf_counter()
-    return output, t2 - t1
+    return self.output, t2 - t1
 
 
 def fill_driver_state(msg, ds_result: DriverStateResult):
@@ -122,7 +102,6 @@ def fill_driver_state(msg, ds_result: DriverStateResult):
   msg.readyProb = [float(sigmoid(x)) for x in ds_result.ready_prob]
   msg.notReadyProb = [float(sigmoid(x)) for x in ds_result.not_ready_prob]
 
-
 def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: int, execution_time: float, gpu_execution_time: float):
   model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(DMonitoringModelResult)).contents
   msg = messaging.new_message('driverStateV2', valid=True)
@@ -139,11 +118,9 @@ def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts:
 
 
 def main():
+  gc.disable()
   setproctitle(PROCESS_NAME)
-  config_realtime_process([0, 1, 2, 3], 5)
-
-  sentry.set_tag("daemon", PROCESS_NAME)
-  cloudlog.bind(daemon=PROCESS_NAME)
+  set_realtime_priority(1)
 
   cl_context = CLContext()
   model = ModelState(cl_context)
@@ -160,33 +137,22 @@ def main():
   pm = PubMaster(["driverStateV2"])
 
   calib = np.zeros(CALIB_LEN, dtype=np.float32)
-  model_transform = None
 
   while True:
     buf = vipc_client.recv()
     if buf is None:
       continue
 
-    if model_transform is None:
-      cam = _os_fisheye if buf.width == _os_fisheye.width else _ar_ox_fisheye
-      model_transform = np.linalg.inv(np.dot(dmonitoringmodel_intrinsics, np.linalg.inv(cam.intrinsics))).astype(np.float32)
-
     sm.update(0)
     if sm.updated["liveCalibration"]:
       calib[:] = np.array(sm["liveCalibration"].rpyCalib)
 
     t1 = time.perf_counter()
-    model_output, gpu_execution_time = model.run(buf, calib, model_transform)
+    model_output, gpu_execution_time = model.run(buf, calib)
     t2 = time.perf_counter()
 
     pm.send("driverStateV2", get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time))
 
 
 if __name__ == "__main__":
-  try:
-    main()
-  except KeyboardInterrupt:
-    cloudlog.warning(f"child {PROCESS_NAME} got SIGINT")
-  except Exception:
-    sentry.capture_exception()
-    raise
+  main()

selfdrive/modeld/fill_model_msg.py

@@ -8,7 +8,6 @@ SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 
 ConfidenceClass = log.ModelDataV2.ConfidenceClass
 
-
 class PublishState:
   def __init__(self):
     self.disengage_buffer = np.zeros(ModelConstants.CONFIDENCE_BUFFER_LEN*ModelConstants.DISENGAGE_WIDTH, dtype=np.float32)
@@ -56,10 +55,9 @@ def fill_lane_line_meta(builder, lane_lines, lane_line_probs):
   builder.rightProb = lane_line_probs[2]
 
 def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._DynamicStructBuilder,
-                   net_output_data: dict[str, np.ndarray], v_ego: float, delay: float,
-                   publish_state: PublishState, vipc_frame_id: int, vipc_frame_id_extra: int,
-                   frame_id: int, frame_drop: float, timestamp_eof: int, model_execution_time: float,
-                   valid: bool) -> None:
+                   net_output_data: dict[str, np.ndarray], publish_state: PublishState,
+                   vipc_frame_id: int, vipc_frame_id_extra: int, frame_id: int, frame_drop: float,
+                   timestamp_eof: int, model_execution_time: float, valid: bool) -> None:
   frame_age = frame_id - vipc_frame_id if frame_id > vipc_frame_id else 0
   frame_drop_perc = frame_drop * 100
   extended_msg.valid = valid
@@ -71,7 +69,9 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   driving_model_data.frameIdExtra = vipc_frame_id_extra
   driving_model_data.frameDropPerc = frame_drop_perc
   driving_model_data.modelExecutionTime = model_execution_time
-  driving_model_data.action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
+
+  action = driving_model_data.action
+  action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
 
   modelV2 = extended_msg.modelV2
   modelV2.frameId = vipc_frame_id
@@ -82,24 +82,31 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   modelV2.modelExecutionTime = model_execution_time
 
   # plan
-  fill_xyzt(modelV2.position, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.POSITION].T, *net_output_data['plan_stds'][0,:,Plan.POSITION].T)
-  fill_xyzt(modelV2.velocity, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.VELOCITY].T)
-  fill_xyzt(modelV2.acceleration, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ACCELERATION].T)
-  fill_xyzt(modelV2.orientation, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.T_FROM_CURRENT_EULER].T)
-  fill_xyzt(modelV2.orientationRate, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ORIENTATION_RATE].T)
+  position = modelV2.position
+  fill_xyzt(position, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.POSITION].T, *net_output_data['plan_stds'][0,:,Plan.POSITION].T)
+  velocity = modelV2.velocity
+  fill_xyzt(velocity, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.VELOCITY].T)
+  acceleration = modelV2.acceleration
+  fill_xyzt(acceleration, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ACCELERATION].T)
+  orientation = modelV2.orientation
+  fill_xyzt(orientation, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.T_FROM_CURRENT_EULER].T)
+  orientation_rate = modelV2.orientationRate
+  fill_xyzt(orientation_rate, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ORIENTATION_RATE].T)
 
   # temporal pose
   temporal_pose = modelV2.temporalPose
-  temporal_pose.trans = net_output_data['sim_pose'][0,:ModelConstants.POSE_WIDTH//2].tolist()
-  temporal_pose.transStd = net_output_data['sim_pose_stds'][0,:ModelConstants.POSE_WIDTH//2].tolist()
-  temporal_pose.rot = net_output_data['sim_pose'][0,ModelConstants.POSE_WIDTH//2:].tolist()
-  temporal_pose.rotStd = net_output_data['sim_pose_stds'][0,ModelConstants.POSE_WIDTH//2:].tolist()
+  temporal_pose.trans = net_output_data['plan'][0,0,Plan.VELOCITY].tolist()
+  temporal_pose.transStd = net_output_data['plan_stds'][0,0,Plan.VELOCITY].tolist()
+  temporal_pose.rot = net_output_data['plan'][0,0,Plan.ORIENTATION_RATE].tolist()
+  temporal_pose.rotStd = net_output_data['plan_stds'][0,0,Plan.ORIENTATION_RATE].tolist()
 
   # poly path
-  fill_xyz_poly(driving_model_data.path, ModelConstants.POLY_PATH_DEGREE, *net_output_data['plan'][0,:,Plan.POSITION].T)
+  poly_path = driving_model_data.path
+  fill_xyz_poly(poly_path, ModelConstants.POLY_PATH_DEGREE, *net_output_data['plan'][0,:,Plan.POSITION].T)
 
   # lateral planning
-  modelV2.action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
+  action = modelV2.action
+  action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
 
   # times at X_IDXS according to model plan
   PLAN_T_IDXS = [np.nan] * ModelConstants.IDX_N
@@ -128,7 +135,8 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   modelV2.laneLineStds = net_output_data['lane_lines_stds'][0,:,0,0].tolist()
   modelV2.laneLineProbs = net_output_data['lane_lines_prob'][0,1::2].tolist()
 
-  fill_lane_line_meta(driving_model_data.laneLineMeta, modelV2.laneLines, modelV2.laneLineProbs)
+  lane_line_meta = driving_model_data.laneLineMeta
+  fill_lane_line_meta(lane_line_meta, modelV2.laneLines, modelV2.laneLineProbs)
 
   # road edges
   modelV2.init('roadEdges', 2)

selfdrive/modeld/modeld

@@ -1,4 +1,10 @@
 #!/usr/bin/env bash
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR/../../"
+
+if [ -f "$DIR/libthneed.so" ]; then
+  export LD_PRELOAD="$DIR/libthneed.so"
+fi
+
 exec "$DIR/modeld.py" "$@"

selfdrive/modeld/modeld.py

@@ -1,15 +1,5 @@
 #!/usr/bin/env python3
 import os
-from openpilot.system.hardware import TICI
-
-#
-if TICI:
-  from tinygrad.tensor import Tensor
-  from tinygrad.dtype import dtypes
-  from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address
-  os.environ['QCOM'] = '1'
-else:
-  from openpilot.selfdrive.modeld.runners.ort_helpers import make_onnx_cpu_runner
 import time
 import pickle
 import numpy as np
@@ -28,19 +18,22 @@ from openpilot.common.transformations.camera import DEVICE_CAMERAS
 from openpilot.common.transformations.model import get_warp_matrix
 from openpilot.system import sentry
 from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper
+from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
 from openpilot.selfdrive.modeld.parse_model_outputs import Parser
 from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState
 from openpilot.selfdrive.modeld.constants import ModelConstants
-from openpilot.selfdrive.modeld.models.commonmodel_pyx import DrivingModelFrame, CLContext
-
+from openpilot.selfdrive.modeld.models.commonmodel_pyx import ModelFrame, CLContext
 
 PROCESS_NAME = "selfdrive.modeld.modeld"
 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 
-MODEL_PATH = Path(__file__).parent / 'models/supercombo.onnx'
-MODEL_PKL_PATH = Path(__file__).parent / 'models/supercombo_tinygrad.pkl'
+MODEL_PATHS = {
+  ModelRunner.THNEED: Path(__file__).parent / 'models/supercombo.thneed',
+  ModelRunner.ONNX: Path(__file__).parent / 'models/supercombo.onnx'}
+
 METADATA_PATH = Path(__file__).parent / 'models/supercombo_metadata.pkl'
 
+
 class FrameMeta:
   frame_id: int = 0
   timestamp_sof: int = 0
@@ -51,39 +44,43 @@ class FrameMeta:
       self.frame_id, self.timestamp_sof, self.timestamp_eof = vipc.frame_id, vipc.timestamp_sof, vipc.timestamp_eof
 
 class ModelState:
-  frames: dict[str, DrivingModelFrame]
+  frame: ModelFrame
+  wide_frame: ModelFrame
   inputs: dict[str, np.ndarray]
   output: np.ndarray
   prev_desire: np.ndarray  # for tracking the rising edge of the pulse
+  model: ModelRunner
 
   def __init__(self, context: CLContext):
-    self.frames = {'input_imgs': DrivingModelFrame(context), 'big_input_imgs': DrivingModelFrame(context)}
+    self.frame = ModelFrame(context)
+    self.wide_frame = ModelFrame(context)
     self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)
+    self.full_features_20Hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN, ModelConstants.FEATURE_LEN), dtype=np.float32)
+    self.desire_20Hz =  np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN + 1, ModelConstants.DESIRE_LEN), dtype=np.float32)
+    self.prev_desired_curv_20hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN + 1, ModelConstants.PREV_DESIRED_CURV_LEN), dtype=np.float32)
 
     # img buffers are managed in openCL transform code
-    self.numpy_inputs = {
-      'desire': np.zeros((1, (ModelConstants.FULL_HISTORY_BUFFER_LEN+1), ModelConstants.DESIRE_LEN), dtype=np.float32),
-      'traffic_convention': np.zeros((1, ModelConstants.TRAFFIC_CONVENTION_LEN), dtype=np.float32),
-      'lateral_control_params': np.zeros((1, ModelConstants.LATERAL_CONTROL_PARAMS_LEN), dtype=np.float32),
-      'prev_desired_curv': np.zeros((1, (ModelConstants.FULL_HISTORY_BUFFER_LEN+1), ModelConstants.PREV_DESIRED_CURV_LEN), dtype=np.float32),
-      'features_buffer': np.zeros((1, ModelConstants.FULL_HISTORY_BUFFER_LEN,  ModelConstants.FEATURE_LEN), dtype=np.float32),
+    self.inputs = {
+      'desire': np.zeros(ModelConstants.DESIRE_LEN * (ModelConstants.HISTORY_BUFFER_LEN+1), dtype=np.float32),
+      'traffic_convention': np.zeros(ModelConstants.TRAFFIC_CONVENTION_LEN, dtype=np.float32),
+      'lateral_control_params': np.zeros(ModelConstants.LATERAL_CONTROL_PARAMS_LEN, dtype=np.float32),
+      'prev_desired_curv': np.zeros(ModelConstants.PREV_DESIRED_CURV_LEN * (ModelConstants.HISTORY_BUFFER_LEN+1), dtype=np.float32),
+      'features_buffer': np.zeros(ModelConstants.HISTORY_BUFFER_LEN * ModelConstants.FEATURE_LEN, dtype=np.float32),
     }
 
     with open(METADATA_PATH, 'rb') as f:
       model_metadata = pickle.load(f)
-    self.input_shapes =  model_metadata['input_shapes']
 
     self.output_slices = model_metadata['output_slices']
     net_output_size = model_metadata['output_shapes']['outputs'][1]
     self.output = np.zeros(net_output_size, dtype=np.float32)
     self.parser = Parser()
 
-    if TICI:
-      self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()}
-      with open(MODEL_PKL_PATH, "rb") as f:
-        self.model_run = pickle.load(f)
-    else:
-      self.onnx_cpu_runner = make_onnx_cpu_runner(MODEL_PATH)
+    self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, context)
+    self.model.addInput("input_imgs", None)
+    self.model.addInput("big_input_imgs", None)
+    for k,v in self.inputs.items():
+      self.model.addInput(k, v)
 
   def slice_outputs(self, model_outputs: np.ndarray) -> dict[str, np.ndarray]:
     parsed_model_outputs = {k: model_outputs[np.newaxis, v] for k,v in self.output_slices.items()}
@@ -98,40 +95,32 @@ class ModelState:
     new_desire = np.where(inputs['desire'] - self.prev_desire > .99, inputs['desire'], 0)
     self.prev_desire[:] = inputs['desire']
 
-    self.numpy_inputs['desire'][0,:-1] = self.numpy_inputs['desire'][0,1:]
-    self.numpy_inputs['desire'][0,-1] = new_desire
+    self.desire_20Hz[:-1] = self.desire_20Hz[1:]
+    self.desire_20Hz[-1] = new_desire
+    self.inputs['desire'][:] = self.desire_20Hz.reshape((25,4,-1)).max(axis=1).flatten()
 
-    self.numpy_inputs['traffic_convention'][:] = inputs['traffic_convention']
-    self.numpy_inputs['lateral_control_params'][:] = inputs['lateral_control_params']
-    imgs_cl = {'input_imgs': self.frames['input_imgs'].prepare(buf, transform.flatten()),
-               'big_input_imgs': self.frames['big_input_imgs'].prepare(wbuf, transform_wide.flatten())}
+    self.inputs['traffic_convention'][:] = inputs['traffic_convention']
+    self.inputs['lateral_control_params'][:] = inputs['lateral_control_params']
 
-    if TICI:
-      # The imgs tensors are backed by opencl memory, only need init once
-      for key in imgs_cl:
-        if key not in self.tensor_inputs:
-          self.tensor_inputs[key] = qcom_tensor_from_opencl_address(imgs_cl[key].mem_address, self.input_shapes[key], dtype=dtypes.uint8)
-    else:
-      for key in imgs_cl:
-        self.numpy_inputs[key] = self.frames[key].buffer_from_cl(imgs_cl[key]).reshape(self.input_shapes[key]).astype(dtype=np.float32)
+    self.model.setInputBuffer("input_imgs", self.frame.prepare(buf, transform.flatten(), self.model.getCLBuffer("input_imgs")))
+    self.model.setInputBuffer("big_input_imgs", self.wide_frame.prepare(wbuf, transform_wide.flatten(), self.model.getCLBuffer("big_input_imgs")))
 
     if prepare_only:
       return None
 
-    if TICI:
-      self.output = self.model_run(**self.tensor_inputs).numpy().flatten()
-    else:
-      self.output = self.onnx_cpu_runner.run(None, self.numpy_inputs)[0].flatten()
-
+    self.model.execute()
     outputs = self.parser.parse_outputs(self.slice_outputs(self.output))
 
-    self.numpy_inputs['features_buffer'][0,:-1] = self.numpy_inputs['features_buffer'][0,1:]
-    self.numpy_inputs['features_buffer'][0,-1] = outputs['hidden_state'][0, :]
+    self.full_features_20Hz[:-1] = self.full_features_20Hz[1:]
+    self.full_features_20Hz[-1] = outputs['hidden_state'][0, :]
 
+    self.prev_desired_curv_20hz[:-1] = self.prev_desired_curv_20hz[1:]
+    self.prev_desired_curv_20hz[-1] = outputs['desired_curvature'][0, :]
 
-    # TODO model only uses last value now
-    self.numpy_inputs['prev_desired_curv'][0,:-1] = self.numpy_inputs['prev_desired_curv'][0,1:]
-    self.numpy_inputs['prev_desired_curv'][0,-1,:] = outputs['desired_curvature'][0, :]
+    idxs = np.arange(-4,-100,-4)[::-1]
+    self.inputs['features_buffer'][:] = self.full_features_20Hz[idxs].flatten()
+    # TODO model only uses last value now, once that changes we need to input strided action history buffer
+    self.inputs['prev_desired_curv'][-ModelConstants.PREV_DESIRED_CURV_LEN:] = 0. * self.prev_desired_curv_20hz[-4, :]
     return outputs
 
 
@@ -197,7 +186,7 @@ def main(demo=False):
     CP = get_demo_car_params()
   else:
     CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
-  cloudlog.info("modeld got CarParams: %s", CP.brand)
+  cloudlog.info("modeld got CarParams: %s", CP.carName)
 
   # TODO this needs more thought, use .2s extra for now to estimate other delays
   steer_delay = CP.steerActuatorDelay + .2
@@ -285,8 +274,7 @@ def main(demo=False):
       modelv2_send = messaging.new_message('modelV2')
       drivingdata_send = messaging.new_message('drivingModelData')
       posenet_send = messaging.new_message('cameraOdometry')
-      fill_model_msg(drivingdata_send, modelv2_send, model_output, v_ego, steer_delay,
-                     publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id,
+      fill_model_msg(drivingdata_send, modelv2_send, model_output, publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id,
                      frame_drop_ratio, meta_main.timestamp_eof, model_execution_time, live_calib_seen)
 
       desire_state = modelv2_send.modelV2.meta.desireState
@@ -303,6 +291,7 @@ def main(demo=False):
       pm.send('modelV2', modelv2_send)
       pm.send('drivingModelData', drivingdata_send)
       pm.send('cameraOdometry', posenet_send)
+
     last_vipc_frame_id = meta_main.frame_id