Merge branch 'model-switcher-v2' into cp-sp-model

Moved the initialization of 20Hz-specific variables to be conditional based on the `is_20hz` flag. This ensures that unnecessary memory allocations are avoided when the model is not running at 20Hz, improving efficiency and clarity.

.github/PULL_REQUEST_TEMPLATE/bugfix.md

@@ -1,15 +0,0 @@
----
-name: Bug fix
-about: For openpilot bug fixes
-title: ''
-labels: 'bugfix'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the bug and the fix. Also link the issue if it exists. -->
-
-**Verification**
-
-<!-- Explain how you tested this bug fix. -->

.github/PULL_REQUEST_TEMPLATE/car_bugfix.md

@@ -1,19 +0,0 @@
----
-name: Car Bug fix
-about: For vehicle/brand specific bug fixes
-title: ''
-labels: 'car bug fix'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the bug and the fix. Also link the issue if it exists. -->
-
-**Verification**
-
-<!-- Explain how you tested this bug fix. -->
-
-**Route**
-
-Route: [a route with the bug fix]

.github/PULL_REQUEST_TEMPLATE/car_port.md

@@ -1,15 +0,0 @@
----
-name: Car port
-about: For new car ports
-title: ''
-labels: 'car port'
-assignees: ''
----
-
-**Checklist**
-
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
-- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
-- [ ] route with openpilot:
-- [ ] route with stock system:
-- [ ] car harness used (if comma doesn't sell it, put N/A):

.github/PULL_REQUEST_TEMPLATE/fingerprint.md

@@ -1,13 +0,0 @@
----
-name: Fingerprint
-about: For adding fingerprints to existing cars
-title: ''
-labels: 'fingerprint'
-assignees: ''
----
-
-**Car**
-Which car (make, model, year) this fingerprint is for
-
-**Route**
-A route with the fingerprint

.github/PULL_REQUEST_TEMPLATE/refactor.md

@@ -1,15 +0,0 @@
----
-name: Refactor
-about: For code refactors
-title: ''
-labels: 'refactor'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of the refactor, including the goals it accomplishes. -->
-
-**Verification**
-
-<!-- Explain how you tested the refactor for regressions. -->

.github/PULL_REQUEST_TEMPLATE/tuning.md

@@ -1,31 +0,0 @@
----
-name: Tuning
-about: For openpilot tuning changes
-title: ''
-labels: 'tuning'
-assignees: ''
----
-
-**Description**
-
-<!-- A description of what is wrong with the current tuning and how the PR addresses this. -->
-
-**Verification**
-
-<!-- To verify tuning, capture the following scenarios (broadly, not exactly), with current tune and this tune.
-Use the PlotJuggler tuning layout to compare planned versus actual behavior.
-
-Run ./juggle.py <route> --layout layouts/tuning.xml , screenshot the full tab of interest, and paste into this PR.
-
-Longitudinal:
-* Maintaining speed at 25, 40, 65mph
-* Driving up and down hills
-* Accelerating from a stop
-* Decelerating to a stop
-* Following large changes in set speed
-* Coming to a stop behind a lead car
-
-Lateral:
-* Straight driving at ~25, ~45 and ~65mph
-* Turns driving at ~25, ~45 and ~65mph
--->

.github/build.py

@@ -1,30 +0,0 @@
-import pathlib
-
-GITHUB_FOLDER = pathlib.Path(__file__).parent
-
-PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
-
-order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
-
-def create_pull_request_template():
-  with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
-    f.write("<!-- Please copy and paste the relevant template -->\n\n")
-
-    for t in order:
-      template = PULL_REQUEST_TEMPLATES / f"{t}.md"
-      text = template.read_text()
-
-      # Remove metadata for GitHub
-      start = text.find("---")
-      end = text.find("---", start+1)
-      text = text[end + 4:]
-
-      # Remove comments
-      text = text.replace("<!-- ", "").replace("-->", "")
-
-      f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
-      f.write(text)
-      f.write("\n\n")
-      f.write("-->\n\n")
-
-create_pull_request_template()

.github/release-drafter.yml

@@ -0,0 +1,43 @@
+exclude-labels:
+  - 'no-changelog'
+categories:
+  - title: '🚀 Features'
+    labels:
+      - 'feature'
+      - 'enhancement'
+  - title: '🐛 Bug Fixes'
+    collapse-after: 5
+    labels:
+      - 'fix'
+      - 'bugfix'
+      - 'bug'
+  - title: '🧰 Maintenance'
+    collapse-after: 5
+    label: 'chore'
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+change-title-escapes: '\<*_&'
+replacers:
+  - search: '/[Ss][Uu][Nn][Nn][Yy][Pp][Ii][Ll][Oo][Tt]/g'
+    replace: 'sunnypilot'
+  - search: '/\b[Ss][Pp]\b/g'
+    replace: 'SP'
+version-resolver:
+  major:
+    labels:
+      - 'major'
+  minor:
+    labels:
+      - 'minor'
+  patch:
+    labels:
+      - 'patch'
+  default: patch
+name-template: 'v$RESOLVED_VERSION 🚀'
+tag-template: 'v$RESOLVED_VERSION'
+version-template: "0.$MAJOR.$MINOR.$PATCH" # The day OP becomes v1, we need to bump this
+tag-prefix: "v0." # The day OP becomes v1, we need to bump this
+prerelease-identifier: "staging"
+template: |
+  ## Changes
+
+  $CHANGES

.github/workflows/release-drafter.yml

@@ -0,0 +1,29 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - master-new
+      - master
+    tags:
+      - 'v*'
+  pull_request_target:
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  update_release_draft:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v6
+        with:
+          config-name: release-drafter.yml
+          prerelease: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/selfdrive_tests.yaml

@@ -67,15 +67,7 @@ jobs:
       timeout-minutes: 1
       run: |
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "release/check-dirty.sh && \
-                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car system/manager"
-    - name: Static analysis
-      timeout-minutes: 1
-      run: |
-        cd $GITHUB_WORKSPACE
-        cp pyproject.toml $STRIPPED_DIR
-        cd $STRIPPED_DIR
-        ${{ env.RUN }} "scripts/lint/lint.sh --skip check_added_large_files"
+        ${{ env.RUN }} "release/check-dirty.sh"
 
   build:
     runs-on:
@@ -352,7 +344,7 @@ jobs:
       - name: Build openpilot
         run: ${{ env.RUN }} "scons -j$(nproc)"
       - name: Create Test Report
-        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }}
+        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 2 || 4) }}
         run: >
             ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                             source selfdrive/test/setup_xvfb.sh &&

.github/workflows/sunnypilot-build-model.yaml

@@ -18,6 +18,18 @@ on:
         description: 'Custom name for the model'
         required: false
         type: string
+      file_name:
+        description: 'File name prefix for the model files'
+        required: false
+        type: string
+      is_20hz:
+        description: 'Is this a 20Hz model'
+        required: false
+        type: boolean
+        default: false
+
+
+run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
 
 jobs:
   build_model:
@@ -37,12 +49,15 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
             scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
             scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
-            scons-${{ runner.os }}-${{ runner.arch }}-master-new
-            scons-${{ runner.os }}-${{ runner.arch }}-master
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}
 
       - name: Setup build environment
@@ -69,6 +84,8 @@ jobs:
         run: |
           sudo rm -rf ${OUTPUT_DIR}
           mkdir -p ${OUTPUT_DIR}
+          
+          # Copy the model files
           rsync -avm \
             --include='*.dlc' \
             --include='*.thneed' \
@@ -78,6 +95,36 @@ jobs:
             --delete-excluded \
             --chown=comma:comma \
             ./selfdrive/modeld/models/ ${OUTPUT_DIR}/
+          
+          # Rename files if file_name is provided
+          if [ ! -z "${{ inputs.file_name }}" ]; then
+            mv ${OUTPUT_DIR}/supercombo.thneed ${OUTPUT_DIR}/supercombo-${{ inputs.file_name }}.thneed
+            mv ${OUTPUT_DIR}/supercombo_metadata.pkl ${OUTPUT_DIR}/supercombo-${{ inputs.file_name }}_metadata.pkl
+          fi
+          
+          # Calculate SHA256 hashes
+          HASH=$(sha256sum ${OUTPUT_DIR}/supercombo*.thneed | cut -d' ' -f1)
+          METADATA_HASH=$(sha256sum ${OUTPUT_DIR}/supercombo*_metadata.pkl | cut -d' ' -f1)
+          
+          # Create metadata.json
+          cat > ${OUTPUT_DIR}/metadata.json << EOF
+          {
+            "display_name": "${{ inputs.custom_name || inputs.upstream_branch }}",
+            "full_name": "${{ inputs.file_name || 'default' }}",
+            "is_20hz": ${{ inputs.is_20hz || false }},
+            "files": {
+              "drive_model": {
+                "file_name": "$(basename ${OUTPUT_DIR}/supercombo*.thneed)",
+                "sha256": "${HASH}"
+              },
+              "metadata": {
+                "file_name": "$(basename ${OUTPUT_DIR}/supercombo*_metadata.pkl)",
+                "sha256": "${METADATA_HASH}"
+              }
+            },
+            "ref": "${{ inputs.upstream_branch }}",
+            "build_time": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+          }     
 
       - name: Upload Build Artifacts
         uses: actions/upload-artifact@v4

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -51,9 +51,10 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
+          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }}
+            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
             scons-${{ runner.os }}-${{ runner.arch }}
@@ -119,6 +120,7 @@ jobs:
           if [[ "${{ runner.debug }}" == "1" ]]; then
             printenv
           fi
+          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
 
       - name: Build Panda
         run: |
@@ -187,6 +189,11 @@ jobs:
           name: prebuilt
           path: prebuilt.tar.gz
 
+      - name: Re-enable powersave
+        if: always()
+        run: |
+          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
+
   publish:
     concurrency:
       group: publish-${{ github.head_ref || github.ref_name }}

.github/workflows/ui_preview.yaml

@@ -86,7 +86,7 @@ jobs:
         run: >-
           sudo apt-get install -y imagemagick
 
-          scenes="homescreen settings_device settings_software settings_sunnylink settings_toggles settings_sunnypilot settings_sunnypilot_mads settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard"
+          scenes="homescreen settings_device settings_software settings_sunnylink settings_toggles settings_sunnypilot settings_sunnypilot_mads settings_trips settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard keyboard_uppercase"
           A=($scenes)
 
           DIFF=""

.gitignore

@@ -74,6 +74,9 @@ comma*.sh
 selfdrive/modeld/thneed/compile
 selfdrive/modeld/models/*.thneed
 selfdrive/modeld/models/*.pkl
+sunnypilot/modeld/thneed/compile
+sunnypilot/modeld/models/*.thneed
+sunnypilot/modeld/models/*.pkl
 
 *.bz2
 *.zst

.gitmodules

@@ -15,4 +15,4 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/tinygrad/tinygrad.git
+  url = https://github.com/commaai/tinygrad.git

RELEASES.md

@@ -1,16 +1,23 @@
-Version 0.9.8 (2024-XX-XX)
+Version 0.9.9 (2025-03-30)
+========================
+* Coming soon
+  * Rivian support
+  * F-150 & Mach-E support
+  * Tesla Model 3 support
+
+Version 0.9.8 (2025-01-30)
 ========================
-* New driving model
-  * Trained in brand new ML simulator
-  * Model now gates applying positive accel in Chill mode
 * New driving monitoring model
   * Reduced false positives related to passengers
 * Image processing pipeline moved to the ISP
   * More GPU time for driving models
-  * Power draw reduced 0.5W, which means your device runs cooler 
+  * Power draw reduced 0.5W, which means your device runs cooler
 * Added toggle to enable driver monitoring even when openpilot is not engaged
 * Enable openpilot longitudinal control for Ford Q3 vehicles
 * New Toyota TSS2 longitudinal tune
+* Coming soon
+  * New driving model with gas gating
+  * Training data upload mode
 
 Version 0.9.7 (2024-06-13)
 ========================

SConstruct

@@ -49,10 +49,6 @@ AddOption('--ccflags',
           default='',
           help='pass arbitrary flags over the command line')
 
-AddOption('--snpe',
-          action='store_true',
-          help='use SNPE on PC')
-
 AddOption('--external-sconscript',
           action='store',
           metavar='FILE',
@@ -396,6 +392,8 @@ SConscript(['third_party/SConscript'])
 
 SConscript(['selfdrive/SConscript'])
 
+SConscript(['sunnypilot/SConscript'])
+
 if Dir('#tools/cabana/').exists() and GetOption('extras'):
   SConscript(['tools/replay/SConscript'])
   if arch != "larch64":

cereal/README.md

@@ -29,6 +29,50 @@ then means breaking backwards-compatibility with all old logs of your fork. So w
 [custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
+An example of compatible changes:
+```diff
+diff --git a/cereal/custom.capnp b/cereal/custom.capnp
+index 3348e859e..3365c7b98 100644
+--- a/cereal/custom.capnp
++++ b/cereal/custom.capnp
+@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
+ # DO rename the structs
+ # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
+-struct CustomReserved0 @0x81c2f05a394cf4af {
++struct SteeringInfo @0x81c2f05a394cf4af {
++  active @0 :Bool;
++  steeringAngleDeg @1 :Float32;
++  steeringRateDeg @2 :Float32;
++  steeringAccelDeg @3 :Float32;
+ }
+
+ struct CustomReserved1 @0xaedffd8f31e7b55d {
+diff --git a/cereal/log.capnp b/cereal/log.capnp
+index 1209f3fd9..b189f58b6 100644
+--- a/cereal/log.capnp
++++ b/cereal/log.capnp
+@@ -2558,14 +2558,14 @@ struct Event {
+
+     # DO change the name of the field
+     # DON'T change anything after the "@"
+-    customReservedRawData0 @124 :Data;
++    rawCanData @124 :Data;
+     customReservedRawData1 @125 :Data;
+     customReservedRawData2 @126 :Data;
+
+     # DO change the name of the field and struct
+     # DON'T change the ID (e.g. @107)
+     # DON'T change which struct it points to
+-    customReserved0 @107 :Custom.CustomReserved0;
++    steeringInfo @107 :Custom.SteeringInfo;
+     customReserved1 @108 :Custom.CustomReserved1;
+     customReserved2 @109 :Custom.CustomReserved2;
+     customReserved3 @110 :Custom.CustomReserved3;
+```
+
+---
+
 Example
 ---
 ```python

cereal/custom.capnp

@@ -7,7 +7,9 @@ $Cxx.namespace("cereal");
 # These structs are guaranteed to remain reserved and empty in mainline
 # cereal, so use these if you want custom events in your fork.
 
-# you can rename the struct, but don't change the identifier
+# DO rename the structs
+# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
+
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
   mads @0 :ModularAssistiveDrivingSystem;
 
@@ -65,6 +67,11 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
     eta @2 :UInt32;
   }
 
+  enum Runner {
+    snpe @0;
+    tinygrad @1;
+  }
+
   struct ModelBundle {
     index @0 :UInt32;
     internalName @1 :Text;
@@ -73,16 +80,62 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
     status @4 :DownloadStatus;
     generation @5 :UInt32;
     environment @6 :Text;
+    runner @7 :Runner;
+    is20hz @8 :Bool;
   }
 }
 
-struct CustomReserved2 @0xf35cc4560bbf6ec2 {
+struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
+  dec @0 :DynamicExperimentalControl;
+
+  struct DynamicExperimentalControl {
+    state @0 :DynamicExperimentalControlState;
+    enabled @1 :Bool;
+    active @2 :Bool;
+
+    enum DynamicExperimentalControlState {
+      acc @0;
+      blended @1;
+    }
+  }
 }
 
-struct CustomReserved3 @0xda96579883444c35 {
+struct OnroadEventSP @0xda96579883444c35 {
+  name @0 :EventName;
+
+  # event types
+  enable @1 :Bool;
+  noEntry @2 :Bool;
+  warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
+  userDisable @4 :Bool;
+  softDisable @5 :Bool;
+  immediateDisable @6 :Bool;
+  preEnable @7 :Bool;
+  permanent @8 :Bool; # alerts presented regardless of openpilot state
+  overrideLateral @10 :Bool;
+  overrideLongitudinal @9 :Bool;
+
+  enum EventName {
+    lkasEnable @0;
+    lkasDisable @1;
+    manualSteeringRequired @2;
+    manualLongitudinalRequired @3;
+    silentLkasEnable @4;
+    silentLkasDisable @5;
+    silentBrakeHold @6;
+    silentWrongGear @7;
+    silentReverseGear @8;
+    silentDoorOpen @9;
+    silentSeatbeltNotLatched @10;
+    silentParkBrake @11;
+    controlsMismatchLateral @12;
+    hyundaiRadarTracksConfirmed @13;
+    experimentalModeSwitched @14;
+  }
 }
 
-struct CustomReserved4 @0x80ae746ee2596b11 {
+struct CarParamsSP @0x80ae746ee2596b11 {
+  flags @0 :UInt32;  # flags for car specific quirks in sunnypilot
 }
 
 struct CustomReserved5 @0xa5cd762cd951a455 {
@@ -99,3 +152,33 @@ struct CustomReserved8 @0xf416ec09499d9d19 {
 
 struct CustomReserved9 @0xa1680744031fdb2d {
 }
+
+struct CustomReserved10 @0xcb9fd56c7057593a {
+}
+
+struct CustomReserved11 @0xc2243c65e0340384 {
+}
+
+struct CustomReserved12 @0x9ccdc8676701b412 {
+}
+
+struct CustomReserved13 @0xcd96dafb67a082d0 {
+}
+
+struct CustomReserved14 @0xb057204d7deadf3f {
+}
+
+struct CustomReserved15 @0xbd443b539493bc68 {
+}
+
+struct CustomReserved16 @0xfc6241ed8877b611 {
+}
+
+struct CustomReserved17 @0xa30662f84033036c {
+}
+
+struct CustomReserved18 @0xc86a3d38d13eb3ef {
+}
+
+struct CustomReserved19 @0xa4f1eb3323f5f582 {
+}

cereal/log.capnp

@@ -125,79 +125,6 @@ struct OnroadEvent @0xc4fa6047f024e718 {
     espActive @90;
     personalityChanged @91;
     aeb @92;
-    eventReserved93 @93;
-    eventReserved94 @94;
-    eventReserved95 @95;
-    eventReserved96 @96;
-    eventReserved97 @97;
-    eventReserved98 @98;
-    eventReserved99 @99;
-    eventReserved100 @100;
-    eventReserved101 @101;
-    eventReserved102 @102;
-    eventReserved103 @103;
-    eventReserved104 @104;
-    eventReserved105 @105;
-    eventReserved106 @106;
-    eventReserved107 @107;
-    eventReserved108 @108;
-    eventReserved109 @109;
-    eventReserved110 @110;
-    eventReserved111 @111;
-    eventReserved112 @112;
-    eventReserved113 @113;
-    eventReserved114 @114;
-    eventReserved115 @115;
-    eventReserved116 @116;
-    eventReserved117 @117;
-    eventReserved118 @118;
-    eventReserved119 @119;
-    eventReserved120 @120;
-    eventReserved121 @121;
-    eventReserved122 @122;
-    eventReserved123 @123;
-    eventReserved124 @124;
-    eventReserved125 @125;
-    eventReserved126 @126;
-    eventReserved127 @127;
-    eventReserved128 @128;
-    eventReserved129 @129;
-    eventReserved130 @130;
-    eventReserved131 @131;
-    eventReserved132 @132;
-    eventReserved133 @133;
-    eventReserved134 @134;
-    eventReserved135 @135;
-    eventReserved136 @136;
-    eventReserved137 @137;
-    eventReserved138 @138;
-    eventReserved139 @139;
-    eventReserved140 @140;
-    eventReserved141 @141;
-    eventReserved142 @142;
-    eventReserved143 @143;
-    eventReserved144 @144;
-    eventReserved145 @145;
-    eventReserved146 @146;
-    eventReserved147 @147;
-    eventReserved148 @148;
-    eventReserved149 @149;
-    eventReserved150 @150;
-
-    # sunnypilot
-    lkasEnable @151;
-    lkasDisable @152;
-    manualSteeringRequired @153;
-    manualLongitudinalRequired @154;
-    silentLkasEnable @155;
-    silentLkasDisable @156;
-    silentBrakeHold @157;
-    silentWrongGear @158;
-    silentReverseGear @159;
-    silentDoorOpen @160;
-    silentSeatbeltNotLatched @161;
-    silentParkBrake @162;
-    controlsMismatchLateral @163;
 
     soundsUnavailableDEPRECATED @47;
   }
@@ -482,6 +409,7 @@ struct GpsLocationData {
   speedAccuracy @12 :Float32;
 
   hasFix @13 :Bool;
+  satelliteCount @14 :Int8;
 
   enum SensorSource {
     android @0;
@@ -662,7 +590,6 @@ struct PandaState @0xa7649e2575e4591e {
 
   # safety stuff
   controlsAllowed @3 :Bool;
-  controlsAllowedLat @5 :Bool;
   safetyRxInvalid @19 :UInt32;
   safetyTxBlocked @24 :UInt32;
   safetyModel @14 :Car.CarParams.SafetyModel;
@@ -770,6 +697,7 @@ struct PandaState @0xa7649e2575e4591e {
   }
 
   gasInterceptorDetectedDEPRECATED @4 :Bool;
+  startedSignalDetectedDEPRECATED @5 :Bool;
   hasGpsDEPRECATED @6 :Bool;
   gmlanSendErrsDEPRECATED @9 :UInt32;
   fanSpeedRpmDEPRECATED @11 :UInt16;
@@ -2626,21 +2554,37 @@ struct Event {
     livestreamWideRoadEncodeData @121 :EncodeData;
     livestreamDriverEncodeData @122 :EncodeData;
 
+    # *********** Custom: reserved for forks ***********
+
+    # DO change the name of the field
+    # DON'T change anything after the "@"
     customReservedRawData0 @124 :Data;
     customReservedRawData1 @125 :Data;
     customReservedRawData2 @126 :Data;
 
-    # *********** Custom: reserved for forks ***********
+    # DO change the name of the field and struct
+    # DON'T change the ID (e.g. @107)
+    # DON'T change which struct it points to
     selfdriveStateSP @107 :Custom.SelfdriveStateSP;
     modelManagerSP @108 :Custom.ModelManagerSP;
-    customReserved2 @109 :Custom.CustomReserved2;
-    customReserved3 @110 :Custom.CustomReserved3;
-    customReserved4 @111 :Custom.CustomReserved4;
+    longitudinalPlanSP @109 :Custom.LongitudinalPlanSP;
+    onroadEventsSP @110 :List(Custom.OnroadEventSP);
+    carParamsSP @111 :Custom.CarParamsSP;
     customReserved5 @112 :Custom.CustomReserved5;
     customReserved6 @113 :Custom.CustomReserved6;
     customReserved7 @114 :Custom.CustomReserved7;
     customReserved8 @115 :Custom.CustomReserved8;
     customReserved9 @116 :Custom.CustomReserved9;
+    customReserved10 @136 :Custom.CustomReserved10;
+    customReserved11 @137 :Custom.CustomReserved11;
+    customReserved12 @138 :Custom.CustomReserved12;
+    customReserved13 @139 :Custom.CustomReserved13;
+    customReserved14 @140 :Custom.CustomReserved14;
+    customReserved15 @141 :Custom.CustomReserved15;
+    customReserved16 @142 :Custom.CustomReserved16;
+    customReserved17 @143 :Custom.CustomReserved17;
+    customReserved18 @144 :Custom.CustomReserved18;
+    customReserved19 @145 :Custom.CustomReserved19;
 
     # *********** legacy + deprecated ***********
     model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated

cereal/maptile.capnp

@@ -1,49 +0,0 @@
-using Cxx = import "./include/c++.capnp";
-$Cxx.namespace("cereal");
-
-@0xa086df597ef5d7a0;
-
-# Geometry
-struct Point {
-  x @0: Float64;
-  y @1: Float64;
-  z @2: Float64;
-}
-
-struct PolyLine {
-  points @0: List(Point);
-}
-
-# Map features
-struct Lane {
-  id @0 :Text;
-
-  leftBoundary @1 :LaneBoundary;
-  rightBoundary @2 :LaneBoundary;
-
-  leftAdjacentId @3 :Text;
-  rightAdjacentId @4 :Text;
-
-  inboundIds @5 :List(Text);
-  outboundIds @6 :List(Text);
-
-  struct LaneBoundary {
-    polyLine @0 :PolyLine;
-    startHeading @1 :Float32; # WRT north
-  }
-}
-
-# Map tiles
-struct TileSummary {
-  version @0 :Text;
-  updatedAt @1 :UInt64; # Millis since epoch
-
-  level @2 :UInt8;
-  x @3 :UInt16;
-  y @4 :UInt16;
-}
-
-struct MapTile {
-  summary @0 :TileSummary;
-  lanes @1 :List(Lane);
-}

cereal/services.py

@@ -77,6 +77,9 @@ _services: dict[str, tuple] = {
   # sunnypilot
   "modelManagerSP": (False, 1., 1),
   "selfdriveStateSP": (True, 100., 10),
+  "longitudinalPlanSP": (True, 20., 10),
+  "onroadEventsSP": (True, 1., 1),
+  "carParamsSP": (True, 0.02, 1),
 
   # debug
   "uiDebug": (True, 0., 1),

common/clutil.cc

@@ -79,6 +79,10 @@ cl_context cl_create_context(cl_device_id device_id) {
   return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
 }
 
+void cl_release_context(cl_context context) {
+  clReleaseContext(context);
+}
+
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
   return cl_program_from_source(ctx, device_id, util::read_file(path), args);
 }

common/clutil.h

@@ -23,6 +23,7 @@
 
 cl_device_id cl_get_device_id(cl_device_type device_type);
 cl_context cl_create_context(cl_device_id device_id);
+void cl_release_context(cl_context context);
 cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
 cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/params.cc

@@ -133,7 +133,6 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"GsmRoaming", PERSISTENT | BACKUP},
     {"HardwareSerial", PERSISTENT},
     {"HasAcceptedTerms", PERSISTENT},
-    {"IMEI", PERSISTENT},
     {"InstallDate", PERSISTENT},
     {"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
     {"IsEngaged", PERSISTENT},
@@ -202,7 +201,14 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"Version", PERSISTENT},
 
     // --- sunnypilot params --- //
+    {"ApiCache_DriveStats", PERSISTENT},
+    {"CarParamsSP", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsSPCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsSPPersistent", PERSISTENT},
     {"EnableGithubRunner", PERSISTENT | BACKUP},
+    {"ModelRunnerTypeCache", CLEAR_ON_ONROAD_TRANSITION},
+    {"OffroadMode", CLEAR_ON_MANAGER_START},
+    {"OffroadMode_Status", CLEAR_ON_MANAGER_START},
 
     // MADS params
     {"Mads", PERSISTENT | BACKUP},
@@ -212,7 +218,7 @@ std::unordered_map<std::string, uint32_t> keys = {
 
     // Model Manager params
     {"ModelManager_ActiveBundle", PERSISTENT},
-    {"ModelManager_DownloadIndex", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_ONROAD_TRANSITION},
+    {"ModelManager_DownloadIndex", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"ModelManager_LastSyncTime", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"ModelManager_ModelsCache", PERSISTENT | BACKUP},
 
@@ -222,6 +228,14 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"SunnylinkDongleId", PERSISTENT},
     {"SunnylinkdPid", PERSISTENT},
     {"SunnylinkEnabled", PERSISTENT},
+
+    // sunnypilot car specific params
+    {"HyundaiRadarTracks", PERSISTENT},
+    {"HyundaiRadarTracksConfirmed", PERSISTENT},
+    {"HyundaiRadarTracksPersistent", PERSISTENT},
+    {"HyundaiRadarTracksToggle", PERSISTENT},
+
+    {"DynamicExperimentalControl", PERSISTENT},
 };
 
 } // namespace

common/pid.py

@@ -1,9 +1,6 @@
 import numpy as np
 from numbers import Number
 
-from openpilot.common.numpy_fast import clip, interp
-
-
 class PIDController:
   def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
     self._k_p = k_p
@@ -28,15 +25,15 @@ class PIDController:
 
   @property
   def k_p(self):
-    return interp(self.speed, self._k_p[0], self._k_p[1])
+    return np.interp(self.speed, self._k_p[0], self._k_p[1])
 
   @property
   def k_i(self):
-    return interp(self.speed, self._k_i[0], self._k_i[1])
+    return np.interp(self.speed, self._k_i[0], self._k_i[1])
 
   @property
   def k_d(self):
-    return interp(self.speed, self._k_d[0], self._k_d[1])
+    return np.interp(self.speed, self._k_d[0], self._k_d[1])
 
   @property
   def error_integral(self):
@@ -64,10 +61,10 @@ class PIDController:
 
         # Clip i to prevent exceeding control limits
         control_no_i = self.p + self.d + self.f
-        control_no_i = clip(control_no_i, self.neg_limit, self.pos_limit)
-        self.i = clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
+        control_no_i = np.clip(control_no_i, self.neg_limit, self.pos_limit)
+        self.i = np.clip(self.i, self.neg_limit - control_no_i, self.pos_limit - control_no_i)
 
     control = self.p + self.i + self.d + self.f
 
-    self.control = clip(control, self.neg_limit, self.pos_limit)
+    self.control = np.clip(control, self.neg_limit, self.pos_limit)
     return self.control

common/swaglog.cc

@@ -26,6 +26,9 @@ public:
     zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
     zmq_connect(sock, Path::swaglog_ipc().c_str());
 
+    // workaround for https://github.com/dropbox/json11/issues/38
+    setlocale(LC_NUMERIC, "C");
+
     print_level = CLOUDLOG_WARNING;
     if (const char* print_lvl = getenv("LOGPRINT")) {
       if (strcmp(print_lvl, "debug") == 0) {

common/tests/test_numpy_fast.py

@@ -1,21 +0,0 @@
-import numpy as np
-
-from openpilot.common.numpy_fast import interp
-
-
-class TestInterp:
-  def test_correctness_controls(self):
-    _A_CRUISE_MIN_BP = np.asarray([0., 5., 10., 20., 40.])
-    _A_CRUISE_MIN_V = np.asarray([-1.0, -.8, -.67, -.5, -.30])
-    v_ego_arr = [-1, -1e-12, 0, 4, 5, 6, 7, 10, 11, 15.2, 20, 21, 39,
-                 39.999999, 40, 41]
-
-    expected = np.interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-    actual = interp(v_ego_arr, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-
-    np.testing.assert_equal(actual, expected)
-
-    for v_ego in v_ego_arr:
-      expected = np.interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-      actual = interp(v_ego, _A_CRUISE_MIN_BP, _A_CRUISE_MIN_V)
-      np.testing.assert_equal(actual, expected)

common/util.cc

@@ -255,6 +255,29 @@ bool ends_with(const std::string& s, const std::string& suffix) {
          strcmp(s.c_str() + (s.size() - suffix.size()), suffix.c_str()) == 0;
 }
 
+std::string strip(const std::string &str) {
+  auto should_trim = [](unsigned char ch) {
+    // trim whitespace or a null character
+    return std::isspace(ch) || ch == '\0';
+  };
+
+  size_t start = 0;
+  while (start < str.size() && should_trim(static_cast<unsigned char>(str[start]))) {
+    start++;
+  }
+
+  if (start == str.size()) {
+    return "";
+  }
+
+  size_t end = str.size() - 1;
+  while (end > 0 && should_trim(static_cast<unsigned char>(str[end]))) {
+    end--;
+  }
+
+  return str.substr(start, end - start + 1);
+}
+
 std::string check_output(const std::string& command) {
   char buffer[128];
   std::string result;
@@ -271,4 +294,17 @@ std::string check_output(const std::string& command) {
   return result;
 }
 
+bool system_time_valid() {
+  // Default to August 26, 2024
+  tm min_tm = {.tm_year = 2024 - 1900, .tm_mon = 7, .tm_mday = 26};
+  time_t min_date = mktime(&min_tm);
+
+  struct stat st;
+  if (stat("/lib/systemd/systemd", &st) == 0) {
+    min_date = std::max(min_date, st.st_mtime + 86400);  // Add 1 day (86400 seconds)
+  }
+
+  return time(nullptr) > min_date;
+}
+
 }  // namespace util

common/util.h

@@ -74,6 +74,7 @@ float getenv(const char* key, float default_val);
 std::string hexdump(const uint8_t* in, const size_t size);
 bool starts_with(const std::string &s1, const std::string &s2);
 bool ends_with(const std::string &s, const std::string &suffix);
+std::string strip(const std::string &str);
 
 // ***** random helpers *****
 int random_int(int min, int max);
@@ -95,6 +96,8 @@ bool create_directories(const std::string &dir, mode_t mode);
 
 std::string check_output(const std::string& command);
 
+bool system_time_valid();
+
 inline void sleep_for(const int milliseconds) {
   if (milliseconds > 0) {
     std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));

docs/CARS.md

@@ -4,7 +4,7 @@
 
 A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
 
-# 289 Supported Cars
+# 290 Supported Cars
 
 |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br>&nbsp;|Video|
 |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
@@ -56,6 +56,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Genesis|GV60 (Performance Trim) 2022-23[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV60 (Performance Trim) 2022-23">Buy Here</a></sub></details>||
 |Genesis|GV70 (2.5T Trim, without HDA II) 2022-23[<sup>5</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai L connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 (2.5T Trim, without HDA II) 2022-23">Buy Here</a></sub></details>||
 |Genesis|GV70 (3.5T Trim, without HDA II) 2022-23[<sup>5</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai M connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 (3.5T Trim, without HDA II) 2022-23">Buy Here</a></sub></details>||
+|Genesis|GV70 Electrified (Australia Only) 2022[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 Electrified (Australia Only) 2022">Buy Here</a></sub></details>||
 |Genesis|GV70 Electrified (with HDA II) 2023[<sup>5</sup>](#footnotes)|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV70 Electrified (with HDA II) 2023">Buy Here</a></sub></details>||
 |Genesis|GV80 2023[<sup>5</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai M connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Genesis&model=GV80 2023">Buy Here</a></sub></details>||
 |GMC|Sierra 1500 2020-21|Driver Alert Package II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|6 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 GM connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=GMC&model=Sierra 1500 2020-21">Buy Here</a></sub></details>|<a href="https://youtu.be/5HbNoBLzRwE" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
@@ -79,7 +80,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Honda|Odyssey 2018-20|Honda Sensing|openpilot|26 mph|0 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Odyssey 2018-20">Buy Here</a></sub></details>||
 |Honda|Passport 2019-23|All|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Passport 2019-23">Buy Here</a></sub></details>||
 |Honda|Pilot 2016-22|Honda Sensing|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Pilot 2016-22">Buy Here</a></sub></details>||
-|Honda|Ridgeline 2017-24|Honda Sensing|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Ridgeline 2017-24">Buy Here</a></sub></details>||
+|Honda|Ridgeline 2017-25|Honda Sensing|openpilot|26 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Ridgeline 2017-25">Buy Here</a></sub></details>||
 |Hyundai|Azera 2022|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Azera 2022">Buy Here</a></sub></details>||
 |Hyundai|Azera Hybrid 2019|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Azera Hybrid 2019">Buy Here</a></sub></details>||
 |Hyundai|Azera Hybrid 2020|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Azera Hybrid 2020">Buy Here</a></sub></details>||
@@ -87,11 +88,11 @@ A supported vehicle is one that just works when you install a comma device. All
 |Hyundai|Elantra 2017-18|Smart Cruise Control (SCC)|Stock|19 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra 2017-18">Buy Here</a></sub></details>||
 |Hyundai|Elantra 2019|Smart Cruise Control (SCC)|Stock|19 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra 2019">Buy Here</a></sub></details>||
 |Hyundai|Elantra 2021-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra 2021-23">Buy Here</a></sub></details>|<a href="https://youtu.be/_EdYQtV52-c" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-|Hyundai|Elantra GT 2017-19|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra GT 2017-19">Buy Here</a></sub></details>||
+|Hyundai|Elantra GT 2017-20|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra GT 2017-20">Buy Here</a></sub></details>||
 |Hyundai|Elantra Hybrid 2021-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Elantra Hybrid 2021-23">Buy Here</a></sub></details>|<a href="https://youtu.be/_EdYQtV52-c" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Hyundai|Genesis 2015-16|Smart Cruise Control (SCC)|Stock|19 mph|37 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai J connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Genesis 2015-16">Buy Here</a></sub></details>||
 |Hyundai|i30 2017-19|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=i30 2017-19">Buy Here</a></sub></details>||
-|Hyundai|Ioniq 5 (Non-US only) 2022-24[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (Non-US only) 2022-24">Buy Here</a></sub></details>||
+|Hyundai|Ioniq 5 (Southeast Asia and Europe only) 2022-24[<sup>5</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (Southeast Asia and Europe only) 2022-24">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 5 (with HDA II) 2022-24[<sup>5</sup>](#footnotes)|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (with HDA II) 2022-24">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 5 (without HDA II) 2022-24[<sup>5</sup>](#footnotes)|Highway Driving Assist|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (without HDA II) 2022-24">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 6 (with HDA II) 2023-24[<sup>5</sup>](#footnotes)|Highway Driving Assist II|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai P connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 6 (with HDA II) 2023-24">Buy Here</a></sub></details>||
@@ -294,7 +295,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Volkswagen|Teramont 2018-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Teramont 2018-22">Buy Here</a></sub></details>||
 |Volkswagen|Teramont Cross Sport 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Teramont Cross Sport 2021-22">Buy Here</a></sub></details>||
 |Volkswagen|Teramont X 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Teramont X 2021-22">Buy Here</a></sub></details>||
-|Volkswagen|Tiguan 2018-24|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Tiguan 2018-24">Buy Here</a></sub></details>||
+|Volkswagen|Tiguan 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Tiguan 2018-23">Buy Here</a></sub></details>||
 |Volkswagen|Tiguan eHybrid 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Tiguan eHybrid 2021-23">Buy Here</a></sub></details>||
 |Volkswagen|Touran 2016-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Touran 2016-23">Buy Here</a></sub></details>||
 

pyproject.toml

@@ -103,7 +103,6 @@ dev = [
   "lru-dict",
   "matplotlib",
   "parameterized >=0.8, <0.9",
-  #"pprofile",
   "pyautogui",
   "pyopencl; platform_machine != 'aarch64'", # broken on arm64
   "pytools < 2024.1.11; platform_machine != 'aarch64'", # pyopencl use a broken version
@@ -112,13 +111,10 @@ dev = [
   "tabulate",
   "types-requests",
   "types-tabulate",
-
-  # this is only pinned since 5.15.11 is broken
-  "pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
 ]
 
 tools = [
-  "metadrive-simulator @ https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl ; (platform_machine != 'aarch64')",
+  "metadrive-simulator @ https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl ; (platform_machine != 'aarch64')",
   "rerun-sdk >= 0.18",
 ]
 

release/ci/install_github_runner.sh

@@ -4,6 +4,7 @@ set -e
 # Default values
 DEFAULT_REPO_URL="https://github.com/sunnypilot"
 START_AT_BOOT=false
+RESTORE_MODE=false
 
 # Parse command line arguments
 while [[ $# -gt 0 ]]; do
@@ -20,6 +21,10 @@ while [[ $# -gt 0 ]]; do
             REPO_URL="$2"
             shift 2
             ;;
+        --restore)
+            RESTORE_MODE=true
+            shift
+            ;;
         *)
             if [ -z "$GITHUB_TOKEN" ]; then
                 GITHUB_TOKEN="$1"
@@ -31,19 +36,6 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-# Check required arguments
-if [ -z "$GITHUB_TOKEN" ]; then
-    echo "Usage: $0 [--start-at-boot] [--token <github_token>] [--repo <repository_url>]"
-    echo "Required argument: github_token"
-    echo "Optional arguments:"
-    echo "  --start-at-boot    Enable auto-start at boot (default: false)"
-    echo "  --repo            Repository URL (default: ${DEFAULT_REPO_URL})"
-    exit 1
-fi
-
-# Set repository URL if not provided
-REPO_URL="${REPO_URL:-$DEFAULT_REPO_URL}"
-
 # Determine BASE_DIR based on mount point
 if mountpoint -q /data/media; then
     BASE_DIR="/data/media/0/github"
@@ -60,20 +52,19 @@ LOGS_DIR="${BASE_DIR}/logs"
 CACHE_DIR="${BASE_DIR}/cache"
 OPENPILOT_DIR="${BASE_DIR}/openpilot"
 
-create_directories() {
-    sudo mkdir -p "$RUNNER_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR"
-    mkdir -p "/data/openpilot"
-    sudo chown -R comma:comma "/data/openpilot"
+# Basic utility functions (no dependencies)
+remount_rw() {
+    sudo mount -o remount,rw /
 }
 
-download_and_setup_runner() {
-    cd "$RUNNER_DIR"
-    curl -o actions-runner-linux-arm64-2.321.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.321.0/actions-runner-linux-arm64-2.321.0.tar.gz
-    tar xzf ./actions-runner-linux-arm64-2.321.0.tar.gz
-    rm ./actions-runner-linux-arm64-2.321.0.tar.gz
-    chmod +x ./config.sh
+remount_ro() {
+    sync || true  # Try to sync but continue even if it fails
+    sudo mount -o remount,ro /  # Always try to remount as read-only
 }
 
+# Always ensure we try to remount as read-only on exit
+trap remount_ro EXIT
+
 setup_runner_user() {
     sudo useradd --comment 'GitHub Runner' --create-home --home-dir ${BASE_DIR} ${RUNNER_USER} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${RUNNER_USER}
     export BASE_DIR
@@ -84,18 +75,45 @@ create_sudoers_entry() {
     sudo grep -qxF "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers
 }
 
-configure_runner() {
-    cd "$RUNNER_DIR"
-    sudo -u ${RUNNER_USER} ./config.sh --url "$REPO_URL" --token "$GITHUB_TOKEN" --name $(hostname) --runnergroup "tici-tizi" --labels "tici" --work "$BUILDS_DIR" --unattended
-}
-
 set_directory_permissions() {
     sudo chown -R ${RUNNER_USER}:comma "$BASE_DIR"
     sudo chmod g+rwx "$BASE_DIR"
     sudo chmod g+s "$BASE_DIR"
 }
 
-modify_service_template() {
+setup_directories() {
+    echo "Creating necessary directories..."
+    sudo mkdir -p "$RUNNER_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR"
+    mkdir -p "/data/openpilot"
+    sudo chown -R comma:comma "/data/openpilot"
+}
+
+# System configuration functions (depends on basic utility functions)
+setup_system_configs() {
+    echo "Setting up system configurations..."
+    setup_runner_user
+    create_sudoers_entry
+    set_directory_permissions
+}
+
+# Runner setup functions
+install_runner() {
+    echo "Downloading and setting up runner..."
+    cd "$RUNNER_DIR"
+    curl -o actions-runner-linux-arm64-2.321.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.321.0/actions-runner-linux-arm64-2.321.0.tar.gz
+    tar xzf ./actions-runner-linux-arm64-2.321.0.tar.gz
+    rm ./actions-runner-linux-arm64-2.321.0.tar.gz
+    chmod +x ./config.sh
+}
+
+configure_runner() {
+    echo "Configuring runner..."
+    cd "$RUNNER_DIR"
+    sudo -u ${RUNNER_USER} ./config.sh --url "$REPO_URL" --token "$GITHUB_TOKEN" --name $(hostname) --runnergroup "tici-tizi" --labels "tici" --work "$BUILDS_DIR" --unattended
+}
+
+create_service_template() {
+    echo "Creating service template..."
     cat <<EOL > "$RUNNER_DIR/bin/actions.runner.service.template"
 [Unit]
 Description={{Description}}
@@ -120,28 +138,116 @@ WantedBy=multi-user.target
 EOL
 }
 
-# Make filesystem writable
-sudo mount -o remount,rw /
+install_service() {
+    echo "Installing systemd service..."
+    cd "$RUNNER_DIR"
+    sudo ./svc.sh install $RUNNER_USER
 
-# Ensure filesystem is remounted as read-only on script exit
-trap "sudo mount -o remount,ro /" EXIT
+    if [ "$START_AT_BOOT" = false ]; then
+        local service_name
+        if [ -f "${RUNNER_DIR}/.service" ]; then
+            service_name=$(cat "${RUNNER_DIR}/.service")
+        else
+            service_name="actions.runner.sunnypilot.$(uname -n)"
+        fi
+        sudo systemctl disable "${service_name}"
+    fi
+}
 
-# Execute installation steps
-setup_runner_user
-create_sudoers_entry
-create_directories
-download_and_setup_runner
-modify_service_template
-configure_runner
-set_directory_permissions
+check_restore_prerequisites() {
+    local needs_restore=false
+    local can_restore=false
+    local service_name=""
 
-# Install and start service using built-in installer
-cd "$RUNNER_DIR"
-sudo ./svc.sh install $RUNNER_USER
+    # Check if base runner directory exists
+    if [ ! -d "${RUNNER_DIR}" ]; then
+        echo "ERROR: Runner directory ${RUNNER_DIR} does not exist"
+        echo "This directory is required for restore operations"
+        exit 1
+    fi
 
-# Handle auto-start configuration
-if [ "$START_AT_BOOT" = false ]; then
-    sudo systemctl disable actions.runner.sunnypilot.$(uname -n)
-fi
+    # First check if we have the required files for restoration
+    if [ -f "${RUNNER_DIR}/.credentials" ] && [ -f "${RUNNER_DIR}/.service" ]; then
+        can_restore=true
+        service_name=$(cat "${RUNNER_DIR}/.service")
+        echo "Found required runner configuration files"
+    else
+        echo "Missing required runner configuration files"
+        echo "Required: .credentials and .service files in ${RUNNER_DIR}"
+        exit 1
+    fi
 
-sudo ./svc.sh start
+    # Then check if restoration is needed (if either service or user is missing)
+    if ! systemctl list-unit-files "${service_name}" &>/dev/null; then
+        echo "Service ${service_name} not found in systemd"
+        needs_restore=true
+    fi
+
+    if ! id "${RUNNER_USER}" &>/dev/null; then
+        echo "User ${RUNNER_USER} does not exist"
+        needs_restore=true
+    fi
+
+    # Only proceed if we can restore AND need to restore
+    if [ "$can_restore" = true ] && [ "$needs_restore" = true ]; then
+        echo "Restoration is needed and possible"
+        return 0
+    else
+        if [ "$needs_restore" = false ]; then
+            echo "System is already properly configured (user and service exist)"
+        fi
+        exit 0
+    fi
+}
+
+perform_restore() {
+    echo "Starting runner restoration..."
+    setup_directories
+    remount_rw
+    setup_system_configs
+    install_service
+    remount_ro
+    echo "Runner restoration completed successfully"
+}
+
+perform_install() {
+    echo "Starting fresh installation..."
+    setup_directories
+    install_runner
+    create_service_template
+    remount_rw
+    setup_system_configs
+    configure_runner
+    install_service
+    remount_ro
+    echo "Installation completed successfully"
+}
+
+main() {
+    if [ "$RESTORE_MODE" = true ]; then
+        echo "Running in restore mode - will only restore system configurations..."
+        check_restore_prerequisites
+        perform_restore
+    else
+        # Check required arguments for normal installation
+        if [ -z "$GITHUB_TOKEN" ]; then
+            echo "Usage: $0 [--start-at-boot] [--token <github_token>] [--repo <repository_url>] [--restore]"
+            echo "Required argument (except for --restore): github_token"
+            echo "Optional arguments:"
+            echo "  --start-at-boot    Enable auto-start at boot (default: false)"
+            echo "  --repo            Repository URL (default: ${DEFAULT_REPO_URL})"
+            echo "  --restore         Restore existing runner configuration"
+            exit 1
+        fi
+
+        # Set repository URL if not provided
+        REPO_URL="${REPO_URL:-$DEFAULT_REPO_URL}"
+        perform_install
+    fi
+
+    echo "Starting runner service..."
+    cd "$RUNNER_DIR"
+    sudo ./svc.sh start
+}
+
+main

release/release_files.py

@@ -6,41 +6,11 @@ from pathlib import Path
 HERE = os.path.abspath(os.path.dirname(__file__))
 ROOT = HERE + "/.."
 
-# blacklisting is for two purposes:
-# - minimizing release download size
-# - keeping the diff readable
 blacklist = [
-  "panda/drivers/",
-  "panda/examples/",
-  "panda/tests/safety/",
-
-  "opendbc_repo/dbc/.*.dbc$",
-  "opendbc_repo/dbc/generator/",
-
-  "cereal/.*test.*",
-  "^common/tests/",
-
-  # particularly large text files
-  "uv.lock",
-  "third_party/catch2",
-  "selfdrive/car/tests/test_models.*",
-
-  "^tools/",
-  "^tinygrad_repo/",
+  ".git/",
 
   "matlab.*.md",
 
-  ".git/",
-  ".github/",
-  ".devcontainer/",
-  "Darwin/",
-  ".vscode",
-
-  # common things
-  "LICENSE",
-  "Dockerfile",
-  ".pre-commit",
-
   # no LFS or submodules in release
   ".lfsconfig",
   ".gitattributes",
@@ -48,153 +18,10 @@ blacklist = [
   ".gitmodules",
 ]
 
-# Sunnypilot blacklist
-sunnypilot_blacklist = [
-  "system/loggerd/sunnylink_uploader.py",  # Temporarily, until we are ready to roll it out widely
-  ".idea/",
-  ".run/",
-  ".*__pycache__/.*",
-  ".*\\.pyc",
-  "teleoprtc/*",
-  "third_party/snpe/x86_64/*",
-  "body/board/canloader.py",
-  "body/board/flash_base.sh",
-  "body/board/flash_knee.sh",
-  "body/board/recover.sh",
-  ".*/test/",
-  ".*/tests/",
-  ".*tinygrad_repo/tinygrad/renderer/",
-  "README.md",
-  ".*internal/",
-  "docs/.*",
-  ".sconsign.dblite",
-  "release/ci/scons_cache/",
-  ".gitlab-ci.yml",
-  ".clang-tidy",
-  ".dockerignore",
-  ".editorconfig",
-  ".python-version",
-  "SECURITY.md",
-  "codecov.yml",
-  "conftest.py",
-  "poetry.lock",
-  ".venv/",
-]
-
-# Merge the blacklists
-blacklist += sunnypilot_blacklist
-
 # gets you through the blacklist
-whitelist = [
-  "tools/lib/",
-  "tools/bodyteleop/",
-  "tools/joystick/",
-  "tools/longitudinal_maneuvers/",
-
-  "tinygrad_repo/examples/openpilot/compile3.py",
-  "tinygrad_repo/extra/onnx.py",
-  "tinygrad_repo/extra/onnx_ops.py",
-  "tinygrad_repo/extra/thneed.py",
-  "tinygrad_repo/extra/utils.py",
-  "tinygrad_repo/tinygrad/codegen/kernel.py",
-  "tinygrad_repo/tinygrad/codegen/linearizer.py",
-  "tinygrad_repo/tinygrad/features/image.py",
-  "tinygrad_repo/tinygrad/features/search.py",
-  "tinygrad_repo/tinygrad/nn/*",
-  "tinygrad_repo/tinygrad/renderer/cstyle.py",
-  "tinygrad_repo/tinygrad/renderer/opencl.py",
-  "tinygrad_repo/tinygrad/runtime/lib.py",
-  "tinygrad_repo/tinygrad/runtime/ops_cpu.py",
-  "tinygrad_repo/tinygrad/runtime/ops_disk.py",
-  "tinygrad_repo/tinygrad/runtime/ops_gpu.py",
-  "tinygrad_repo/tinygrad/shape/*",
-  "tinygrad_repo/tinygrad/.*.py",
-
-  # TODO: do this automatically
-  "opendbc_repo/dbc/comma_body.dbc",
-  "opendbc_repo/dbc/chrysler_ram_hd_generated.dbc",
-  "opendbc_repo/dbc/chrysler_ram_dt_generated.dbc",
-  "opendbc_repo/dbc/chrysler_pacifica_2017_hybrid_generated.dbc",
-  "opendbc_repo/dbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc",
-  "opendbc_repo/dbc/gm_global_a_powertrain_generated.dbc",
-  "opendbc_repo/dbc/gm_global_a_object.dbc",
-  "opendbc_repo/dbc/gm_global_a_chassis.dbc",
-  "opendbc_repo/dbc/FORD_CADS.dbc",
-  "opendbc_repo/dbc/ford_fusion_2018_adas.dbc",
-  "opendbc_repo/dbc/ford_lincoln_base_pt.dbc",
-  "opendbc_repo/dbc/honda_accord_2018_can_generated.dbc",
-  "opendbc_repo/dbc/acura_ilx_2016_can_generated.dbc",
-  "opendbc_repo/dbc/acura_rdx_2018_can_generated.dbc",
-  "opendbc_repo/dbc/acura_rdx_2020_can_generated.dbc",
-  "opendbc_repo/dbc/honda_civic_touring_2016_can_generated.dbc",
-  "opendbc_repo/dbc/honda_civic_hatchback_ex_2017_can_generated.dbc",
-  "opendbc_repo/dbc/honda_crv_touring_2016_can_generated.dbc",
-  "opendbc_repo/dbc/honda_crv_ex_2017_can_generated.dbc",
-  "opendbc_repo/dbc/honda_crv_ex_2017_body_generated.dbc",
-  "opendbc_repo/dbc/honda_crv_executive_2016_can_generated.dbc",
-  "opendbc_repo/dbc/honda_fit_ex_2018_can_generated.dbc",
-  "opendbc_repo/dbc/honda_odyssey_exl_2018_generated.dbc",
-  "opendbc_repo/dbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc",
-  "opendbc_repo/dbc/honda_insight_ex_2019_can_generated.dbc",
-  "opendbc_repo/dbc/acura_ilx_2016_nidec.dbc",
-  "opendbc_repo/dbc/honda_civic_ex_2022_can_generated.dbc",
-  "opendbc_repo/dbc/hyundai_canfd.dbc",
-  "opendbc_repo/dbc/hyundai_kia_generic.dbc",
-  "opendbc_repo/dbc/hyundai_kia_mando_front_radar_generated.dbc",
-  "opendbc_repo/dbc/mazda_2017.dbc",
-  "opendbc_repo/dbc/nissan_x_trail_2017_generated.dbc",
-  "opendbc_repo/dbc/nissan_leaf_2018_generated.dbc",
-  "opendbc_repo/dbc/subaru_global_2017_generated.dbc",
-  "opendbc_repo/dbc/subaru_global_2020_hybrid_generated.dbc",
-  "opendbc_repo/dbc/subaru_outback_2015_generated.dbc",
-  "opendbc_repo/dbc/subaru_outback_2019_generated.dbc",
-  "opendbc_repo/dbc/subaru_forester_2017_generated.dbc",
-  "opendbc_repo/dbc/toyota_tnga_k_pt_generated.dbc",
-  "opendbc_repo/dbc/toyota_new_mc_pt_generated.dbc",
-  "opendbc_repo/dbc/toyota_nodsu_pt_generated.dbc",
-  "opendbc_repo/dbc/toyota_adas.dbc",
-  "opendbc_repo/dbc/toyota_tss2_adas.dbc",
-  "opendbc_repo/dbc/vw_golf_mk4.dbc",
-  "opendbc_repo/dbc/vw_mqb_2010.dbc",
-  "opendbc_repo/dbc/tesla_can.dbc",
-  "opendbc_repo/dbc/tesla_radar_bosch_generated.dbc",
-  "opendbc_repo/dbc/tesla_radar_continental_generated.dbc",
-  "opendbc_repo/dbc/tesla_powertrain.dbc",
+whitelist: list[str] = [
 ]
 
-# Sunnypilot whitelist
-sunnypilot_whitelist = [
-  "^README.md",
-  ".*selfdrive/test/fuzzy_generation.py",
-  ".*selfdrive/test/helpers.py",
-  ".*selfdrive/test/__init__.py",
-  ".*selfdrive/test/setup_device_ci.sh",
-  ".*selfdrive/test/test_time_to_onroad.py",
-  ".*selfdrive/test/test_onroad.py",
-  ".*system/manager/test/test_manager.py",
-  ".*system/manager/test/__init__.py",
-  ".*system/qcomgpsd/tests/test_qcomgpsd.py",
-  ".*system/updated/casync/tests/test_casync.py",
-  ".*system/updated/tests/test_git.py",
-  ".*system/updated/tests/test_base.py",
-  ".*selfdrive/ui/tests/test_translations.py",
-  ".*selfdrive/car/tests/__init__.py",
-  ".*selfdrive/car/tests/test_car_interfaces.py",
-  ".*selfdrive/navd/tests/test_navd.py",
-  ".*selfdrive/navd/tests/test_map_renderer.py",
-  ".*selfdrive/boardd/tests/test_boardd_loopback.py",
-  ".*INTEGRATION.md",
-  ".*HOW-TOS.md",
-  ".*CARS.md",
-  ".*LIMITATIONS.md",
-  ".*CONTRIBUTING.md",
-  ".*sunnyhaibin0850_qrcode_paypal.me.png",
-  "opendbc/.*.dbc",
-]
-
-# Merge the whitelists
-whitelist += sunnypilot_whitelist
-
 
 if __name__ == "__main__":
   for f in Path(ROOT).rglob("**/*"):

scripts/build_small.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd $DIR
-
-# git clone --mirror
-SRC=/tmp/openpilot.git/
-OUT=/tmp/smallpilot/
-
-echo "starting size $(du -hs .git/)"
-
-rm -rf $OUT
-
-cd $SRC
-git remote update
-
-# copy contents
-#rsync -a --exclude='.git/' $DIR $OUT
-
-cp -r $SRC $OUT
-
-cd $OUT
-
-# remove all tags
-git tag -l | xargs git tag -d
-
-# remove non-master branches
-BRANCHES="release2 release3 devel master-ci nightly"
-for branch in $BRANCHES; do
-  git branch -D $branch
-  git branch -D ${branch}-staging || true
-done
-
-#git gc
-git reflog expire --expire=now --all
-git gc --prune=now
-git gc --aggressive --prune=now
-echo "new one is $(du -hs .)"

scripts/code_stats.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-import os
-import ast
-import stat
-import subprocess
-
-fouts = {x.decode('utf-8') for x in subprocess.check_output(['git', 'ls-files']).strip().split()}
-
-pyf = []
-for d in ["cereal", "common", "scripts", "selfdrive", "tools"]:
-  for root, _, files in os.walk(d):
-    for f in files:
-      if f.endswith(".py"):
-        pyf.append(os.path.join(root, f))
-
-imps: set[str] = set()
-
-class Analyzer(ast.NodeVisitor):
-  def visit_Import(self, node):
-    for alias in node.names:
-      imps.add(alias.name)
-    self.generic_visit(node)
-
-  def visit_ImportFrom(self, node):
-    imps.add(node.module)
-    self.generic_visit(node)
-
-tlns = 0
-carlns = 0
-scriptlns = 0
-testlns = 0
-for f in sorted(pyf):
-  if f not in fouts:
-    continue
-  xbit = bool(os.stat(f)[stat.ST_MODE] & stat.S_IXUSR)
-  src = open(f).read()
-  lns = len(src.split("\n"))
-  tree = ast.parse(src)
-  Analyzer().visit(tree)
-  print(f"{lns:5d} {f} {xbit}")
-  if 'test' in f:
-    testlns += lns
-  elif f.startswith(('tools/', 'scripts/', 'selfdrive/debug')):
-    scriptlns += lns
-  elif f.startswith('selfdrive/car'):
-    carlns += lns
-  else:
-    tlns += lns
-
-print(f"{tlns} lines of openpilot python")
-print(f"{carlns} lines of car ports")
-print(f"{scriptlns} lines of tools/scripts/debug")
-print(f"{testlns} lines of tests")
-#print(sorted(list(imps)))

scripts/count_cars.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-from collections import Counter
-from pprint import pprint
-
-from opendbc.car.docs import get_all_car_docs
-
-if __name__ == "__main__":
-  cars = get_all_car_docs()
-  make_count = Counter(l.make for l in cars)
-  print("\n", "*" * 20, len(cars), "total", "*" * 20, "\n")
-  pprint(make_count)

scripts/git_rewrite/rewrite-git-history.sh

@@ -1,391 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-SRC=/tmp/openpilot/
-SRC_CLONE=/tmp/openpilot-clone/
-OUT=/tmp/openpilot-tiny/
-
-REWRITE_IGNORE_BRANCHES=(
-  dashcam3
-  devel
-  master-ci
-  nightly
-  release2
-  release3
-  release3-staging
-)
-
-VALIDATE_IGNORE_FILES=(
-  ".github/ISSUE_TEMPLATE/bug_report.md"
-  ".github/pull_request_template.md"
-)
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd $DIR
-
-LOGS_DIR=$DIR/git-rewrite-$(date +"%Y-%m-%dT%H:%M:%S%z")
-mkdir -p $LOGS_DIR
-
-GIT_REWRITE_LOG=$LOGS_DIR/git-rewrite-log.txt
-BRANCH_DIFF_LOG=$LOGS_DIR/branch-diff-log.txt
-COMMIT_DIFF_LOG=$LOGS_DIR/commit-diff-log.txt
-
-START_TIME=$(date +%s)
-exec > >(while IFS= read -r line; do
-  CURRENT_TIME=$(date +%s)
-  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
-  echo "[${ELAPSED_TIME}s] $line"
-done | tee -a "$GIT_REWRITE_LOG") 2>&1
-
-# INSTALL git-filter-repo
-if [ ! -f /tmp/git-filter-repo ]; then
-  echo "Installing git-filter-repo..."
-  curl -sSo /tmp/git-filter-repo https://raw.githubusercontent.com/newren/git-filter-repo/main/git-filter-repo
-  chmod +x /tmp/git-filter-repo
-fi
-
-# MIRROR openpilot
-if [ ! -d $SRC ]; then
-  echo "Mirroring openpilot..."
-  git clone --mirror https://github.com/commaai/openpilot.git $SRC # 4.18 GiB (488034 objects)
-
-  cd $SRC
-
-  echo "Starting size $(du -sh .)"
-
-  git remote update
-
-  # the git-filter-repo analysis is bliss - can be found in the repo root/filter-repo/analysis
-  echo "Analyzing with git-filter-repo..."
-  /tmp/git-filter-repo --force --analyze
-
-  echo "Pushing to openpilot-archive..."
-  # push to archive repo - in smaller parts because the 2 GB push limit - https://docs.github.com/en/get-started/using-git/troubleshooting-the-2-gb-push-limit
-  ARCHIVE_REPO=git@github.com:commaai/openpilot-archive.git
-  git push --prune $ARCHIVE_REPO +refs/heads/master:refs/heads/master # push master first so it's the default branch (when openpilot-archive is an empty repo)
-  git push --prune $ARCHIVE_REPO +refs/heads/*:refs/heads/* # 956.39 MiB (110725 objects)
-  git push --prune $ARCHIVE_REPO +refs/tags/*:refs/tags/* # 1.75 GiB (21694 objects)
-  # git push --mirror $ARCHIVE_REPO || true # fails to push refs/pull/* (deny updating a hidden ref) for pull requests
-  # we fail and continue - more reading: https://stackoverflow.com/a/34266401/639708 and https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
-fi
-
-# REWRITE master and tags
-if [ ! -d $SRC_CLONE ]; then
-  echo "Cloning $SRC..."
-  GIT_LFS_SKIP_SMUDGE=1 git clone $SRC $SRC_CLONE
-
-  cd $SRC_CLONE
-
-  echo "Checking out old history..."
-
-  git checkout tags/v0.7.1 > /dev/null 2>&1
-  # checkout as main, since we need master ref later
-  git checkout -b main
-
-  echo "Creating setup commits..."
-
-  # rm these so we don't get conflicts later
-  git rm -r cereal opendbc panda selfdrive/ui/ui > /dev/null
-  git commit -m "removed conflicting files" > /dev/null
-
-  # skip-smudge to get rid of some lfs errors that it can't find the reference of some lfs files
-  # we don't care about fetching/pushing lfs right now
-  git lfs install --skip-smudge --local
-
-  # squash initial setup commits
-  git cherry-pick -n -X theirs 6c33a5c..59b3d06 > /dev/null
-  git commit -m "switching to master" > /dev/null
-
-  # squash the two commits
-  git reset --soft HEAD~2
-  git commit -m "switching to master" -m "$(git log --reverse --format=%B 6c33a5c..59b3d06)" -m "removed conflicting files" > /dev/null
-
-  # get commits we want to cherry-pick
-  # will start with the next commit after #59b3d06 tools is local now
-  COMMITS=$(git rev-list --reverse 59b3d06..master)
-
-  # we need this for logging
-  TOTAL_COMMITS=$(echo $COMMITS | wc -w | xargs)
-  CURRENT_COMMIT_NUMBER=0
-
-  # empty this file
-  > commit-map.txt
-
-  echo "Rewriting master commits..."
-
-  for COMMIT in $COMMITS; do
-      CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
-      # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"\\r
-      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"
-
-      # set environment variables to preserve author/committer and dates
-      export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
-      export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
-      export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
-      export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
-      export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
-      export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
-
-      # cherry-pick the commit
-      if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
-        # check if the failure is because of an empty commit
-        if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
-          echo "Empty commit detected. Skipping commit $COMMIT"
-          git cherry-pick --skip
-          # log it was empty to the mapping file
-          echo "$COMMIT EMPTY" >> commit-map.txt
-        else
-          # handle other errors or conflicts
-          echo "Cherry-pick failed. Handling error..."
-          echo "$GIT_OUTPUT"
-          exit 1
-        fi
-      else
-        # capture the new commit hash
-        NEW_COMMIT=$(git rev-parse HEAD)
-
-        # save the old and new commit hashes to the mapping file
-        echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
-
-        # append the old commit ID to the commit message
-        git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
-      fi
-
-    # prune every 3000 commits to avoid gc errors
-    if [ $((CURRENT_COMMIT_NUMBER % 3000)) -eq 0 ]; then
-      echo "Pruning repo..."
-      git gc
-    fi
-  done
-
-  echo "Rewriting tags..."
-
-  # remove all old tags
-  git tag -l | xargs git tag -d
-
-  # read each line from the tag-commit-map.txt
-  while IFS=' ' read -r TAG OLD_COMMIT; do
-    # search for the new commit in commit-map.txt corresponding to the old commit
-    NEW_COMMIT=$(grep "^$OLD_COMMIT " "commit-map.txt" | awk '{print $2}')
-
-    # check if this is a rebased commit
-    if [ -z "$NEW_COMMIT" ]; then
-      # if not, then just use old commit hash
-      NEW_COMMIT=$OLD_COMMIT
-    fi
-
-    echo "Rewriting tag $TAG from commit $NEW_COMMIT"
-    git tag -f "$TAG" "$NEW_COMMIT"
-  done < "$DIR/tag-commit-map.txt"
-
-  # uninstall lfs since we don't want to touch (push to) lfs right now
-  # git push will also push lfs, if we don't uninstall (--local so just for this repo)
-  git lfs uninstall --local
-
-  # force push new master
-  git push --force origin main:master
-
-  # force push new tags
-  git push --force --tags
-fi
-
-# REWRITE branches based on master
-if [ ! -f "$SRC_CLONE/rewrite-branches-done" ]; then
-  cd $SRC_CLONE
-  > rewrite-branches-done
-
-  # empty file
-  > $BRANCH_DIFF_LOG
-
-  echo "Rewriting branches based on master..."
-
-  # will store raw diffs here, if exist
-  mkdir -p differences
-
-  # get a list of all branches except master and REWRITE_IGNORE_BRANCHES
-  BRANCHES=$(git branch -r | grep -v ' -> ' | sed 's/.*origin\///' | grep -v '^master$' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n'))
-
-  for BRANCH in $BRANCHES; do
-    # check if the branch is based on master history
-    MERGE_BASE=$(git merge-base master origin/$BRANCH) || true
-    if [ -n "$MERGE_BASE" ]; then
-      echo "Rewriting branch: $BRANCH"
-
-      # create a new branch based on the new master
-      NEW_MERGE_BASE=$(grep "^$MERGE_BASE " "commit-map.txt" | awk '{print $2}')
-      if [ -z "$NEW_MERGE_BASE" ]; then
-        echo "Error: could not find new merge base for branch $BRANCH" >> $BRANCH_DIFF_LOG
-        continue
-      fi
-      git checkout -b ${BRANCH}_new $NEW_MERGE_BASE
-
-      # get the range of commits unique to this branch
-      COMMITS=$(git rev-list --reverse $MERGE_BASE..origin/${BRANCH})
-
-      HAS_ERROR=0
-
-      # simple delimiter
-      echo "BRANCH ${BRANCH}" >> commit-map.txt
-
-      for COMMIT in $COMMITS; do
-        # set environment variables to preserve author/committer and dates
-        export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
-        export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
-        export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
-        export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
-        export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
-        export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
-
-        # cherry-pick the commit
-        if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
-          # check if the failure is because of an empty commit
-          if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
-            echo "Empty commit detected. Skipping commit $COMMIT"
-            git cherry-pick --skip
-            # log it was empty to the mapping file
-            echo "$COMMIT EMPTY" >> commit-map.txt
-          else
-            # handle other errors or conflicts
-            echo "Cherry-pick of ${BRANCH} branch failed. Removing branch upstream..." >> $BRANCH_DIFF_LOG
-            echo "$GIT_OUTPUT" > "$LOGS_DIR/branch-${BRANCH}"
-            git cherry-pick --abort
-            git push --delete origin ${BRANCH}
-            HAS_ERROR=1
-            break
-          fi
-        else
-          # capture the new commit hash
-          NEW_COMMIT=$(git rev-parse HEAD)
-
-          # save the old and new commit hashes to the mapping file
-          echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
-
-          # append the old commit ID to the commit message
-          git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
-        fi
-      done
-
-      # force push the new branch
-      if [ $HAS_ERROR -eq 0 ]; then
-        # git lfs goes haywire here, so we need to install and uninstall
-        # git lfs install --skip-smudge --local
-        git lfs uninstall --local > /dev/null
-        git push -f origin ${BRANCH}_new:${BRANCH}
-      fi
-
-      # clean up local branch
-      git checkout master > /dev/null
-      git branch -D ${BRANCH}_new > /dev/null
-    else
-      echo "Deleting branch $BRANCH as it's not based on master history" >> $BRANCH_DIFF_LOG
-      git push --delete origin ${BRANCH}
-    fi
-  done
-fi
-
-# VALIDATE cherry-pick
-if [ ! -f "$SRC_CLONE/validation-done" ]; then
-  cd $SRC_CLONE
-  > validation-done
-
-  TOTAL_COMMITS=$(grep -cve '^\s*$' commit-map.txt)
-  CURRENT_COMMIT_NUMBER=0
-  COUNT_SAME=0
-  COUNT_DIFF=0
-
-  # empty file
-  > $COMMIT_DIFF_LOG
-
-  echo "Validating commits..."
-
-  # will store raw diffs here, if exist
-  mkdir -p differences
-
-  # read each line from commit-map.txt
-  while IFS=' ' read -r OLD_COMMIT NEW_COMMIT; do
-    if [ "$NEW_COMMIT" == "EMPTY" ]; then
-      continue
-    fi
-    if [ "$OLD_COMMIT" == "BRANCH" ]; then
-      echo "Branch ${NEW_COMMIT} below:" >> $COMMIT_DIFF_LOG
-      continue
-    fi
-    CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
-    # retrieve short hashes and dates for the old and new commits
-    OLD_COMMIT_SHORT=$(git rev-parse --short $OLD_COMMIT)
-    NEW_COMMIT_SHORT=$(git rev-parse --short $NEW_COMMIT)
-    OLD_DATE=$(git show -s --format='%cd' $OLD_COMMIT)
-    NEW_DATE=$(git show -s --format='%cd' $NEW_COMMIT)
-
-    # echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"\\r
-    echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"
-
-    # generate lists of files and their hashes for the old and new commits, excluding ignored files
-    OLD_FILES=$(git ls-tree -r $OLD_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
-    NEW_FILES=$(git ls-tree -r $NEW_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
-
-    # Compare the diffs
-    if diff <(echo "$OLD_FILES") <(echo "$NEW_FILES") > /dev/null; then
-      # echo "Old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT are equivalent."
-      COUNT_SAME=$((COUNT_SAME + 1))
-    else
-      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Difference found between old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT" >> $COMMIT_DIFF_LOG
-      COUNT_DIFF=$((COUNT_DIFF + 1))
-      set +e
-      diff -u <(echo "$OLD_FILES") <(echo "$NEW_FILES") > "$LOGS_DIR/commit-$CURRENT_COMMIT_NUMBER-$OLD_COMMIT_SHORT-$NEW_COMMIT_SHORT"
-      set -e
-    fi
-  done < "commit-map.txt"
-
-  echo "Summary:" >> $COMMIT_DIFF_LOG
-  echo "Equivalent commits: $COUNT_SAME" >> $COMMIT_DIFF_LOG
-  echo "Different commits: $COUNT_DIFF" >> $COMMIT_DIFF_LOG
-fi
-
-if [ ! -d $OUT ]; then
-  cp -r $SRC $OUT
-
-  cd $OUT
-
-  # remove all non-master branches
-  # git branch | grep -v "^  master$" | grep -v "\*" | xargs git branch -D
-
-  # echo "cleaning up refs"
-  # delete pull request refs since we can't alter them anyway (https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#error-failed-to-push-some-refs)
-  # git for-each-ref --format='%(refname)' | grep '^refs/pull/' | xargs -I {} git update-ref -d {}
-
-  echo "importing new lfs files"
-  # import "almost" everything to lfs
-  BRANCHES=$(git for-each-ref --format='%(refname)' refs/heads/ | sed 's%refs/heads/%%g' | grep -v -f <(echo "${REWRITE_IGNORE_BRANCHES[*]}" | tr ' ' '\n') | tr '\n' ' ')
-  git lfs migrate import --include="*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,selfdrive/car/tests/test_models_segs.txt,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,flask/**/*,panda/**/*,board/**/*,messaging/**/*,opendbc/**/*,tools/cabana/chartswidget.cc,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,selfdrive/ui/paint.cc,werkzeug/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,selfdrive/locationd/laikad.py,selfdrive/locationd/test/test_laikad.py,tools/gpstest/test_laikad.py,selfdrive/locationd/laikad_helpers.py,tools/nui/**/*,jsonrpc/**/*,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,selfdrive/camerad/cameras/camera_qcom.cc,selfdrive/manager.py,selfdrive/modeld/models/driving.cc,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,gunicorn/**/*,*.qm,jinja2/**/*,click/**/*,dbcs/**/*,websocket/**/*" $BRANCHES
-
-  echo "reflog and gc"
-  # this is needed after lfs import
-  git reflog expire --expire=now --all
-  git gc --prune=now --aggressive
-
-  # check the git-filter-repo analysis again - can be found in the repo root/filter-repo/analysis
-  echo "Analyzing with git-filter-repo..."
-  /tmp/git-filter-repo --force --analyze
-
-  echo "New size is $(du -sh .)"
-fi
-
-cd $OUT
-
-# fetch all lfs files from https://github.com/commaai/openpilot.git
-# some lfs files are missing on gitlab, but they can be found on github
-git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
-git config lfs.pushurl ssh://git@github.com/commaai/openpilot.git
-git lfs fetch --all || true
-
-# also fetch all lfs files from https://gitlab.com/commaai/openpilot-lfs.git
-git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
-git lfs fetch --all || true
-
-# final push - will also push lfs
-# TODO: switch to git@github.com:commaai/openpilot.git when ready
-# git push --mirror git@github.com:commaai/openpilot-tiny.git
-# using this instead to ignore refs/pull/* - since this is also what --mirror does - https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
-git push --prune git@github.com:commaai/openpilot-tiny.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/rewrite.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd $DIR
-
-git clone --bare https://github.com/commaai/openpilot
-cp -r openpilot.git openpilot_backup
-cd openpilot.git
-
-# backup old repo
-git push git@github.com:commaai/openpilot-archive.git +refs/heads/master:refs/heads/master
-git push git@github.com:commaai/openpilot-archive.git +refs/heads/*:refs/heads/*
-git push git@github.com:commaai/openpilot-archive.git +refs/tags/*:refs/tags/*
-git push --mirror git@github.com:commaai/openpilot-archive.git
-
-# ignore all release branches
-git for-each-ref --format='delete %(refname)' | grep 'dashcam3\|devel\|master-ci\|nightly\|release2\|release3\|release3-staging' | git update-ref --stdin
-
-# re-tag old releases on master
-declare -A TAGS=( ["f8cb04e4a8b032b72a909f68b808a50936184bee"]="v0.9.7" ["0b4d08fab8e35a264bc7383e878538f8083c33e5"]="v0.9.6" ["3b1e9017c560499786d8a0e46aaaeea65037acac"]="v0.9.5" ["fa310d9e2542cf497d92f007baec8fd751ffa99c"]="v0.9.4" ["8704c1ff952b5c85a44f50143bbd1a4f7b4887e2"]="v0.9.3" ["c7d3b28b93faa6c955fb24bc64031512ee985ee9"]="v0.9.2" ["89f68bf0cbf53a81b0553d3816fdbe522f941fa1"]="v0.9.1" ["58b84fb401a804967aa0dd5ee66fafa90194fd30"]="v0.9.0" ["f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e"]="v0.8.16" ["5a7c2f90361e72e9c35e88abd2e11acdc4aba354"]="v0.8.15" ["71901c94dbbaa2f9f156a80c14cc7ea65219fc7c"]="v0.8.14" ["95da47079510afc91665263619e5939126da637c"]="v0.8.13" ["472177e2a8a1d002e56f9096326fd2dff62e54f9"]="v0.8.12" ["08078acbd0b4f7da469c7dff6159000e358974a9"]="v0.8.11" ["687925c775c375495f9827946138a724bde00b9d"]="v0.8.10" ["204e5a090735a059d69c29145a4cee49450da07e"]="v0.8.9" ["4be956f8861ecbb521ef9503a3c87b07c9d36721"]="v0.8.8" ["589f82c76627d634761a31a34b2488403556eb0b"]="v0.8.7" ["507cfc8910f74ddb8810039d68b880b426ff9ff9"]="v0.8.6" ["d47b00b45a866bef088f51d1ff31de5885ab04e9"]="v0.8.5" ["553e7d1cce314e7eb0587186b1764c3ff43bed62"]="v0.8.4" ["9896438d1511602a1ff87f7c4eb3c7172b30104a"]="v0.8.3" ["280192ed1443f112463417c2d815ea8ee2762fbd"]="v0.8.2" ["8039361567e4659eae2a084e6f39f34acadf4cac"]="v0.8.1" ["d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc"]="v0.8" ["3d456e5d0fbf0c9887d0499dee812f2b029edf6d"]="v0.7.10" ["81763a18b5d0e379b749e090ecce36a91fca7c43"]="v0.7.9" ["9bc0b350fd273bbb2deb3dcaef0312944e4f6cfd"]="v0.7.8" ["ede5b632b58c55e4ff003f948efae07fe03c2280"]="v0.7.7" ["775acd11ba2e0a8c2f5a5655338718d796491b36"]="v0.7.6.1" ["302417b4cf0dcf00d45e4995b5410e543ad121d1"]="v0.7.5" ["12ff088b42221dd17d9d97decb1fc61a7cb0a861"]="v0.7.4" ["9563f7730252451fdcba9bc3d9fe36dab9c86a26"]="v0.7.3" ["8321cf283abbc2ca3fda7e0c7a069a77a492fe0c"]="v0.7.2" ["1e1de64a1e59476b7b3d3558b92149246d5c3292"]="v0.7.1" ["a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3"]="v0.7" ["d4eb5a6eafdd4803d09e6f3963918216cca5a81f"]="v0.6.6" ["70d17cd69b80e7627dcad8fd5b6438f2309ac307"]="v0.6.5" ["58f376002e0c654fbc2de127765fa297cf694a33"]="v0.6.4" ["d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3"]="v0.6.3" ["095ef5f9f60fca1b269aabcc3cfd322b17b9e674"]="v0.6.2" ["cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22"]="v0.6.1" ["60a20537c5f3fcc7f11946d81aebc8f90c08c117"]="v0.6" ["dd34ccfe288ebda8e2568cf550994ae890379f45"]="v0.5.13" ["3f9059fea886f1fa3b0c19a62a981d891dcc84eb"]="v0.5.12" ["2f92d577f995ff6ae1945ef6b89df3cb69b92999"]="v0.5.11" ["5a9d89ed42ddcd209d001a10d7eb828ef0e6d9de"]="v0.5.10" ["0207a970400ee28d3e366f2e8f5c551281accf02"]="v0.5.9" ["b967da5fc1f7a07e3561db072dd714d325e857b0"]="v0.5.8" ["210db686bb89f8696aa040e6e16de65424b808c9"]="v0.5.7" ["860a48765d1016ba226fb2c64aea35a45fe40e4a"]="v0.5.6" ["8f3539a27b28851153454eb737da9624cccaed2d"]="v0.5.5" ["a422246dc30bce11e970514f13f7c110f4470cc3"]="v0.5.4" ["285c52eb693265a0a530543e9ca0aeb593a2a55e"]="v0.5.3" ["0129a8a4ff8da5314e8e4d4d3336e89667ff6d54"]="v0.5.2" ["6f3d10a4c475c4c4509f0b370805419acd13912d"]="v0.5.1" ["de33bc46452b1046387ee2b3a03191b2c71135fb"]="v0.5" ["ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9"]="v0.4.7" ["c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1"]="v0.4.6" ["37285038d3f91fa1b49159c4a35a8383168e644f"]="v0.4.5" ["9a9ff839a9b70cb2601d7696af743f5652395389"]="v0.4.4" ["28c0797d30175043bbfa31307b63aab4197cf996"]="v0.4.2" ["4474b9b3718653aeb0aee26422caefb90460cc0e"]="v0.4.1" ["da52d065a4c4f52d6017a537f3a80326f5af8bdc"]="v0.4.0.2" ["9d3963559ae7b15193057937ff3e72481899f40d"]="v0.3.5"  ["1b8c44b5067525a5d266b6e99799d8097da76a29"]="v0.3.4" ["5cf91d0496688fed4f2a6c7021349b1fc0e057a2"]="v0.3.3" ["7fe46f1e1df5dec08a940451ba0feefd5c039165"]="v0.3.2" ["41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24"]="v0.3.1" ["c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8"]="v0.3.0" ["693bcb0f83478f2651db6bac9be5ca5ad60d03f3"]="v0.2.9" ["95a349abcc050712c50d4d85a1c8a804eee7f6c2"]="v0.2.8" ["c6ba5dc5391d3ca6cda479bf1923b88ce45509a0"]="v0.2.7" ["6c3afeec0fb439070b2912978b8dbb659033b1d9"]="v0.2.6" ["29c58b45882ac79595356caf98580c1d2a626011"]="v0.2.5" ["ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0"]="v0.2.4" ["adaa4ed350acda4067fc0b455ad15b54cdf4c768"]="v0.2.3" ["a64b9aa9b8cb5863c917b6926516291a63c02fe5"]="v0.2.2" ["17d9becd3c673091b22f09aa02559a9ed9230f50"]="v0.2.1" ["449b482cc3236ccf31829830b4f6a44b2dcc06c2"]="v0.2" ["e94a30bec07e719c5a7b037ca1f4db8312702cce"]="v0.1" )
-for tag in "${!TAGS[@]}"; do git tag -f "${TAGS[$tag]}" "$tag" ; done
-
-# get master root commit
-ROOT_COMMIT=$(git rev-list --max-parents=0 HEAD | tail -n 1)
-
-# link master and devel
-git replace --graft $ROOT_COMMIT v0.7.1
-git-filter-repo --prune-empty never --force --commit-callback 'h=commit.original_id.decode("utf-8");m=commit.message.decode("utf-8");commit.message=str.encode(m + "\n" + "old-commit-hash: " + h)'
-
-# delete replace refs
-git for-each-ref --format='delete %(refname)' refs/replace | git update-ref --stdin
-
-# machine validation
-tail -n +2 "filter-repo/commit-map" | tr ' ' '\n' | xargs -P $(nproc) -n 2 bash -c 'H1=$(cd ../openpilot_backup && git ls-tree -r $0 | sha1sum) && H2=$(git ls-tree -r $1 | sha1sum) && echo "$H1 $H2" >> /tmp/GIT_HASHES && diff <(echo $H1) <(echo $H2) || exit 255'
-# human validation
-less /tmp/GIT_HASH
-
-# cleanup
-git reflog expire --expire=now --all
-git gc --prune=now --aggressive
-
-# get all lfs files
-set +e
-git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
-git lfs fetch --all
-git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-git lfs fetch --all
-set -e
-
-# add new files to lfs
-git lfs migrate import --everything --include="*.ico,*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,pyextra/**/*,panda/board/**/inc/*.h,panda/board/obj/*.elf,board/inc/*.h,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,*.pro,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,poetry.lock,*.qm"
-
-# set new lfs endpoint
-git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
-
-# push all branch+tag (scary stuff...)
-git push -f --set-upstream git@github.com:commaai/openpilot.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*

scripts/git_rewrite/tag-commit-map.txt

@@ -1,82 +0,0 @@
-v0.1 e94a30bec07e719c5a7b037ca1f4db8312702cce
-v0.2 449b482cc3236ccf31829830b4f6a44b2dcc06c2
-v0.2.1 17d9becd3c673091b22f09aa02559a9ed9230f50
-v0.2.2 a64b9aa9b8cb5863c917b6926516291a63c02fe5
-v0.2.3 adaa4ed350acda4067fc0b455ad15b54cdf4c768
-v0.2.4 ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0
-v0.2.5 29c58b45882ac79595356caf98580c1d2a626011
-v0.2.6 6c3afeec0fb439070b2912978b8dbb659033b1d9
-v0.2.7 c6ba5dc5391d3ca6cda479bf1923b88ce45509a0
-v0.2.8 95a349abcc050712c50d4d85a1c8a804eee7f6c2
-v0.2.9 693bcb0f83478f2651db6bac9be5ca5ad60d03f3
-v0.3.0 c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8
-v0.3.1 41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24
-v0.3.2 7fe46f1e1df5dec08a940451ba0feefd5c039165
-v0.3.3 5cf91d0496688fed4f2a6c7021349b1fc0e057a2
-v0.3.4 1b8c44b5067525a5d266b6e99799d8097da76a29
-v0.3.5 b111277f464cf66fa34b67819a83ea683e0f64df
-v0.4.0.2 da52d065a4c4f52d6017a537f3a80326f5af8bdc
-v0.4.1 4474b9b3718653aeb0aee26422caefb90460cc0e
-v0.4.2 28c0797d30175043bbfa31307b63aab4197cf996
-v0.4.4 9a9ff839a9b70cb2601d7696af743f5652395389
-v0.4.5 37285038d3f91fa1b49159c4a35a8383168e644f
-v0.4.6 c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1
-v0.4.7 ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9
-v0.5 de33bc46452b1046387ee2b3a03191b2c71135fb
-v0.5.1 8f22f52235c48eada586795ac57edb22688e4d08
-v0.5.2 0129a8a4ff8da5314e8e4d4d3336e89667ff6d54
-v0.5.3 285c52eb693265a0a530543e9ca0aeb593a2a55e
-v0.5.4 a422246dc30bce11e970514f13f7c110f4470cc3
-v0.5.5 8f3539a27b28851153454eb737da9624cccaed2d
-v0.5.6 860a48765d1016ba226fb2c64aea35a45fe40e4a
-v0.5.7 9ce3045f139ee29bf0eea5ec59dfe7df9c3d2c51
-v0.5.8 2cee2e05ba0f3824fdbb8b957958800fa99071a1
-v0.5.9 ad145da3bcded0fe75306df02061d07a633963c3
-v0.5.10 ff4c1557d8358f158f4358788ff18ef93d2470ef
-v0.5.11 d1866845df423c6855e2b365ff230cf7d89a420b
-v0.5.12 f6e8ef27546e9a406724841e75f8df71cc4c2c97
-v0.5.13 dd34ccfe288ebda8e2568cf550994ae890379f45
-v0.6 60a20537c5f3fcc7f11946d81aebc8f90c08c117
-v0.6.1 cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22
-v0.6.2 095ef5f9f60fca1b269aabcc3cfd322b17b9e674
-v0.6.3 d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3
-v0.6.4 58f376002e0c654fbc2de127765fa297cf694a33
-v0.6.5 70d17cd69b80e7627dcad8fd5b6438f2309ac307
-v0.6.6 d4eb5a6eafdd4803d09e6f3963918216cca5a81f
-v0.7 a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3
-v0.7.1 1e1de64a1e59476b7b3d3558b92149246d5c3292
-v0.7.2 59bd58c940673b4c4a6a86f299022614bcf42b22
-v0.7.3 d7acd8b68f8131e0e714400cf124a3e228638643
-v0.7.4 e93649882c5e914eec4a8b8b593dc0587e497033
-v0.7.5 8abc0afe464626a461d2c7e192c912eeebeccc65
-v0.7.6 69aacd9d179fe6dd3110253a099c38b34cff7899
-v0.7.7 f1caed7299cdba5e45635d8377da6cc1e5fd7072
-v0.7.8 2189fe8741b635d8394d55dee28959425cfd5ad0
-v0.7.9 86dc54b836a973f132ed26db9f5a60b29f9b25b2
-v0.7.10 47a42ff432db8a2494e922ca5e767e58020f0446
-v0.7.11 f46ed718ba8d6bb4d42cd7b0f0150c406017c373
-v0.8 d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc
-v0.8.1 cd6f26664cb8d32a13847d6648567c47c580e248
-v0.8.2 7cc0999aebfe63b6bb6dd83c1dff62c3915c4820
-v0.8.3 986500fe2f10870018f1fba1e5465476b8915977
-v0.8.4 f0d0b82b8d6f5f450952113e234d0a5a49e80c48
-v0.8.5 f5d9ddc6c2a2802a61e5ce590c6b6688bf736a69
-v0.8.6 75904ed7452c6cbfb2a70cd379a899d8a75b97c2
-v0.8.7 4f9e568019492126e236da85b5ca0a059f292900
-v0.8.8 a949a49d5efaaf2d881143d23e9fb5ff9e28e88c
-v0.8.9 a034926264cd1025c69d6ceb3fe444965f960b75
-v0.8.10 59accdd814398b884167c0f41dbf46dcccf0c29c
-v0.8.11 d630ec9092f039cb5e51c5dd6d92fc47b91407e4
-v0.8.12 57871c99031cf597ffa0d819057ac1401e129f32
-v0.8.13 e43e6e876513450d235124fcb711f1724ed9814c
-v0.8.14 71901c94dbbaa2f9f156a80c14cc7ea65219fc7c
-v0.8.15 5a7c2f90361e72e9c35e88abd2e11acdc4aba354
-v0.8.16 f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e
-v0.9.0 58b84fb401a804967aa0dd5ee66fafa90194fd30
-v0.9.1 89f68bf0cbf53a81b0553d3816fdbe522f941fa1
-v0.9.2 c7d3b28b93faa6c955fb24bc64031512ee985ee9
-v0.9.3 8704c1ff952b5c85a44f50143bbd1a4f7b4887e2
-v0.9.4 fa310d9e2542cf497d92f007baec8fd751ffa99c
-v0.9.5 3b1e9017c560499786d8a0e46aaaeea65037acac
-v0.9.6 0b4d08fab8e35a264bc7383e878538f8083c33e5
-v0.9.7 f8cb04e4a8b032b72a909f68b808a50936184bee

scripts/manage-powersave.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import argparse
+import multiprocessing
+from openpilot.system.hardware import HARDWARE
+
+
+def main():
+  parser = argparse.ArgumentParser(description='Control power saving mode')
+  parser.add_argument('--enable', action='store_true', help='Enable power saving mode')
+  parser.add_argument('--disable', action='store_true', help='Disable power saving mode')
+  args = parser.parse_args()
+
+  if args.enable and args.disable:
+    parser.error("Cannot specify both --enable and --disable")
+  elif not (args.enable or args.disable):
+    parser.error("Must specify either --enable or --disable")
+
+  print(f"Number of CPU cores available before: [{multiprocessing.cpu_count()}]")
+  HARDWARE.set_power_save(args.enable)
+
+  state = "enabled" if args.enable else "disabled"
+  print(f"Power save mode set to: [{state}]")
+  print(f"Number of CPU cores available now: [{multiprocessing.cpu_count()}]")
+
+
+if __name__ == "__main__":
+  main()

scripts/pyqt_demo.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-
-from PyQt5.QtWidgets import QApplication, QLabel
-from openpilot.selfdrive.ui.qt.python_helpers import set_main_window
-
-
-if __name__ == "__main__":
-  app = QApplication([])
-  label = QLabel('Hello World!')
-
-  # Set full screen and rotate
-  set_main_window(label)
-
-  app.exec_()

selfdrive/assets/offroad/icon_plus.png

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fa3f1c39a4e82adfb52d43fc0ad6773a70dbaa4fc79109a7d6b6c1f73b298eac
-size 2833

selfdrive/car/car_specific.py

@@ -122,7 +122,7 @@ class CarSpecificEvents:
 
     elif self.CP.carName == 'volkswagen':
       events = self.create_common_events(CS, CS_prev, extra_gears=[GearShifter.eco, GearShifter.sport, GearShifter.manumatic],
-                                         pcm_enable=not self.CP.openpilotLongitudinalControl,
+                                         pcm_enable=self.CP.pcmCruise,
                                          enable_buttons=(ButtonType.setCruise, ButtonType.resumeCruise))
 
       # Low speed steer alert hysteresis logic
@@ -149,7 +149,7 @@ class CarSpecificEvents:
       # Main button also can trigger an engagement on these cars
       self.cruise_buttons.append(any(ev.type in HYUNDAI_ENABLE_BUTTONS for ev in CS.buttonEvents))
       events = self.create_common_events(CS, CS_prev, extra_gears=(GearShifter.sport, GearShifter.manumatic),
-                                         pcm_enable=self.CP.pcmCruise, allow_enable=any(self.cruise_buttons))
+                                         pcm_enable=self.CP.pcmCruise, allow_enable=any(self.cruise_buttons), allow_button_cancel=False)
 
       # low speed steer alert hysteresis logic (only for cars with steer cut off above 10 m/s)
       if CS.vEgo < (self.CP.minSteerSpeed + 2.) and self.CP.minSteerSpeed > 10.:
@@ -165,7 +165,7 @@ class CarSpecificEvents:
     return events
 
   def create_common_events(self, CS: structs.CarState, CS_prev: car.CarState, extra_gears=None, pcm_enable=True,
-                           allow_enable=True, enable_buttons=(ButtonType.accelCruise, ButtonType.decelCruise)):
+                           allow_enable=True, allow_button_cancel=True, enable_buttons=(ButtonType.accelCruise, ButtonType.decelCruise)):
     events = Events()
 
     if CS.doorOpen:
@@ -216,7 +216,8 @@ class CarSpecificEvents:
       if not self.CP.pcmCruise and (b.type in enable_buttons and not b.pressed):
         events.add(EventName.buttonEnable)
       # Disable on rising and falling edge of cancel for both stock and OP long
-      if b.type == ButtonType.cancel:
+      # TODO: only check the cancel button with openpilot longitudinal on all brands to match panda safety
+      if b.type == ButtonType.cancel and (allow_button_cancel or not self.CP.pcmCruise):
         events.add(EventName.buttonCancel)
 
     # Handle permanent and temporary steering faults

selfdrive/car/card.py

@@ -5,7 +5,7 @@ import threading
 
 import cereal.messaging as messaging
 
-from cereal import car, log
+from cereal import car, log, custom
 
 from panda import ALTERNATIVE_EXPERIENCE
 
@@ -21,8 +21,10 @@ from opendbc.car.interfaces import CarInterfaceBase, RadarInterfaceBase
 from openpilot.selfdrive.pandad import can_capnp_to_list, can_list_to_can_capnp
 from openpilot.selfdrive.car.cruise import VCruiseHelper
 from openpilot.selfdrive.car.car_specific import MockCarState
+from openpilot.selfdrive.car.helpers import convert_to_capnp
 
 from openpilot.sunnypilot.mads.mads import MadsParams
+from openpilot.sunnypilot.selfdrive.car import interfaces
 
 REPLAY = "REPLAY" in os.environ
 
@@ -65,15 +67,21 @@ class Car:
   CI: CarInterfaceBase
   RI: RadarInterfaceBase
   CP: car.CarParams
+  CP_SP: structs.CarParamsSP
+  CP_SP_capnp: custom.CarParamsSP
 
   def __init__(self, CI=None, RI=None) -> None:
     self.can_sock = messaging.sub_sock('can', timeout=20)
     self.sm = messaging.SubMaster(['pandaStates', 'carControl', 'onroadEvents'])
     self.pm = messaging.PubMaster(['sendcan', 'carState', 'carParams', 'carOutput', 'liveTracks'])
 
+    sock_services = list(self.pm.sock.keys()) + ['carParamsSP']
+    self.pm = messaging.PubMaster(sock_services)
+
     self.can_rcv_cum_timeout_counter = 0
 
     self.CC_prev = car.CarControl.new_message()
+    self.CS_prev = car.CarState.new_message()
     self.initialized_prev = False
 
     self.last_actuators_output = structs.CarControl.Actuators()
@@ -100,13 +108,15 @@ class Car:
           cached_params = _cached_params
 
       self.CI = get_car(*self.can_callbacks, obd_callback(self.params), experimental_long_allowed, num_pandas, cached_params)
-      self.RI = get_radar_interface(self.CI.CP)
+      interfaces.setup_car_interface_sp(self.CI.CP, self.CI.CP_SP, self.params)
+      self.RI = get_radar_interface(self.CI.CP, self.CI.CP_SP)
       self.CP = self.CI.CP
+      self.CP_SP = self.CI.CP_SP
 
       # continue onto next fingerprinting step in pandad
       self.params.put_bool("FirmwareQueryDone", True)
     else:
-      self.CI, self.CP = CI, CI.CP
+      self.CI, self.CP, self.CP_SP = CI, CI.CP, CI.CP_SP
       self.RI = RI
 
     # set alternative experiences from parameters
@@ -117,7 +127,10 @@ class Car:
 
     # mads
     MadsParams().set_alternative_experience(self.CP)
-    MadsParams().set_car_specific_params(self.CP)
+    MadsParams().set_car_specific_params(self.CP, self.CP_SP)
+
+    # Dynamic Experimental Control
+    self.dynamic_experimental_control = self.params.get_bool("DynamicExperimentalControl")
 
     openpilot_enabled_toggle = self.params.get_bool("OpenpilotEnabledToggle")
 
@@ -130,6 +143,15 @@ class Car:
       self.CP.safetyConfigs = [safety_config]
 
     if self.CP.secOcRequired and not self.params.get_bool("IsReleaseBranch"):
+      # Copy user key if available
+      try:
+        with open("/cache/params/SecOCKey") as f:
+          user_key = f.readline().strip()
+          if len(user_key) == 32:
+            self.params.put("SecOCKey", user_key)
+      except Exception:
+        pass
+
       secoc_key = self.params.get("SecOCKey", encoding='utf8')
       if secoc_key is not None:
         saved_secoc_key = bytes.fromhex(secoc_key.strip())
@@ -152,6 +174,14 @@ class Car:
     self.params.put_nonblocking("CarParamsCache", cp_bytes)
     self.params.put_nonblocking("CarParamsPersistent", cp_bytes)
 
+    # Write CarParamsSP for controls
+    # convert to pycapnp representation for caching and logging
+    self.CP_SP_capnp = convert_to_capnp(self.CP_SP)
+    cp_sp_bytes = self.CP_SP_capnp.to_bytes()
+    self.params.put("CarParamsSP", cp_sp_bytes)
+    self.params.put_nonblocking("CarParamsSPCache", cp_sp_bytes)
+    self.params.put_nonblocking("CarParamsSPPersistent", cp_sp_bytes)
+
     self.mock_carstate = MockCarState()
     self.v_cruise_helper = VCruiseHelper(self.CP)
 
@@ -161,6 +191,9 @@ class Car:
     # card is driven by can recv, expected at 100Hz
     self.rk = Ratekeeper(100, print_delay_threshold=None)
 
+    # log fingerprint in sentry
+    interfaces.log_fingerprint(self.CP)
+
   def state_update(self) -> tuple[car.CarState, structs.RadarDataT | None]:
     """carState update loop, driven by can"""
 
@@ -186,8 +219,12 @@ class Car:
     if can_rcv_valid and REPLAY:
       self.can_log_mono_time = messaging.log_from_bytes(can_strs[0]).logMonoTime
 
-    # TODO: mirror the carState.cruiseState struct?
     self.v_cruise_helper.update_v_cruise(CS, self.sm['carControl'].enabled, self.is_metric)
+    if self.sm['carControl'].enabled and not self.CC_prev.enabled:
+      # Use CarState w/ buttons from the step selfdrived enables on
+      self.v_cruise_helper.initialize_v_cruise(self.CS_prev, self.experimental_mode, self.dynamic_experimental_control)
+
+    # TODO: mirror the carState.cruiseState struct?
     CS.vCruise = float(self.v_cruise_helper.v_cruise_kph)
     CS.vCruiseCluster = float(self.v_cruise_helper.v_cruise_cluster_kph)
 
@@ -223,13 +260,21 @@ class Car:
       tracks_msg.liveTracks = RD
       self.pm.send('liveTracks', tracks_msg)
 
+    # carParamsSP - logged every 50 seconds (> 1 per segment)
+    if self.sm.frame % int(50. / DT_CTRL) == 0:
+      cp_sp_send = messaging.new_message('carParamsSP')
+      cp_sp_send.valid = True
+      cp_sp_send.carParamsSP = self.CP_SP_capnp
+      self.pm.send('carParamsSP', cp_sp_send)
+
   def controls_update(self, CS: car.CarState, CC: car.CarControl):
     """control update loop, driven by carControl"""
 
     if not self.initialized_prev:
       # Initialize CarInterface, once controls are ready
       # TODO: this can make us miss at least a few cycles when doing an ECU knockout
-      self.CI.init(self.CP, *self.can_callbacks)
+      self.CI.init(self.CP, self.CP_SP, *self.can_callbacks)
+      interfaces.initialize_car_interface_sp(self.CP, self.CP_SP, self.params, *self.can_callbacks)
       # signal pandad to switch to car safety mode
       self.params.put_bool_nonblocking("ControlsReady", True)
 
@@ -244,9 +289,6 @@ class Car:
   def step(self):
     CS, RD = self.state_update()
 
-    if self.sm['carControl'].enabled and not self.CC_prev.enabled:
-      self.v_cruise_helper.initialize_v_cruise(CS, self.experimental_mode)
-
     self.state_publish(CS, RD)
 
     initialized = (not any(e.name == EventName.selfdriveInitializing for e in self.sm['onroadEvents']) and
@@ -255,11 +297,16 @@ class Car:
       self.controls_update(CS, self.sm['carControl'])
 
     self.initialized_prev = initialized
+    self.CS_prev = CS
 
   def params_thread(self, evt):
     while not evt.is_set():
       self.is_metric = self.params.get_bool("IsMetric")
       self.experimental_mode = self.params.get_bool("ExperimentalMode") and self.CP.openpilotLongitudinalControl
+
+      # sunnypilot
+      self.dynamic_experimental_control = self.params.get_bool("DynamicExperimentalControl")
+
       time.sleep(0.1)
 
   def card_thread(self):

selfdrive/car/cruise.py

@@ -1,8 +1,8 @@
 import math
+import numpy as np
 
 from cereal import car
 from openpilot.common.conversions import Conversions as CV
-from openpilot.common.numpy_fast import clip
 
 
 # WARNING: this value was determined based on the model's training distribution,
@@ -106,7 +106,7 @@ class VCruiseHelper:
     if CS.gasPressed and button_type in (ButtonType.decelCruise, ButtonType.setCruise):
       self.v_cruise_kph = max(self.v_cruise_kph, CS.vEgo * CV.MS_TO_KPH)
 
-    self.v_cruise_kph = clip(round(self.v_cruise_kph, 1), V_CRUISE_MIN, V_CRUISE_MAX)
+    self.v_cruise_kph = np.clip(round(self.v_cruise_kph, 1), V_CRUISE_MIN, V_CRUISE_MAX)
 
   def update_button_timers(self, CS, enabled):
     # increment timer for buttons still pressed
@@ -120,16 +120,17 @@ class VCruiseHelper:
         self.button_timers[b.type.raw] = 1 if b.pressed else 0
         self.button_change_states[b.type.raw] = {"standstill": CS.cruiseState.standstill, "enabled": enabled}
 
-  def initialize_v_cruise(self, CS, experimental_mode: bool) -> None:
+  def initialize_v_cruise(self, CS, experimental_mode: bool, dynamic_experimental_control: bool) -> None:
     # initializing is handled by the PCM
     if self.CP.pcmCruise:
       return
 
-    initial = V_CRUISE_INITIAL_EXPERIMENTAL_MODE if experimental_mode else V_CRUISE_INITIAL
+    initial_experimental_mode = experimental_mode and not dynamic_experimental_control
+    initial = V_CRUISE_INITIAL_EXPERIMENTAL_MODE if initial_experimental_mode else V_CRUISE_INITIAL
 
     if any(b.type in (ButtonType.accelCruise, ButtonType.resumeCruise) for b in CS.buttonEvents) and self.v_cruise_initialized:
       self.v_cruise_kph = self.v_cruise_kph_last
     else:
-      self.v_cruise_kph = int(round(clip(CS.vEgo * CV.MS_TO_KPH, initial, V_CRUISE_MAX)))
+      self.v_cruise_kph = int(round(np.clip(CS.vEgo * CV.MS_TO_KPH, initial, V_CRUISE_MAX)))
 
     self.v_cruise_cluster_kph = self.v_cruise_kph

selfdrive/car/helpers.py

@@ -0,0 +1,46 @@
+import capnp
+from typing import Any
+
+from cereal import custom
+from opendbc.car import structs
+
+_FIELDS = '__dataclass_fields__'  # copy of dataclasses._FIELDS
+
+
+def is_dataclass(obj):
+  """Similar to dataclasses.is_dataclass without instance type check checking"""
+  return hasattr(obj, _FIELDS)
+
+
+def _asdictref_inner(obj) -> dict[str, Any] | Any:
+  if is_dataclass(obj):
+    ret = {}
+    for field in getattr(obj, _FIELDS):  # similar to dataclasses.fields()
+      ret[field] = _asdictref_inner(getattr(obj, field))
+    return ret
+  elif isinstance(obj, (tuple, list)):
+    return type(obj)(_asdictref_inner(v) for v in obj)
+  else:
+    return obj
+
+
+def asdictref(obj) -> dict[str, Any]:
+  """
+  Similar to dataclasses.asdict without recursive type checking and copy.deepcopy
+  Note that the resulting dict will contain references to the original struct as a result
+  """
+  if not is_dataclass(obj):
+    raise TypeError("asdictref() should be called on dataclass instances")
+
+  return _asdictref_inner(obj)
+
+
+def convert_to_capnp(struct: structs.CarParamsSP) -> capnp.lib.capnp._DynamicStructBuilder:
+  struct_dict = asdictref(struct)
+
+  if isinstance(struct, structs.CarParamsSP):
+    struct_capnp = custom.CarParamsSP.new_message(**struct_dict)
+  else:
+    raise ValueError(f"Unsupported struct type: {type(struct)}")
+
+  return struct_capnp

selfdrive/car/tests/test_car_interfaces.py

@@ -40,9 +40,12 @@ class TestCarInterfaces:
 
     car_params = CarInterface.get_params(car_name, args['fingerprints'], args['car_fw'],
                                          experimental_long=args['experimental_long'], docs=False)
+    car_params, car_params_sp = CarInterface.get_params_sp(car_params, car_name, args['fingerprints'], args['car_fw'],
+                                                           experimental_long=args['experimental_long'], docs=False)
     car_params = car_params.as_reader()
-    car_interface = CarInterface(car_params, CarController, CarState)
+    car_interface = CarInterface(car_params, car_params_sp, CarController, CarState)
     assert car_params
+    assert car_params_sp
     assert car_interface
 
     assert car_params.mass > 1

selfdrive/car/tests/test_cruise_speed.py

@@ -57,16 +57,16 @@ class TestVCruiseHelper:
     for _ in range(2):
       self.v_cruise_helper.update_v_cruise(car.CarState(cruiseState={"available": False}), enabled=False, is_metric=False)
 
-  def enable(self, v_ego, experimental_mode):
+  def enable(self, v_ego, experimental_mode, dynamic_experimental_control):
     # Simulates user pressing set with a current speed
-    self.v_cruise_helper.initialize_v_cruise(car.CarState(vEgo=v_ego), experimental_mode)
+    self.v_cruise_helper.initialize_v_cruise(car.CarState(vEgo=v_ego), experimental_mode, dynamic_experimental_control)
 
   def test_adjust_speed(self):
     """
     Asserts speed changes on falling edges of buttons.
     """
 
-    self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False)
+    self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False, False)
 
     for btn in (ButtonType.accelCruise, ButtonType.decelCruise):
       for pressed in (True, False):
@@ -90,7 +90,7 @@ class TestVCruiseHelper:
       CS.buttonEvents = [ButtonEvent(type=ButtonType.decelCruise, pressed=pressed)]
       self.v_cruise_helper.update_v_cruise(CS, enabled=enabled, is_metric=False)
       if pressed:
-        self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False)
+        self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False, False)
 
       # Expected diff on enabling. Speed should not change on falling edge of pressed
       assert not pressed == self.v_cruise_helper.v_cruise_kph == self.v_cruise_helper.v_cruise_kph_last
@@ -100,7 +100,7 @@ class TestVCruiseHelper:
     Asserts we don't increment set speed if user presses resume/accel to exit cruise standstill.
     """
 
-    self.enable(0, False)
+    self.enable(0, False, False)
 
     for standstill in (True, False):
       for pressed in (True, False):
@@ -120,7 +120,7 @@ class TestVCruiseHelper:
 
     for v_ego in np.linspace(0, 100, 101):
       self.reset_cruise_speed_state()
-      self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False)
+      self.enable(V_CRUISE_INITIAL * CV.KPH_TO_MS, False, False)
 
       # first decrement speed, then perform gas pressed logic
       expected_v_cruise_kph = self.v_cruise_helper.v_cruise_kph - IMPERIAL_INCREMENT
@@ -142,10 +142,11 @@ class TestVCruiseHelper:
     """
 
     for experimental_mode in (True, False):
-      for v_ego in np.linspace(0, 100, 101):
-        self.reset_cruise_speed_state()
-        assert not self.v_cruise_helper.v_cruise_initialized
+      for dynamic_experimental_control in (True, False):
+        for v_ego in np.linspace(0, 100, 101):
+          self.reset_cruise_speed_state()
+          assert not self.v_cruise_helper.v_cruise_initialized
 
-        self.enable(float(v_ego), experimental_mode)
-        assert V_CRUISE_INITIAL <= self.v_cruise_helper.v_cruise_kph <= V_CRUISE_MAX
-        assert self.v_cruise_helper.v_cruise_initialized
+          self.enable(float(v_ego), experimental_mode, dynamic_experimental_control)
+          assert V_CRUISE_INITIAL <= self.v_cruise_helper.v_cruise_kph <= V_CRUISE_MAX
+          assert self.v_cruise_helper.v_cruise_initialized

selfdrive/car/tests/test_models.py

@@ -1,4 +1,5 @@
 import capnp
+import copy
 import os
 import pytest
 import random
@@ -158,7 +159,9 @@ class TestCarModelBase(unittest.TestCase):
 
     cls.CarInterface, cls.CarController, cls.CarState, cls.RadarInterface = interfaces[cls.platform]
     cls.CP = cls.CarInterface.get_params(cls.platform,  cls.fingerprint, car_fw, experimental_long, docs=False)
+    cls.CP, cls.CP_SP = cls.CarInterface.get_params_sp(cls.CP, cls.platform,  cls.fingerprint, car_fw, experimental_long, docs=False)
     assert cls.CP
+    assert cls.CP_SP
     assert cls.CP.carFingerprint == cls.platform
 
     os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
@@ -168,7 +171,7 @@ class TestCarModelBase(unittest.TestCase):
     del cls.can_msgs
 
   def setUp(self):
-    self.CI = self.CarInterface(self.CP.copy(), self.CarController, self.CarState)
+    self.CI = self.CarInterface(self.CP.copy(), copy.deepcopy(self.CP_SP), self.CarController, self.CarState)
     assert self.CI
 
     Params().put_bool("OpenpilotEnabledToggle", self.openpilot_enabled)
@@ -217,7 +220,7 @@ class TestCarModelBase(unittest.TestCase):
     self.assertEqual(can_invalid_cnt, 0)
 
   def test_radar_interface(self):
-    RI = self.RadarInterface(self.CP)
+    RI = self.RadarInterface(self.CP, self.CP_SP)
     assert RI
 
     # Since OBD port is multiplexed to bus 1 (commonly radar bus) while fingerprinting,
@@ -277,7 +280,7 @@ class TestCarModelBase(unittest.TestCase):
     def test_car_controller(car_control):
       now_nanos = 0
       msgs_sent = 0
-      CI = self.CarInterface(self.CP, self.CarController, self.CarState)
+      CI = self.CarInterface(self.CP, self.CP_SP, self.CarController, self.CarState)
       for _ in range(round(10.0 / DT_CTRL)):  # make sure we hit the slowest messages
         CI.update([])
         _, sendcan = CI.apply(car_control, now_nanos)
@@ -387,7 +390,7 @@ class TestCarModelBase(unittest.TestCase):
     controls_allowed_prev = False
     CS_prev = car.CarState.new_message()
     checks = defaultdict(int)
-    selfdrived = SelfdriveD(CP=self.CP)
+    selfdrived = SelfdriveD(CP=self.CP, CP_SP=self.CP_SP)
     selfdrived.initialized = True
     for idx, can in enumerate(self.can_msgs):
       CS = self.CI.update(can_capnp_to_list((can.as_builder().to_bytes(), ))).as_reader()

selfdrive/controls/controlsd.py

@@ -2,7 +2,7 @@
 import math
 from typing import SupportsFloat
 
-from cereal import car, log
+from cereal import car, log, custom
 import cereal.messaging as messaging
 from openpilot.common.conversions import Conversions as CV
 from openpilot.common.params import Params
@@ -34,7 +34,11 @@ class Controls:
     self.CP = messaging.log_from_bytes(self.params.get("CarParams", block=True), car.CarParams)
     cloudlog.info("controlsd got CarParams")
 
-    self.CI = get_car_interface(self.CP)
+    cloudlog.info("controlsd is waiting for CarParamsSP")
+    self.CP_SP = messaging.log_from_bytes(self.params.get("CarParamsSP", block=True), custom.CarParamsSP)
+    cloudlog.info("controlsd got CarParamsSP")
+
+    self.CI = get_car_interface(self.CP, self.CP_SP)
 
     self.sm = messaging.SubMaster(['liveParameters', 'liveTorqueParameters', 'modelV2', 'selfdriveState',
                                    'liveCalibration', 'livePose', 'longitudinalPlan', 'carState', 'carOutput',
@@ -119,15 +123,16 @@ class Controls:
 
     # accel PID loop
     pid_accel_limits = self.CI.get_pid_accel_limits(self.CP, CS.vEgo, CS.vCruise * CV.KPH_TO_MS)
-    actuators.accel = self.LoC.update(CC.longActive, CS, long_plan.aTarget, long_plan.shouldStop, pid_accel_limits)
+    actuators.accel = float(self.LoC.update(CC.longActive, CS, long_plan.aTarget, long_plan.shouldStop, pid_accel_limits))
 
     # Steering PID loop and lateral MPC
     self.desired_curvature = clip_curvature(CS.vEgo, self.desired_curvature, model_v2.action.desiredCurvature)
-    actuators.curvature = self.desired_curvature
-    actuators.steer, actuators.steeringAngleDeg, lac_log = self.LaC.update(CC.latActive, CS, self.VM, lp,
+    actuators.curvature = float(self.desired_curvature)
+    steer, steeringAngleDeg, lac_log = self.LaC.update(CC.latActive, CS, self.VM, lp,
                                                                             self.steer_limited, self.desired_curvature,
                                                                             self.calibrated_pose) # TODO what if not available
-
+    actuators.steer = float(steer)
+    actuators.steeringAngleDeg = float(steeringAngleDeg)
     # Ensure no NaNs/Infs
     for p in ACTUATOR_FIELDS:
       attr = getattr(actuators, p)
@@ -192,7 +197,7 @@ class Controls:
 
     cs.longitudinalPlanMonoTime = self.sm.logMonoTime['longitudinalPlan']
     cs.lateralPlanMonoTime = self.sm.logMonoTime['modelV2']
-    cs.desiredCurvature = self.desired_curvature
+    cs.desiredCurvature = float(self.desired_curvature)
     cs.longControlState = self.LoC.long_control_state
     cs.upAccelCmd = float(self.LoC.pid.p)
     cs.uiAccelCmd = float(self.LoC.pid.i)

selfdrive/controls/lib/drive_helpers.py

@@ -1,5 +1,5 @@
+import numpy as np
 from cereal import log
-from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_CTRL
 
 MIN_SPEED = 1.0
@@ -13,10 +13,10 @@ MAX_LATERAL_JERK = 5.0
 MAX_VEL_ERR = 5.0
 
 def clip_curvature(v_ego, prev_curvature, new_curvature):
-  new_curvature = clip(new_curvature, -MAX_CURVATURE, MAX_CURVATURE)
+  new_curvature = np.clip(new_curvature, -MAX_CURVATURE, MAX_CURVATURE)
   v_ego = max(MIN_SPEED, v_ego)
   max_curvature_rate = MAX_LATERAL_JERK / (v_ego**2) # inexact calculation, check https://github.com/commaai/openpilot/pull/24755
-  safe_desired_curvature = clip(new_curvature,
+  safe_desired_curvature = np.clip(new_curvature,
                                 prev_curvature - max_curvature_rate * DT_CTRL,
                                 prev_curvature + max_curvature_rate * DT_CTRL)
 
@@ -26,6 +26,6 @@ def clip_curvature(v_ego, prev_curvature, new_curvature):
 def get_speed_error(modelV2: log.ModelDataV2, v_ego: float) -> float:
   # ToDo: Try relative error, and absolute speed
   if len(modelV2.temporalPose.trans):
-    vel_err = clip(modelV2.temporalPose.trans[0] - v_ego, -MAX_VEL_ERR, MAX_VEL_ERR)
+    vel_err = np.clip(modelV2.temporalPose.trans[0] - v_ego, -MAX_VEL_ERR, MAX_VEL_ERR)
     return float(vel_err)
   return 0.0

selfdrive/controls/lib/latcontrol.py

@@ -1,6 +1,6 @@
+import numpy as np
 from abc import abstractmethod, ABC
 
-from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_CTRL
 
 MIN_LATERAL_CONTROL_SPEED = 0.3  # m/s
@@ -28,5 +28,5 @@ class LatControl(ABC):
       self.sat_count += self.sat_count_rate
     else:
       self.sat_count -= self.sat_count_rate
-    self.sat_count = clip(self.sat_count, 0.0, self.sat_limit)
+    self.sat_count = np.clip(self.sat_count, 0.0, self.sat_limit)
     return self.sat_count > (self.sat_limit - 1e-3)

selfdrive/controls/lib/latcontrol_angle.py

@@ -23,7 +23,7 @@ class LatControlAngle(LatControl):
       angle_steers_des += params.angleOffsetDeg
 
     angle_control_saturated = abs(angle_steers_des - CS.steeringAngleDeg) > STEER_ANGLE_SATURATION_THRESHOLD
-    angle_log.saturated = self._check_saturation(angle_control_saturated, CS, False)
+    angle_log.saturated = bool(self._check_saturation(angle_control_saturated, CS, False))
     angle_log.steeringAngleDeg = float(CS.steeringAngleDeg)
     angle_log.steeringAngleDesiredDeg = angle_steers_des
     return 0, float(angle_steers_des), angle_log

selfdrive/controls/lib/latcontrol_pid.py

@@ -39,10 +39,10 @@ class LatControlPID(LatControl):
       output_steer = self.pid.update(error, override=CS.steeringPressed,
                                      feedforward=steer_feedforward, speed=CS.vEgo)
       pid_log.active = True
-      pid_log.p = self.pid.p
-      pid_log.i = self.pid.i
-      pid_log.f = self.pid.f
-      pid_log.output = output_steer
-      pid_log.saturated = self._check_saturation(self.steer_max - abs(output_steer) < 1e-3, CS, steer_limited)
+      pid_log.p = float(self.pid.p)
+      pid_log.i = float(self.pid.i)
+      pid_log.f = float(self.pid.f)
+      pid_log.output = float(output_steer)
+      pid_log.saturated = bool(self._check_saturation(self.steer_max - abs(output_steer) < 1e-3, CS, steer_limited))
 
     return output_steer, angle_steers_des, pid_log

selfdrive/controls/lib/latcontrol_torque.py

@@ -1,8 +1,8 @@
 import math
+import numpy as np
 
 from cereal import log
 from opendbc.car.interfaces import LatControlInputs
-from openpilot.common.numpy_fast import interp
 from openpilot.selfdrive.controls.lib.latcontrol import LatControl
 from openpilot.common.pid import PIDController
 from openpilot.selfdrive.controls.lib.vehicle_model import ACCELERATION_DUE_TO_GRAVITY
@@ -51,7 +51,7 @@ class LatControlTorque(LatControl):
       else:
         assert calibrated_pose is not None
         actual_curvature_pose = calibrated_pose.angular_velocity.yaw / CS.vEgo
-        actual_curvature = interp(CS.vEgo, [2.0, 5.0], [actual_curvature_vm, actual_curvature_pose])
+        actual_curvature = np.interp(CS.vEgo, [2.0, 5.0], [actual_curvature_vm, actual_curvature_pose])
         curvature_deadzone = 0.0
       desired_lateral_accel = desired_curvature * CS.vEgo ** 2
 
@@ -60,7 +60,7 @@ class LatControlTorque(LatControl):
       actual_lateral_accel = actual_curvature * CS.vEgo ** 2
       lateral_accel_deadzone = curvature_deadzone * CS.vEgo ** 2
 
-      low_speed_factor = interp(CS.vEgo, LOW_SPEED_X, LOW_SPEED_Y)**2
+      low_speed_factor = np.interp(CS.vEgo, LOW_SPEED_X, LOW_SPEED_Y)**2
       setpoint = desired_lateral_accel + low_speed_factor * desired_curvature
       measurement = actual_lateral_accel + low_speed_factor * actual_curvature
       gravity_adjusted_lateral_accel = desired_lateral_accel - roll_compensation
@@ -68,7 +68,7 @@ class LatControlTorque(LatControl):
                                                             setpoint, lateral_accel_deadzone, friction_compensation=False, gravity_adjusted=False)
       torque_from_measurement = self.torque_from_lateral_accel(LatControlInputs(measurement, roll_compensation, CS.vEgo, CS.aEgo), self.torque_params,
                                                                measurement, lateral_accel_deadzone, friction_compensation=False, gravity_adjusted=False)
-      pid_log.error = torque_from_setpoint - torque_from_measurement
+      pid_log.error = float(torque_from_setpoint - torque_from_measurement)
       ff = self.torque_from_lateral_accel(LatControlInputs(gravity_adjusted_lateral_accel, roll_compensation, CS.vEgo, CS.aEgo), self.torque_params,
                                           desired_lateral_accel - actual_lateral_accel, lateral_accel_deadzone, friction_compensation=True,
                                           gravity_adjusted=True)
@@ -80,14 +80,14 @@ class LatControlTorque(LatControl):
                                       freeze_integrator=freeze_integrator)
 
       pid_log.active = True
-      pid_log.p = self.pid.p
-      pid_log.i = self.pid.i
-      pid_log.d = self.pid.d
-      pid_log.f = self.pid.f
-      pid_log.output = -output_torque
-      pid_log.actualLateralAccel = actual_lateral_accel
-      pid_log.desiredLateralAccel = desired_lateral_accel
-      pid_log.saturated = self._check_saturation(self.steer_max - abs(output_torque) < 1e-3, CS, steer_limited)
+      pid_log.p = float(self.pid.p)
+      pid_log.i = float(self.pid.i)
+      pid_log.d = float(self.pid.d)
+      pid_log.f = float(self.pid.f)
+      pid_log.output = float(-output_torque)
+      pid_log.actualLateralAccel = float(actual_lateral_accel)
+      pid_log.desiredLateralAccel = float(desired_lateral_accel)
+      pid_log.saturated = bool(self._check_saturation(self.steer_max - abs(output_torque) < 1e-3, CS, steer_limited))
 
     # TODO left is positive in this convention
     return -output_torque, 0.0, pid_log

selfdrive/controls/lib/longcontrol.py

@@ -1,5 +1,5 @@
+import numpy as np
 from cereal import car
-from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_CTRL
 from openpilot.selfdrive.controls.lib.drive_helpers import CONTROL_N
 from openpilot.common.pid import PIDController
@@ -84,5 +84,5 @@ class LongControl:
       output_accel = self.pid.update(error, speed=CS.vEgo,
                                      feedforward=a_target)
 
-    self.last_output_accel = clip(output_accel, accel_limits[0], accel_limits[1])
+    self.last_output_accel = np.clip(output_accel, accel_limits[0], accel_limits[1])
     return self.last_output_accel

selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py

@@ -4,7 +4,6 @@ import time
 import numpy as np
 from cereal import log
 from opendbc.car.interfaces import ACCEL_MIN
-from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import DT_MDL
 from openpilot.common.swaglog import cloudlog
 # WARNING: imports outside of constants will not trigger a rebuild
@@ -320,9 +319,9 @@ class LongitudinalMpc:
     # MPC will not converge if immediate crash is expected
     # Clip lead distance to what is still possible to brake for
     min_x_lead = ((v_ego + v_lead)/2) * (v_ego - v_lead) / (-ACCEL_MIN * 2)
-    x_lead = clip(x_lead, min_x_lead, 1e8)
-    v_lead = clip(v_lead, 0.0, 1e8)
-    a_lead = clip(a_lead, -10., 5.)
+    x_lead = np.clip(x_lead, min_x_lead, 1e8)
+    v_lead = np.clip(v_lead, 0.0, 1e8)
+    a_lead = np.clip(a_lead, -10., 5.)
     lead_xv = self.extrapolate_lead(x_lead, v_lead, a_lead, a_lead_tau)
     return lead_xv
 

selfdrive/controls/lib/longitudinal_planner.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 import math
 import numpy as np
-from openpilot.common.numpy_fast import clip, interp
 
 import cereal.messaging as messaging
 from opendbc.car.interfaces import ACCEL_MIN, ACCEL_MAX
@@ -16,6 +15,8 @@ from openpilot.selfdrive.controls.lib.drive_helpers import CONTROL_N, get_speed_
 from openpilot.selfdrive.car.cruise import V_CRUISE_MAX, V_CRUISE_UNSET
 from openpilot.common.swaglog import cloudlog
 
+from openpilot.sunnypilot.selfdrive.controls.lib.longitudinal_planner import LongitudinalPlannerSP
+
 LON_MPC_STEP = 0.2  # first step is 0.2s
 A_CRUISE_MIN = -1.2
 A_CRUISE_MAX_VALS = [1.6, 1.2, 0.8, 0.6]
@@ -30,7 +31,7 @@ _A_TOTAL_MAX_BP = [20., 40.]
 
 
 def get_max_accel(v_ego):
-  return interp(v_ego, A_CRUISE_MAX_BP, A_CRUISE_MAX_VALS)
+  return np.interp(v_ego, A_CRUISE_MAX_BP, A_CRUISE_MAX_VALS)
 
 def get_coast_accel(pitch):
   return np.sin(pitch) * -5.65 - 0.3  # fitted from data using xx/projects/allow_throttle/compute_coast_accel.py
@@ -43,7 +44,7 @@ def limit_accel_in_turns(v_ego, angle_steers, a_target, CP):
   """
   # FIXME: This function to calculate lateral accel is incorrect and should use the VehicleModel
   # The lookup table for turns should also be updated if we do this
-  a_total_max = interp(v_ego, _A_TOTAL_MAX_BP, _A_TOTAL_MAX_V)
+  a_total_max = np.interp(v_ego, _A_TOTAL_MAX_BP, _A_TOTAL_MAX_V)
   a_y = v_ego ** 2 * angle_steers * CV.DEG_TO_RAD / (CP.steerRatio * CP.wheelbase)
   a_x_allowed = math.sqrt(max(a_total_max ** 2 - a_y ** 2, 0.))
 
@@ -55,9 +56,9 @@ def get_accel_from_plan(speeds, accels, action_t=DT_MDL, vEgoStopping=0.05):
     v_now = speeds[0]
     a_now = accels[0]
 
-    v_target = interp(action_t, CONTROL_N_T_IDX, speeds)
+    v_target = np.interp(action_t, CONTROL_N_T_IDX, speeds)
     a_target = 2 * (v_target - v_now) / (action_t) - a_now
-    v_target_1sec = interp(action_t + 1.0, CONTROL_N_T_IDX, speeds)
+    v_target_1sec = np.interp(action_t + 1.0, CONTROL_N_T_IDX, speeds)
   else:
     v_target = 0.0
     v_target_1sec = 0.0
@@ -67,10 +68,11 @@ def get_accel_from_plan(speeds, accels, action_t=DT_MDL, vEgoStopping=0.05):
   return a_target, should_stop
 
 
-class LongitudinalPlanner:
+class LongitudinalPlanner(LongitudinalPlannerSP):
   def __init__(self, CP, init_v=0.0, init_a=0.0, dt=DT_MDL):
     self.CP = CP
     self.mpc = LongitudinalMpc(dt=dt)
+    LongitudinalPlannerSP.__init__(self, self.CP, self.mpc)
     self.fcw = False
     self.dt = dt
     self.allow_throttle = True
@@ -105,7 +107,10 @@ class LongitudinalPlanner:
     return x, v, a, j, throttle_prob
 
   def update(self, sm):
+    LongitudinalPlannerSP.update(self, sm)
     self.mpc.mode = 'blended' if sm['selfdriveState'].experimentalMode else 'acc'
+    if dec_mpc_mode := self.get_mpc_mode():
+      self.mpc.mode = dec_mpc_mode
 
     if len(sm['carControl'].orientationNED) == 3:
       accel_coast = get_coast_accel(sm['carControl'].orientationNED[1])
@@ -139,7 +144,7 @@ class LongitudinalPlanner:
     if reset_state:
       self.v_desired_filter.x = v_ego
       # Clip aEgo to cruise limits to prevent large accelerations when becoming active
-      self.a_desired = clip(sm['carState'].aEgo, accel_limits[0], accel_limits[1])
+      self.a_desired = np.clip(sm['carState'].aEgo, accel_limits[0], accel_limits[1])
 
     # Prevent divergence, smooth in current v_ego
     self.v_desired_filter.x = max(0.0, self.v_desired_filter.update(v_ego))
@@ -151,7 +156,7 @@ class LongitudinalPlanner:
 
     if not self.allow_throttle:
       clipped_accel_coast = max(accel_coast, accel_limits_turns[0])
-      clipped_accel_coast_interp = interp(v_ego, [MIN_ALLOW_THROTTLE_SPEED, MIN_ALLOW_THROTTLE_SPEED*2], [accel_limits_turns[1], clipped_accel_coast])
+      clipped_accel_coast_interp = np.interp(v_ego, [MIN_ALLOW_THROTTLE_SPEED, MIN_ALLOW_THROTTLE_SPEED*2], [accel_limits_turns[1], clipped_accel_coast])
       accel_limits_turns[1] = min(accel_limits_turns[1], clipped_accel_coast_interp)
 
     if force_slow_decel:
@@ -176,7 +181,7 @@ class LongitudinalPlanner:
 
     # Interpolate 0.05 seconds and save as starting point for next iteration
     a_prev = self.a_desired
-    self.a_desired = float(interp(self.dt, CONTROL_N_T_IDX, self.a_desired_trajectory))
+    self.a_desired = float(np.interp(self.dt, CONTROL_N_T_IDX, self.a_desired_trajectory))
     self.v_desired_filter.x = self.v_desired_filter.x + self.dt * (self.a_desired + a_prev) / 2.0
 
   def publish(self, sm, pm):
@@ -200,9 +205,11 @@ class LongitudinalPlanner:
     action_t =  self.CP.longitudinalActuatorDelay + DT_MDL
     a_target, should_stop = get_accel_from_plan(longitudinalPlan.speeds, longitudinalPlan.accels,
                                                 action_t=action_t, vEgoStopping=self.CP.vEgoStopping)
-    longitudinalPlan.aTarget = a_target
-    longitudinalPlan.shouldStop = should_stop
+    longitudinalPlan.aTarget = float(a_target)
+    longitudinalPlan.shouldStop = bool(should_stop)
     longitudinalPlan.allowBrake = True
-    longitudinalPlan.allowThrottle = self.allow_throttle
+    longitudinalPlan.allowThrottle = bool(self.allow_throttle)
 
     pm.send('longitudinalPlan', plan_send)
+
+    self.publish_longitudinal_plan_sp(sm, pm)

selfdrive/controls/lib/tests/test_latcontrol.py

@@ -19,7 +19,8 @@ class TestLatControl:
   def test_saturation(self, car_name, controller):
     CarInterface, CarController, CarState, RadarInterface = interfaces[car_name]
     CP = CarInterface.get_non_essential_params(car_name)
-    CI = CarInterface(CP, CarController, CarState)
+    CP, CP_SP = CarInterface.get_non_essential_params_sp(CP, car_name)
+    CI = CarInterface(CP, CP_SP, CarController, CarState)
     VM = VehicleModel(CP)
 
     controller = controller(CP.as_reader(), CI)

selfdrive/controls/plannerd.py

@@ -18,7 +18,7 @@ def main():
 
   ldw = LaneDepartureWarning()
   longitudinal_planner = LongitudinalPlanner(CP)
-  pm = messaging.PubMaster(['longitudinalPlan', 'driverAssistance'])
+  pm = messaging.PubMaster(['longitudinalPlan', 'driverAssistance', 'longitudinalPlanSP'])
   sm = messaging.SubMaster(['carControl', 'carState', 'controlsState', 'liveParameters', 'radarState', 'modelV2', 'selfdriveState'],
                            poll='modelV2', ignore_avg_freq=['radarState'])
 

selfdrive/controls/radard.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 import math
+import numpy as np
 from collections import deque
 from typing import Any
 
 import capnp
 from cereal import messaging, log, car
-from openpilot.common.numpy_fast import interp
 from openpilot.common.params import Params
 from openpilot.common.realtime import DT_MDL, Priority, config_realtime_process
 from openpilot.common.swaglog import cloudlog
@@ -44,7 +44,7 @@ class KalmanParams:
           0.28144091, 0.27958406, 0.27783249, 0.27617149, 0.27458948, 0.27307714,
           0.27162685, 0.27023228, 0.26888809, 0.26758976, 0.26633338, 0.26511557,
           0.26393339, 0.26278425]
-    self.K = [[interp(dt, dts, K0)], [interp(dt, dts, K1)]]
+    self.K = [[np.interp(dt, dts, K0)], [np.interp(dt, dts, K1)]]
 
 
 class Track:

selfdrive/debug/clear_dtc.py

@@ -2,8 +2,8 @@
 import sys
 import argparse
 from subprocess import check_output, CalledProcessError
+from opendbc.car.uds import UdsClient, MessageTimeoutError, SESSION_TYPE, DTC_GROUP_TYPE
 from panda import Panda
-from panda.python.uds import UdsClient, MessageTimeoutError, SESSION_TYPE, DTC_GROUP_TYPE
 
 parser = argparse.ArgumentParser(description="clear DTC status")
 parser.add_argument("addr", type=lambda x: int(x,0), nargs="?", default=0x7DF) # default is functional (broadcast) address

selfdrive/debug/debug_fw_fingerprinting_offline.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 import argparse
 
+from opendbc.car import uds
 from openpilot.tools.lib.live_logreader import live_logreader
 from openpilot.tools.lib.logreader import LogReader, ReadMode
-from panda.python import uds
 
 
 def main(route: str | None, addrs: list[int]):

selfdrive/debug/hyundai_enable_radar_points.py

@@ -16,8 +16,8 @@ import argparse
 from typing import NamedTuple
 from subprocess import check_output, CalledProcessError
 
+from opendbc.car.uds import UdsClient, SESSION_TYPE, DATA_IDENTIFIER_TYPE
 from panda.python import Panda
-from panda.python.uds import UdsClient, SESSION_TYPE, DATA_IDENTIFIER_TYPE
 
 class ConfigValues(NamedTuple):
   default_config: bytes

selfdrive/debug/live_cpu_and_temp.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 import argparse
+import numpy as np
 import capnp
 from collections import defaultdict
 
 from cereal.messaging import SubMaster
-from openpilot.common.numpy_fast import mean
 
 def cputime_total(ct):
   return ct.user + ct.nice + ct.system + ct.idle + ct.iowait + ct.irq + ct.softirq
@@ -49,7 +49,7 @@ if __name__ == "__main__":
 
     if sm.updated['deviceState']:
       t = sm['deviceState']
-      last_temp = mean(t.cpuTempC)
+      last_temp = np.mean(t.cpuTempC)
       last_mem = t.memoryUsagePercent
 
     if sm.updated['procLog']:
@@ -72,7 +72,7 @@ if __name__ == "__main__":
       total_times = total_times_new[:]
       busy_times = busy_times_new[:]
 
-      print(f"CPU {100.0 * mean(cores):.2f}% - RAM: {last_mem:.2f}% - Temp {last_temp:.2f}C")
+      print(f"CPU {100.0 * np.mean(cores):.2f}% - RAM: {last_mem:.2f}% - Temp {last_temp:.2f}C")
 
       if args.cpu and prev_proclog is not None and prev_proclog_t is not None:
         procs: dict[str, float] = defaultdict(float)

selfdrive/debug/read_dtc_status.py

@@ -2,9 +2,8 @@
 import sys
 import argparse
 from subprocess import check_output, CalledProcessError
+from opendbc.car.uds import UdsClient, SESSION_TYPE, DTC_REPORT_TYPE, DTC_STATUS_MASK_TYPE, get_dtc_num_as_str, get_dtc_status_names
 from panda import Panda
-from panda.python.uds import UdsClient, SESSION_TYPE, DTC_REPORT_TYPE, DTC_STATUS_MASK_TYPE
-from panda.python.uds import get_dtc_num_as_str, get_dtc_status_names
 
 parser = argparse.ArgumentParser(description="read DTC status")
 parser.add_argument("addr", type=lambda x: int(x,0))

selfdrive/debug/touch_replay.py

@@ -30,7 +30,7 @@ if __name__ == '__main__':
         elif event.type == 3 and event.code == 57 and event.value == -1:
           fingers[current_slot] = [-1, -1]
         elif event.type == 3 and event.code == 53:
-          fingers[current_slot][1] = h - (h - event.value)
+          fingers[current_slot][1] = event.value
           if fingers[current_slot][0] != -1:
             touch_points.append(fingers[current_slot].copy())
         elif event.type == 3 and event.code == 54:

selfdrive/debug/uiview.py

@@ -3,14 +3,19 @@ import time
 
 from cereal import car, log, messaging
 from openpilot.common.params import Params
-from openpilot.system.manager.process_config import managed_processes
+from openpilot.system.manager.process_config import managed_processes, is_snpe_model
 from openpilot.system.hardware import HARDWARE
 
 if __name__ == "__main__":
   CP = car.CarParams(notCar=True, wheelbase=1, steerRatio=10)
-  Params().put("CarParams", CP.to_bytes())
+  params = Params()
+  params.put("CarParams", CP.to_bytes())
+  if use_snpe_modeld := is_snpe_model(False, params, CP):
+    print("Using SNPE modeld")
+  HARDWARE.set_power_save(False)
 
-  procs = ['camerad', 'ui', 'modeld', 'calibrationd', 'plannerd', 'dmonitoringmodeld', 'dmonitoringd']
+  procs = ['camerad', 'ui', 'calibrationd', 'plannerd', 'dmonitoringmodeld', 'dmonitoringd']
+  procs += ["modeld_snpe" if use_snpe_modeld else "modeld"]
   for p in procs:
     managed_processes[p].start()
 

selfdrive/debug/vw_mqb_config.py

@@ -3,9 +3,9 @@
 import argparse
 import struct
 from enum import IntEnum
-from panda import Panda
-from panda.python.uds import UdsClient, MessageTimeoutError, NegativeResponseError, SESSION_TYPE,\
+from opendbc.car.uds import UdsClient, MessageTimeoutError, NegativeResponseError, SESSION_TYPE,\
   DATA_IDENTIFIER_TYPE, ACCESS_TYPE
+from panda import Panda
 from datetime import date
 
 # TODO: extend UDS library to allow custom/vendor-defined data identifiers without ignoring type checks

selfdrive/locationd/paramsd.py

@@ -8,7 +8,6 @@ import cereal.messaging as messaging
 from cereal import car, log
 from openpilot.common.params import Params
 from openpilot.common.realtime import config_realtime_process, DT_MDL
-from openpilot.common.numpy_fast import clip
 from openpilot.selfdrive.locationd.models.car_kf import CarKalman, ObservationKind, States
 from openpilot.selfdrive.locationd.models.constants import GENERATED_DIR
 from openpilot.selfdrive.locationd.helpers import PoseCalibrator, Pose
@@ -66,7 +65,7 @@ class ParamsLearner:
         # This is done to bound the road roll estimate when localizer values are invalid
         roll = 0.0
         roll_std = np.radians(10.0)
-      self.roll = clip(roll, self.roll - ROLL_MAX_DELTA, self.roll + ROLL_MAX_DELTA)
+      self.roll = np.clip(roll, self.roll - ROLL_MAX_DELTA, self.roll + ROLL_MAX_DELTA)
 
       yaw_rate_valid = msg.angularVelocityDevice.valid and self.calibrator.calib_valid
       yaw_rate_valid = yaw_rate_valid and 0 < self.yaw_rate_std < 10  # rad/s
@@ -203,11 +202,11 @@ def main():
         learner = ParamsLearner(CP, CP.steerRatio, 1.0, 0.0)
         x = learner.kf.x
 
-      angle_offset_average = clip(math.degrees(x[States.ANGLE_OFFSET].item()),
+      angle_offset_average = np.clip(math.degrees(x[States.ANGLE_OFFSET].item()),
                                   angle_offset_average - MAX_ANGLE_OFFSET_DELTA, angle_offset_average + MAX_ANGLE_OFFSET_DELTA)
-      angle_offset = clip(math.degrees(x[States.ANGLE_OFFSET].item() + x[States.ANGLE_OFFSET_FAST].item()),
+      angle_offset = np.clip(math.degrees(x[States.ANGLE_OFFSET].item() + x[States.ANGLE_OFFSET_FAST].item()),
                           angle_offset - MAX_ANGLE_OFFSET_DELTA, angle_offset + MAX_ANGLE_OFFSET_DELTA)
-      roll = clip(float(x[States.ROAD_ROLL].item()), roll - ROLL_MAX_DELTA, roll + ROLL_MAX_DELTA)
+      roll = np.clip(float(x[States.ROAD_ROLL].item()), roll - ROLL_MAX_DELTA, roll + ROLL_MAX_DELTA)
       roll_std = float(P[States.ROAD_ROLL].item())
       if learner.active and learner.speed > LOW_ACTIVE_SPEED:
         # Account for the opposite signs of the yaw rates
@@ -226,9 +225,9 @@ def main():
       liveParameters.sensorValid = sensors_valid
       liveParameters.steerRatio = float(x[States.STEER_RATIO].item())
       liveParameters.stiffnessFactor = float(x[States.STIFFNESS].item())
-      liveParameters.roll = roll
-      liveParameters.angleOffsetAverageDeg = angle_offset_average
-      liveParameters.angleOffsetDeg = angle_offset
+      liveParameters.roll = float(roll)
+      liveParameters.angleOffsetAverageDeg = float(angle_offset_average)
+      liveParameters.angleOffsetDeg = float(angle_offset)
       liveParameters.valid = all((
         avg_offset_valid,
         total_offset_valid,

selfdrive/modeld/SConscript

@@ -26,10 +26,6 @@ for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transfor
     xenv['CXXFLAGS'].append(f'-D{pathdef}_PATH=\\"{File(fn).abspath}\\"')
 
 # Compile cython
-snpe_rpath_qcom = "/data/pythonpath/third_party/snpe/larch64"
-snpe_rpath_pc = f"{Dir('#').abspath}/third_party/snpe/x86_64-linux-clang"
-snpe_rpath = lenvCython['RPATH'] + [snpe_rpath_qcom if arch == "larch64" else snpe_rpath_pc]
-
 cython_libs = envCython["LIBS"] + libs
 commonmodel_lib = lenv.Library('commonmodel', common_src)
 lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks)

selfdrive/modeld/__init__.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+MODEL_PATH = Path(__file__).parent / 'models/supercombo.onnx'
+MODEL_PKL_PATH = Path(__file__).parent / 'models/supercombo_tinygrad.pkl'
+METADATA_PATH = Path(__file__).parent / 'models/supercombo_metadata.pkl'

selfdrive/modeld/constants.py

@@ -16,7 +16,6 @@ class ModelConstants:
   MODEL_FREQ = 20
   FEATURE_LEN = 512
   FULL_HISTORY_BUFFER_LEN = 99
-  HISTORY_BUFFER_LEN = 24
   DESIRE_LEN = 8
   TRAFFIC_CONVENTION_LEN = 2
   LAT_PLANNER_STATE_LEN = 4
@@ -73,14 +72,13 @@ class Plan:
 class Meta:
   ENGAGED = slice(0, 1)
   # next 2, 4, 6, 8, 10 seconds
-  GAS_DISENGAGE = slice(1, 31, 6)
-  BRAKE_DISENGAGE = slice(2, 31, 6)
-  STEER_OVERRIDE = slice(3, 31, 6)
-  HARD_BRAKE_3 = slice(4, 31, 6)
-  HARD_BRAKE_4 = slice(5, 31, 6)
-  HARD_BRAKE_5 = slice(6, 31, 6)
+  GAS_DISENGAGE = slice(1, 36, 7)
+  BRAKE_DISENGAGE = slice(2, 36, 7)
+  STEER_OVERRIDE = slice(3, 36, 7)
+  HARD_BRAKE_3 = slice(4, 36, 7)
+  HARD_BRAKE_4 = slice(5, 36, 7)
+  HARD_BRAKE_5 = slice(6, 36, 7)
+  GAS_PRESS = slice(7, 36, 7)
   # next 0, 2, 4, 6, 8, 10 seconds
-  GAS_PRESS = slice(31, 55, 4)
-  BRAKE_PRESS = slice(32, 55, 4)
-  LEFT_BLINKER = slice(33, 55, 4)
-  RIGHT_BLINKER = slice(34, 55, 4)
+  LEFT_BLINKER = slice(36, 48, 2)
+  RIGHT_BLINKER = slice(37, 48, 2)

selfdrive/modeld/fill_model_msg.py

@@ -1,24 +1,35 @@
 import os
+
 import capnp
 import numpy as np
-from cereal import log
-from openpilot.selfdrive.modeld.constants import ModelConstants, Plan, Meta
 from openpilot.selfdrive.controls.lib.drive_helpers import MIN_SPEED
+from openpilot.selfdrive.modeld.constants import ModelConstants, Plan
+
+from cereal import log
 
 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 
 ConfidenceClass = log.ModelDataV2.ConfidenceClass
 
+
 def curv_from_psis(psi_target, psi_rate, vego, delay):
   vego = np.clip(vego, MIN_SPEED, np.inf)
   curv_from_psi = psi_target / (vego * delay)  # epsilon to prevent divide-by-zero
-  return 2*curv_from_psi - psi_rate / vego
+  return 2 * curv_from_psi - psi_rate / vego
+
 
 def get_curvature_from_plan(plan, vego, delay):
   psi_target = np.interp(delay, ModelConstants.T_IDXS, plan[:, Plan.T_FROM_CURRENT_EULER][:, 2])
   psi_rate = plan[:, Plan.ORIENTATION_RATE][0, 2]
   return curv_from_psis(psi_target, psi_rate, vego, delay)
 
+
+def get_curvature_from_output(output, vego, delay):
+  if desired_curv := output.get('desired_curvature'):  # If the model outputs the desired curvature, use that directly
+    return float(desired_curv[0, 0])
+
+  return float(get_curvature_from_plan(output['plan'][0], vego, delay))
+
 class PublishState:
   def __init__(self):
     self.disengage_buffer = np.zeros(ModelConstants.CONFIDENCE_BUFFER_LEN*ModelConstants.DISENGAGE_WIDTH, dtype=np.float32)
@@ -69,13 +80,13 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
                    net_output_data: dict[str, np.ndarray], v_ego: float, delay: float,
                    publish_state: PublishState, vipc_frame_id: int, vipc_frame_id_extra: int,
                    frame_id: int, frame_drop: float, timestamp_eof: int, model_execution_time: float,
-                   valid: bool) -> None:
+                   valid: bool, model_meta) -> None:
   frame_age = frame_id - vipc_frame_id if frame_id > vipc_frame_id else 0
   frame_drop_perc = frame_drop * 100
   extended_msg.valid = valid
   base_msg.valid = valid
 
-  desired_curv = float(get_curvature_from_plan(net_output_data['plan'][0], v_ego, delay))
+  desired_curv = float(get_curvature_from_output(net_output_data, v_ego, delay))
 
   driving_model_data = base_msg.drivingModelData
 
@@ -171,23 +182,25 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   meta = modelV2.meta
   meta.desireState = net_output_data['desire_state'][0].reshape(-1).tolist()
   meta.desirePrediction = net_output_data['desire_pred'][0].reshape(-1).tolist()
-  meta.engagedProb = net_output_data['meta'][0,Meta.ENGAGED].item()
+  meta.engagedProb = net_output_data['meta'][0,model_meta.ENGAGED].item()
   meta.init('disengagePredictions')
   disengage_predictions = meta.disengagePredictions
   disengage_predictions.t = ModelConstants.META_T_IDXS
-  disengage_predictions.brakeDisengageProbs = net_output_data['meta'][0,Meta.BRAKE_DISENGAGE].tolist()
-  disengage_predictions.gasDisengageProbs = net_output_data['meta'][0,Meta.GAS_DISENGAGE].tolist()
-  disengage_predictions.steerOverrideProbs = net_output_data['meta'][0,Meta.STEER_OVERRIDE].tolist()
-  disengage_predictions.brake3MetersPerSecondSquaredProbs = net_output_data['meta'][0,Meta.HARD_BRAKE_3].tolist()
-  disengage_predictions.brake4MetersPerSecondSquaredProbs = net_output_data['meta'][0,Meta.HARD_BRAKE_4].tolist()
-  disengage_predictions.brake5MetersPerSecondSquaredProbs = net_output_data['meta'][0,Meta.HARD_BRAKE_5].tolist()
-  disengage_predictions.gasPressProbs = net_output_data['meta'][0,Meta.GAS_PRESS].tolist()
-  disengage_predictions.brakePressProbs = net_output_data['meta'][0,Meta.BRAKE_PRESS].tolist()
+  disengage_predictions.brakeDisengageProbs = net_output_data['meta'][0,model_meta.BRAKE_DISENGAGE].tolist()
+  disengage_predictions.gasDisengageProbs = net_output_data['meta'][0,model_meta.GAS_DISENGAGE].tolist()
+  disengage_predictions.steerOverrideProbs = net_output_data['meta'][0,model_meta.STEER_OVERRIDE].tolist()
+  disengage_predictions.brake3MetersPerSecondSquaredProbs = net_output_data['meta'][0,model_meta.HARD_BRAKE_3].tolist()
+  disengage_predictions.brake4MetersPerSecondSquaredProbs = net_output_data['meta'][0,model_meta.HARD_BRAKE_4].tolist()
+  disengage_predictions.brake5MetersPerSecondSquaredProbs = net_output_data['meta'][0,model_meta.HARD_BRAKE_5].tolist()
+
+  if hasattr(model_meta, 'GAS_PRESS') and hasattr(model_meta, 'BRAKE_PRESS'):
+    disengage_predictions.gasPressProbs = net_output_data['meta'][0,model_meta.GAS_PRESS].tolist()
+    disengage_predictions.brakePressProbs = net_output_data['meta'][0,model_meta.BRAKE_PRESS].tolist()
 
   publish_state.prev_brake_5ms2_probs[:-1] = publish_state.prev_brake_5ms2_probs[1:]
-  publish_state.prev_brake_5ms2_probs[-1] = net_output_data['meta'][0,Meta.HARD_BRAKE_5][0]
+  publish_state.prev_brake_5ms2_probs[-1] = net_output_data['meta'][0,model_meta.HARD_BRAKE_5][0]
   publish_state.prev_brake_3ms2_probs[:-1] = publish_state.prev_brake_3ms2_probs[1:]
-  publish_state.prev_brake_3ms2_probs[-1] = net_output_data['meta'][0,Meta.HARD_BRAKE_3][0]
+  publish_state.prev_brake_3ms2_probs[-1] = net_output_data['meta'][0,model_meta.HARD_BRAKE_3][0]
   hard_brake_predicted = (publish_state.prev_brake_5ms2_probs > ModelConstants.FCW_THRESHOLDS_5MS2).all() and \
     (publish_state.prev_brake_3ms2_probs > ModelConstants.FCW_THRESHOLDS_3MS2).all()
   meta.hardBrakePredicted = hard_brake_predicted.item()
@@ -195,9 +208,9 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   # confidence
   if vipc_frame_id % (2*ModelConstants.MODEL_FREQ) == 0:
     # any disengage prob
-    brake_disengage_probs = net_output_data['meta'][0,Meta.BRAKE_DISENGAGE]
-    gas_disengage_probs = net_output_data['meta'][0,Meta.GAS_DISENGAGE]
-    steer_override_probs = net_output_data['meta'][0,Meta.STEER_OVERRIDE]
+    brake_disengage_probs = net_output_data['meta'][0,model_meta.BRAKE_DISENGAGE]
+    gas_disengage_probs = net_output_data['meta'][0,model_meta.GAS_DISENGAGE]
+    steer_override_probs = net_output_data['meta'][0,model_meta.STEER_OVERRIDE]
     any_disengage_probs = 1-((1-brake_disengage_probs)*(1-gas_disengage_probs)*(1-steer_override_probs))
     # independent disengage prob for each 2s slice
     ind_disengage_probs = np.r_[any_disengage_probs[0], np.diff(any_disengage_probs) / (1 - any_disengage_probs[:-1])]

selfdrive/modeld/modeld.py

@@ -1,21 +1,13 @@
 #!/usr/bin/env python3
-import os
 from openpilot.system.hardware import TICI
 
+from openpilot.sunnypilot.modeld_v2.model_runner import ONNXRunner, TinygradRunner
+
 #
-if TICI:
-  from tinygrad.tensor import Tensor
-  from tinygrad.dtype import dtypes
-  from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from_opencl_address
-  os.environ['QCOM'] = '1'
-else:
-  from openpilot.selfdrive.modeld.runners.ort_helpers import make_onnx_cpu_runner
 import time
-import pickle
 import numpy as np
 import cereal.messaging as messaging
 from cereal import car, log
-from pathlib import Path
 from setproctitle import setproctitle
 from cereal.messaging import PubMaster, SubMaster
 from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
@@ -33,13 +25,10 @@ from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_
 from openpilot.selfdrive.modeld.constants import ModelConstants
 from openpilot.selfdrive.modeld.models.commonmodel_pyx import DrivingModelFrame, CLContext
 
+from sunnypilot.modeld_v2.meta_helper import load_meta_constants
 
 PROCESS_NAME = "selfdrive.modeld.modeld"
-SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 
-MODEL_PATH = Path(__file__).parent / 'models/supercombo.onnx'
-MODEL_PKL_PATH = Path(__file__).parent / 'models/supercombo_tinygrad.pkl'
-METADATA_PATH = Path(__file__).parent / 'models/supercombo_metadata.pkl'
 
 class FrameMeta:
   frame_id: int = 0
@@ -57,79 +46,91 @@ class ModelState:
   prev_desire: np.ndarray  # for tracking the rising edge of the pulse
 
   def __init__(self, context: CLContext):
-    self.frames = {'input_imgs': DrivingModelFrame(context), 'big_input_imgs': DrivingModelFrame(context)}
+    try:
+      self.model_runner = TinygradRunner() if TICI else ONNXRunner()
+    except Exception as e:
+      cloudlog.exception(f"Failed to initialize model runner: {str(e)}")
+
+    buffer_length = 5 if self.model_runner.is_20hz else 2
+    self.frames = {'input_imgs': DrivingModelFrame(context, buffer_length), 'big_input_imgs': DrivingModelFrame(context, buffer_length)}
     self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)
-    self.full_features_20Hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN, ModelConstants.FEATURE_LEN), dtype=np.float32)
-    self.desire_20Hz =  np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN + 1, ModelConstants.DESIRE_LEN), dtype=np.float32)
+    if self.model_runner.is_20hz:
+      self.full_features_20Hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN, ModelConstants.FEATURE_LEN), dtype=np.float32)
+      self.desire_20Hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN + 1, ModelConstants.DESIRE_LEN), dtype=np.float32)
+    # Initialize model runner
 
     # img buffers are managed in openCL transform code
-    self.numpy_inputs = {
-      'desire': np.zeros((1, (ModelConstants.HISTORY_BUFFER_LEN+1), ModelConstants.DESIRE_LEN), dtype=np.float32),
-      'traffic_convention': np.zeros((1, ModelConstants.TRAFFIC_CONVENTION_LEN), dtype=np.float32),
-      'features_buffer': np.zeros((1, ModelConstants.HISTORY_BUFFER_LEN,  ModelConstants.FEATURE_LEN), dtype=np.float32),
-    }
+    self.numpy_inputs = {}
 
-    with open(METADATA_PATH, 'rb') as f:
-      model_metadata = pickle.load(f)
-    self.input_shapes =  model_metadata['input_shapes']
+    for key, shape in self.model_runner.input_shapes.items():
+      if key not in self.frames: # Managed by opencl
+        self.numpy_inputs[key] = np.zeros(shape, dtype=np.float32)
 
-    self.output_slices = model_metadata['output_slices']
-    net_output_size = model_metadata['output_shapes']['outputs'][1]
-    self.output = np.zeros(net_output_size, dtype=np.float32)
     self.parser = Parser()
 
-    if TICI:
-      self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()}
-      with open(MODEL_PKL_PATH, "rb") as f:
-        self.model_run = pickle.load(f)
-    else:
-      self.onnx_cpu_runner = make_onnx_cpu_runner(MODEL_PATH)
+    if self.model_runner.is_20hz:
+      net_output_size = self.model_runner.model_metadata['output_shapes']['outputs'][1]
+      self.output = np.zeros(net_output_size, dtype=np.float32)
 
-  def slice_outputs(self, model_outputs: np.ndarray) -> dict[str, np.ndarray]:
-    parsed_model_outputs = {k: model_outputs[np.newaxis, v] for k,v in self.output_slices.items()}
-    if SEND_RAW_PRED:
-      parsed_model_outputs['raw_pred'] = model_outputs.copy()
-    return parsed_model_outputs
+      num_elements = self.numpy_inputs['features_buffer'].shape[1]
+      step_size = int(-100 / num_elements)
+      self.full_features_20Hz_idxs = np.arange(step_size, step_size * (num_elements + 1), step_size)[::-1]
+      self.desire_reshape_dims = (self.numpy_inputs['desire'].shape[0], self.numpy_inputs['desire'].shape[1], -1, self.numpy_inputs['desire'].shape[2])
 
   def run(self, buf: VisionBuf, wbuf: VisionBuf, transform: np.ndarray, transform_wide: np.ndarray,
-                inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None:
+          inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None:
     # Model decides when action is completed, so desire input is just a pulse triggered on rising edge
     inputs['desire'][0] = 0
     new_desire = np.where(inputs['desire'] - self.prev_desire > .99, inputs['desire'], 0)
     self.prev_desire[:] = inputs['desire']
 
-    self.desire_20Hz[:-1] = self.desire_20Hz[1:]
-    self.desire_20Hz[-1] = new_desire
-    self.numpy_inputs['desire'][:] = self.desire_20Hz.reshape((1,25,4,-1)).max(axis=2)
+    if self.model_runner.is_20hz:
+      self.desire_20Hz[:-1] = self.desire_20Hz[1:]
+      self.desire_20Hz[-1] = new_desire
+      self.numpy_inputs['desire'][:] = self.desire_20Hz.reshape(self.desire_reshape_dims).max(axis=2)
+    else:
+      length = inputs['desire'].shape[0]
+      self.numpy_inputs['desire'][0, :-1] = self.numpy_inputs['desire'][0, 1:]
+      self.numpy_inputs['desire'][0, -1, :length] = new_desire[:length]
+
+    for key in self.numpy_inputs:
+      if key in inputs and key not in ['desire']:
+        self.numpy_inputs[key][:] = inputs[key]
 
-    self.numpy_inputs['traffic_convention'][:] = inputs['traffic_convention']
     imgs_cl = {'input_imgs': self.frames['input_imgs'].prepare(buf, transform.flatten()),
                'big_input_imgs': self.frames['big_input_imgs'].prepare(wbuf, transform_wide.flatten())}
 
-    if TICI:
-      # The imgs tensors are backed by opencl memory, only need init once
-      for key in imgs_cl:
-        if key not in self.tensor_inputs:
-          self.tensor_inputs[key] = qcom_tensor_from_opencl_address(imgs_cl[key].mem_address, self.input_shapes[key], dtype=dtypes.uint8)
-    else:
-      for key in imgs_cl:
-        self.numpy_inputs[key] = self.frames[key].buffer_from_cl(imgs_cl[key]).reshape(self.input_shapes[key])
+    # Prepare inputs using the model runner
+    self.model_runner.prepare_inputs(imgs_cl, self.numpy_inputs, self.frames)
 
     if prepare_only:
       return None
 
-    if TICI:
-      self.output = self.model_run(**self.tensor_inputs).numpy().flatten()
+    # Run model inference
+    self.output = self.model_runner.run_model()
+    outputs = self.parser.parse_outputs(self.model_runner.slice_outputs(self.output))
+
+    if self.model_runner.is_20hz:
+      self.full_features_20Hz[:-1] = self.full_features_20Hz[1:]
+      self.full_features_20Hz[-1] = outputs['hidden_state'][0, :]
+      self.numpy_inputs['features_buffer'][:] = self.full_features_20Hz[self.full_features_20Hz_idxs]
     else:
-      self.output = self.onnx_cpu_runner.run(None, self.numpy_inputs)[0].flatten()
+      feature_len = outputs['hidden_state'].shape[1]
+      self.numpy_inputs['features_buffer'][0, :-1] = self.numpy_inputs['features_buffer'][0, 1:]
+      self.numpy_inputs['features_buffer'][0, -1, :feature_len] = outputs['hidden_state'][0, :feature_len]
 
-    outputs = self.parser.parse_outputs(self.slice_outputs(self.output))
+    if "desired_curvature" in outputs:
+      input_name_prev = None
 
-    self.full_features_20Hz[:-1] = self.full_features_20Hz[1:]
-    self.full_features_20Hz[-1] = outputs['hidden_state'][0, :]
+      if "prev_desired_curvs" in self.numpy_inputs.keys():
+        input_name_prev = 'prev_desired_curvs'
+      elif "prev_desired_curv" in self.numpy_inputs.keys():
+        input_name_prev = 'prev_desired_curv'
 
-    idxs = np.arange(-4,-100,-4)[::-1]
-    self.numpy_inputs['features_buffer'][:] = self.full_features_20Hz[idxs]
+      if input_name_prev is not None:
+        length = outputs['desired_curvature'][0].size
+        self.numpy_inputs[input_name_prev][0, :-length, 0] = self.numpy_inputs[input_name_prev][0, length:, 0]
+        self.numpy_inputs[input_name_prev][0, -length:, 0] = outputs['desired_curvature'][0]
     return outputs
 
 
@@ -270,7 +271,10 @@ def main(demo=False):
     inputs:dict[str, np.ndarray] = {
       'desire': vec_desire,
       'traffic_convention': traffic_convention,
-      }
+    }
+
+    if "lateral_control_params" in model.numpy_inputs.keys():
+      inputs['lateral_control_params'] = np.array([v_ego, steer_delay], dtype=np.float32)
 
     mt1 = time.perf_counter()
     model_output = model.run(buf_main, buf_extra, model_transform_main, model_transform_extra, inputs, prepare_only)
@@ -283,7 +287,7 @@ def main(demo=False):
       posenet_send = messaging.new_message('cameraOdometry')
       fill_model_msg(drivingdata_send, modelv2_send, model_output, v_ego, steer_delay,
                      publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id,
-                     frame_drop_ratio, meta_main.timestamp_eof, model_execution_time, live_calib_seen)
+                     frame_drop_ratio, meta_main.timestamp_eof, model_execution_time, live_calib_seen, load_meta_constants())
 
       desire_state = modelv2_send.modelV2.meta.desireState
       l_lane_change_prob = desire_state[log.Desire.laneChangeLeft]

selfdrive/modeld/models/commonmodel.cc

@@ -5,13 +5,14 @@
 
 #include "common/clutil.h"
 
-DrivingModelFrame::DrivingModelFrame(cl_device_id device_id, cl_context context) : ModelFrame(device_id, context) {
+DrivingModelFrame::DrivingModelFrame(cl_device_id device_id, cl_context context, uint8_t buffer_length) : ModelFrame(device_id, context), buffer_length(buffer_length) {
   input_frames = std::make_unique<uint8_t[]>(buf_size);
   input_frames_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, buf_size, NULL, &err));
-  img_buffer_20hz_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, 5*frame_size_bytes, NULL, &err));
-  region.origin = 4 * frame_size_bytes;
+  img_buffer_20hz_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, buffer_length*frame_size_bytes, NULL, &err));
+  region.origin = (buffer_length - 1)  * frame_size_bytes;
   region.size = frame_size_bytes;
   last_img_cl = CL_CHECK_ERR(clCreateSubBuffer(img_buffer_20hz_cl, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err));
+  // printf("Buffer length: %d, region origin: %lu, region size: %lu\n", buffer_length, region.origin, region.size);
 
   loadyuv_init(&loadyuv, context, device_id, MODEL_WIDTH, MODEL_HEIGHT);
   init_transform(device_id, context, MODEL_WIDTH, MODEL_HEIGHT);
@@ -20,7 +21,7 @@ DrivingModelFrame::DrivingModelFrame(cl_device_id device_id, cl_context context)
 cl_mem* DrivingModelFrame::prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection) {
   run_transform(yuv_cl, MODEL_WIDTH, MODEL_HEIGHT, frame_width, frame_height, frame_stride, frame_uv_offset, projection);
 
-  for (int i = 0; i < 4; i++) {
+  for (int i = 0; i < (buffer_length - 1); i++) {
     CL_CHECK(clEnqueueCopyBuffer(q, img_buffer_20hz_cl, img_buffer_20hz_cl, (i+1)*frame_size_bytes, i*frame_size_bytes, frame_size_bytes, 0, nullptr, nullptr));
   }
   loadyuv_queue(&loadyuv, q, y_cl, u_cl, v_cl, last_img_cl);

selfdrive/modeld/models/commonmodel.h

@@ -64,7 +64,7 @@ protected:
 
 class DrivingModelFrame : public ModelFrame {
 public:
-  DrivingModelFrame(cl_device_id device_id, cl_context context);
+  DrivingModelFrame(cl_device_id device_id, cl_context context, uint8_t buffer_length);
   ~DrivingModelFrame();
   cl_mem* prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection);
 
@@ -73,6 +73,7 @@ public:
   const int MODEL_FRAME_SIZE = MODEL_WIDTH * MODEL_HEIGHT * 3 / 2;
   const int buf_size = MODEL_FRAME_SIZE * 2;
   const size_t frame_size_bytes = MODEL_FRAME_SIZE * sizeof(uint8_t);
+  const uint8_t buffer_length;
 
 private:
   LoadYUVState loadyuv;

selfdrive/modeld/models/commonmodel.pxd

@@ -10,6 +10,7 @@ cdef extern from "common/clutil.h":
   cdef unsigned long CL_DEVICE_TYPE_DEFAULT
   cl_device_id cl_get_device_id(unsigned long)
   cl_context cl_create_context(cl_device_id)
+  void cl_release_context(cl_context)
 
 cdef extern from "selfdrive/modeld/models/commonmodel.h":
   cppclass ModelFrame:
@@ -19,7 +20,7 @@ cdef extern from "selfdrive/modeld/models/commonmodel.h":
 
   cppclass DrivingModelFrame:
     int buf_size
-    DrivingModelFrame(cl_device_id, cl_context)
+    DrivingModelFrame(cl_device_id, cl_context, unsigned char)
 
   cppclass MonitoringModelFrame:
     int buf_size

selfdrive/modeld/models/commonmodel_pyx.pyx

@@ -4,11 +4,11 @@
 import numpy as np
 cimport numpy as cnp
 from libc.string cimport memcpy
-from libc.stdint cimport uintptr_t
+from libc.stdint cimport uintptr_t, uint8_t
 
 from msgq.visionipc.visionipc cimport cl_mem
 from msgq.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext
-from .commonmodel cimport CL_DEVICE_TYPE_DEFAULT, cl_get_device_id, cl_create_context
+from .commonmodel cimport CL_DEVICE_TYPE_DEFAULT, cl_get_device_id, cl_create_context, cl_release_context
 from .commonmodel cimport mat3, ModelFrame as cppModelFrame, DrivingModelFrame as cppDrivingModelFrame, MonitoringModelFrame as cppMonitoringModelFrame
 
 
@@ -17,6 +17,10 @@ cdef class CLContext(BaseCLContext):
     self.device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT)
     self.context = cl_create_context(self.device_id)
 
+  def __dealloc__(self):
+    if self.context:
+      cl_release_context(self.context)
+
 cdef class CLMem:
   @staticmethod
   cdef create(void * cmem):
@@ -55,8 +59,8 @@ cdef class ModelFrame:
 cdef class DrivingModelFrame(ModelFrame):
   cdef cppDrivingModelFrame * _frame
 
-  def __cinit__(self, CLContext context):
-    self._frame = new cppDrivingModelFrame(context.device_id, context.context)
+  def __cinit__(self, CLContext context, int buffer_length=2):
+    self._frame = new cppDrivingModelFrame(context.device_id, context.context, buffer_length)
     self.frame = <cppModelFrame*>(self._frame)
     self.buf_size = self._frame.buf_size
 

selfdrive/modeld/models/supercombo.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72d3d6f8d3c98f5431ec86be77b6350d7d4f43c25075c0106f1d1e7ec7c77668
-size 49096168
+oid sha256:39786068cae1ed8c0dc34ef80c281dfcc67ed18a50e06b90765c49bcfdbf7db4
+size 51453312

selfdrive/modeld/parse_model_outputs.py

@@ -96,6 +96,8 @@ class Parser:
                    out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH))
     if 'lat_planner_solution' in outs:
       self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N,ModelConstants.LAT_PLANNER_SOLUTION_WIDTH))
+    if 'desired_curvature' in outs:
+      self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,))
     for k in ['lead_prob', 'lane_lines_prob', 'meta']:
       self.parse_binary_crossentropy(k, outs)
     self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,))

selfdrive/modeld/tests/snpe_benchmark/.gitignore

@@ -1 +0,0 @@
-benchmark

selfdrive/modeld/tests/snpe_benchmark/benchmark.cc

@@ -1,192 +0,0 @@
-#include <SNPE/SNPE.hpp>
-#include <SNPE/SNPEBuilder.hpp>
-#include <SNPE/SNPEFactory.hpp>
-#include <DlContainer/IDlContainer.hpp>
-#include <DlSystem/DlError.hpp>
-#include <DlSystem/ITensor.hpp>
-#include <DlSystem/ITensorFactory.hpp>
-#include <iostream>
-#include <fstream>
-#include <sstream>
-
-using namespace std;
-
-int64_t timespecDiff(struct timespec *timeA_p, struct timespec *timeB_p) {
-  return ((timeA_p->tv_sec * 1000000000) + timeA_p->tv_nsec) - ((timeB_p->tv_sec * 1000000000) + timeB_p->tv_nsec);
-}
-
-void PrintErrorStringAndExit() {
-  cout << "ERROR!" << endl;
-  const char* const errStr = zdl::DlSystem::getLastErrorString();
-  std::cerr << errStr << std::endl;
-  std::exit(EXIT_FAILURE);
-}
-
-
-zdl::DlSystem::Runtime_t checkRuntime() {
-  static zdl::DlSystem::Version_t Version = zdl::SNPE::SNPEFactory::getLibraryVersion();
-  static zdl::DlSystem::Runtime_t Runtime;
-  std::cout << "SNPE Version: " << Version.asString().c_str() << std::endl; //Print Version number
-  if (zdl::SNPE::SNPEFactory::isRuntimeAvailable(zdl::DlSystem::Runtime_t::DSP)) {
-    std::cout << "Using DSP runtime" << std::endl;
-    Runtime = zdl::DlSystem::Runtime_t::DSP;
-  } else if (zdl::SNPE::SNPEFactory::isRuntimeAvailable(zdl::DlSystem::Runtime_t::GPU)) {
-    std::cout << "Using GPU runtime" << std::endl;
-    Runtime = zdl::DlSystem::Runtime_t::GPU;
-  } else {
-    std::cout << "Using cpu runtime" << std::endl;
-    Runtime = zdl::DlSystem::Runtime_t::CPU;
-  }
-  return Runtime;
-}
-
-void test(char *filename) {
-  static zdl::DlSystem::Runtime_t runtime = checkRuntime();
-  std::unique_ptr<zdl::DlContainer::IDlContainer> container;
-  container = zdl::DlContainer::IDlContainer::open(filename);
-
-  if (!container) { PrintErrorStringAndExit(); }
-  cout << "start build" << endl;
-  std::unique_ptr<zdl::SNPE::SNPE> snpe;
-  {
-    snpe = NULL;
-    zdl::SNPE::SNPEBuilder snpeBuilder(container.get());
-    snpe = snpeBuilder.setOutputLayers({})
-      .setRuntimeProcessor(runtime)
-      .setUseUserSuppliedBuffers(false)
-      //.setDebugMode(true)
-      .build();
-    if (!snpe) {
-      cout << "ERROR!" << endl;
-      const char* const errStr = zdl::DlSystem::getLastErrorString();
-      std::cerr << errStr << std::endl;
-    }
-    cout << "ran snpeBuilder" << endl;
-  }
-
-  const auto &strList_opt = snpe->getInputTensorNames();
-  if (!strList_opt) throw std::runtime_error("Error obtaining input tensor names");
-
-  cout << "get input tensor names done" << endl;
-  const auto &strList = *strList_opt;
-  static zdl::DlSystem::TensorMap inputTensorMap;
-  static zdl::DlSystem::TensorMap outputTensorMap;
-  vector<std::unique_ptr<zdl::DlSystem::ITensor> > inputs;
-  for (int i = 0; i < strList.size(); i++) {
-    cout << "input name: " << strList.at(i) << endl;
-
-    const auto &inputDims_opt = snpe->getInputDimensions(strList.at(i));
-    const auto &inputShape = *inputDims_opt;
-    inputs.push_back(zdl::SNPE::SNPEFactory::getTensorFactory().createTensor(inputShape));
-    inputTensorMap.add(strList.at(i), inputs[i].get());
-  }
-
-  struct timespec start, end;
-  cout << "**** starting benchmark ****" << endl;
-  for (int i = 0; i < 50; i++) {
-    clock_gettime(CLOCK_MONOTONIC, &start);
-    int err = snpe->execute(inputTensorMap, outputTensorMap);
-    assert(err == true);
-    clock_gettime(CLOCK_MONOTONIC, &end);
-    uint64_t timeElapsed = timespecDiff(&end, &start);
-    printf("time: %f ms\n", timeElapsed*1.0/1e6);
-  }
-}
-
-void get_testframe(int index, std::unique_ptr<zdl::DlSystem::ITensor> &input) {
-  FILE * pFile;
-  string filepath="/data/ipt/quantize_samples/sample_input_"+std::to_string(index);
-  pFile = fopen(filepath.c_str(), "rb");
-  int length = 1*6*160*320*4;
-  float * frame_buffer = new float[length/4]; // 32/8
-  fread(frame_buffer, length, 1, pFile);
-  // std::cout << *(frame_buffer+length/4-1) << std::endl;
-  std::copy(frame_buffer, frame_buffer+(length/4), input->begin());
-  fclose(pFile);
-}
-
-void SaveITensor(const std::string& path, const zdl::DlSystem::ITensor* tensor)
-{
-   std::ofstream os(path, std::ofstream::binary);
-   if (!os)
-   {
-      std::cerr << "Failed to open output file for writing: " << path << "\n";
-      std::exit(EXIT_FAILURE);
-   }
-   for ( auto it = tensor->cbegin(); it != tensor->cend(); ++it )
-   {
-      float f = *it;
-      if (!os.write(reinterpret_cast<char*>(&f), sizeof(float)))
-      {
-         std::cerr << "Failed to write data to: " << path << "\n";
-         std::exit(EXIT_FAILURE);
-      }
-   }
-}
-
-void testrun(char* modelfile) {
-  static zdl::DlSystem::Runtime_t runtime = checkRuntime();
-  std::unique_ptr<zdl::DlContainer::IDlContainer> container;
-  container = zdl::DlContainer::IDlContainer::open(modelfile);
-
-  if (!container) { PrintErrorStringAndExit(); }
-  cout << "start build" << endl;
-  std::unique_ptr<zdl::SNPE::SNPE> snpe;
-  {
-    snpe = NULL;
-    zdl::SNPE::SNPEBuilder snpeBuilder(container.get());
-    snpe = snpeBuilder.setOutputLayers({})
-      .setRuntimeProcessor(runtime)
-      .setUseUserSuppliedBuffers(false)
-      //.setDebugMode(true)
-      .build();
-    if (!snpe) {
-      cout << "ERROR!" << endl;
-      const char* const errStr = zdl::DlSystem::getLastErrorString();
-      std::cerr << errStr << std::endl;
-    }
-    cout << "ran snpeBuilder" << endl;
-  }
-
-  const auto &strList_opt = snpe->getInputTensorNames();
-  if (!strList_opt) throw std::runtime_error("Error obtaining input tensor names");
-  cout << "get input tensor names done" << endl;
-
-  const auto &strList = *strList_opt;
-  static zdl::DlSystem::TensorMap inputTensorMap;
-  static zdl::DlSystem::TensorMap outputTensorMap;
-
-  assert(strList.size() == 1);
-  const auto &inputDims_opt = snpe->getInputDimensions(strList.at(0));
-  const auto &inputShape = *inputDims_opt;
-  std::cout << "winkwink" << std::endl;
-
-  for (int i=0; i<10000; i++) {
-    std::unique_ptr<zdl::DlSystem::ITensor> input;
-    input = zdl::SNPE::SNPEFactory::getTensorFactory().createTensor(inputShape);
-    get_testframe(i, input);
-    snpe->execute(input.get(), outputTensorMap);
-    zdl::DlSystem::StringList tensorNames = outputTensorMap.getTensorNames();
-    std::for_each(tensorNames.begin(), tensorNames.end(), [&](const char* name) {
-      std::ostringstream path;
-      path << "/data/opt/Result_" << std::to_string(i) << ".raw";
-      auto tensorPtr = outputTensorMap.getTensor(name);
-      SaveITensor(path.str(), tensorPtr);
-    });
-  }
-}
-
-int main(int argc, char* argv[]) {
-  if (argc < 2) {
-    printf("usage: %s <filename>\n", argv[0]);
-    return -1;
-  }
-
-  if (argc == 2) {
-    while (true) test(argv[1]);
-  } else if (argc == 3) {
-    testrun(argv[1]);
-  }
-  return 0;
-}
-

selfdrive/modeld/tests/snpe_benchmark/benchmark.sh

@@ -1,4 +0,0 @@
-#!/bin/sh -e
-clang++ -I /data/openpilot/third_party/snpe/include/ -L/data/pythonpath/third_party/snpe/aarch64 -lSNPE benchmark.cc -o benchmark
-export LD_LIBRARY_PATH="/data/pythonpath/third_party/snpe/aarch64/:$HOME/openpilot/third_party/snpe/x86_64/:$LD_LIBRARY_PATH"
-exec ./benchmark $1

selfdrive/monitoring/helpers.py

@@ -1,9 +1,9 @@
 from math import atan2
+import numpy as np
 
 from cereal import car, log
 import cereal.messaging as messaging
 from openpilot.selfdrive.selfdrived.events import Events
-from openpilot.common.numpy_fast import interp
 from openpilot.common.realtime import DT_DMON
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.stat_live import RunningStatFilter
@@ -205,10 +205,10 @@ class DriverMonitoring:
     bp = model_data.meta.disengagePredictions.brakeDisengageProbs[0] # brake disengage prob in next 2s
     k1 = max(-0.00156*((car_speed-16)**2)+0.6, 0.2)
     bp_normal = max(min(bp / k1, 0.5),0)
-    self.pose.cfactor_pitch = interp(bp_normal, [0, 0.5],
+    self.pose.cfactor_pitch = np.interp(bp_normal, [0, 0.5],
                                            [self.settings._POSE_PITCH_THRESHOLD_SLACK,
                                             self.settings._POSE_PITCH_THRESHOLD_STRICT]) / self.settings._POSE_PITCH_THRESHOLD
-    self.pose.cfactor_yaw = interp(bp_normal, [0, 0.5],
+    self.pose.cfactor_yaw = np.interp(bp_normal, [0, 0.5],
                                            [self.settings._POSE_YAW_THRESHOLD_SLACK,
                                             self.settings._POSE_YAW_THRESHOLD_STRICT]) / self.settings._POSE_YAW_THRESHOLD
 

selfdrive/pandad/panda_safety.cc

@@ -81,3 +81,8 @@ void PandaSafety::setSafetyMode(const std::string &params_string) {
     pandas_[i]->set_safety_model(safety_model, safety_param);
   }
 }
+
+bool PandaSafety::getOffroadMode() {
+  auto offroad_mode = params_.getBool("OffroadMode");
+  return offroad_mode;
+}

selfdrive/pandad/pandad.cc

@@ -158,7 +158,6 @@ void fill_panda_state(cereal::PandaState::Builder &ps, cereal::PandaState::Panda
   ps.setIgnitionLine(health.ignition_line_pkt);
   ps.setIgnitionCan(health.ignition_can_pkt);
   ps.setControlsAllowed(health.controls_allowed_pkt);
-  ps.setControlsAllowedLat(health.controls_allowed_lat_pkt);
   ps.setTxBufferOverflow(health.tx_buffer_overflow_pkt);
   ps.setRxBufferOverflow(health.rx_buffer_overflow_pkt);
   ps.setPandaType(hw_type);
@@ -206,7 +205,7 @@ void fill_panda_can_state(cereal::PandaState::PandaCanState::Builder &cs, const
   cs.setCanCoreResetCnt(can_health.can_core_reset_cnt);
 }
 
-std::optional<bool> send_panda_states(PubMaster *pm, const std::vector<Panda *> &pandas, bool spoofing_started) {
+std::optional<bool> send_panda_states(PubMaster *pm, const std::vector<Panda *> &pandas, bool spoofing_started, PandaSafety *panda_safety) {
   bool ignition_local = false;
   const uint32_t pandas_cnt = pandas.size();
 
@@ -254,7 +253,7 @@ std::optional<bool> send_panda_states(PubMaster *pm, const std::vector<Panda *>
       health.ignition_line_pkt = 0;
     }
 
-    ignition_local |= ((health.ignition_line_pkt != 0) || (health.ignition_can_pkt != 0));
+    ignition_local |= ((health.ignition_line_pkt != 0) || (health.ignition_can_pkt != 0)) && !panda_safety->getOffroadMode();
 
     pandaStates.push_back(health);
   }
@@ -341,7 +340,7 @@ void send_peripheral_state(Panda *panda, PubMaster *pm) {
   pm->send("peripheralState", msg);
 }
 
-void process_panda_state(std::vector<Panda *> &pandas, PubMaster *pm, bool spoofing_started) {
+void process_panda_state(std::vector<Panda *> &pandas, PubMaster *pm, bool spoofing_started, PandaSafety *panda_safety) {
   static SubMaster sm({"selfdriveState", "selfdriveStateSP", "carParams"});
 
   std::vector<std::string> connected_serials;
@@ -350,7 +349,7 @@ void process_panda_state(std::vector<Panda *> &pandas, PubMaster *pm, bool spoof
   }
 
   {
-    auto ignition_opt = send_panda_states(pm, pandas, spoofing_started);
+    auto ignition_opt = send_panda_states(pm, pandas, spoofing_started, panda_safety);
     if (!ignition_opt) {
       LOGE("Failed to get ignition_opt");
       return;
@@ -462,7 +461,7 @@ void pandad_run(std::vector<Panda *> &pandas) {
 
     // Process panda state at 10 Hz
     if (rk.frame() % 10 == 0) {
-      process_panda_state(pandas, &pm, spoofing_started);
+      process_panda_state(pandas, &pm, spoofing_started, &panda_safety);
       panda_safety.configureSafetyMode();
     }
 

selfdrive/pandad/pandad.h

@@ -12,6 +12,7 @@ class PandaSafety {
 public:
   PandaSafety(const std::vector<Panda *> &pandas) : pandas_(pandas) {}
   void configureSafetyMode();
+  bool getOffroadMode();
 
 private:
   void updateMultiplexingMode();

selfdrive/selfdrived/alertmanager.py

@@ -6,7 +6,8 @@ from dataclasses import dataclass
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
-from openpilot.selfdrive.selfdrived.events import Alert, EmptyAlert
+from openpilot.selfdrive.selfdrived.events import Alert
+from openpilot.sunnypilot.selfdrive.selfdrived.events_base import EmptyAlert
 
 
 with open(os.path.join(BASEDIR, "selfdrive/selfdrived/alerts_offroad.json")) as f:

selfdrive/selfdrived/alerts_offroad.json

@@ -44,5 +44,9 @@
   "Offroad_Recalibration": {
     "text": "openpilot detected a change in the device's mounting position. Ensure the device is fully seated in the mount and the mount is firmly secured to the windshield.",
     "severity": 0
+  },
+  "OffroadMode_Status": {
+    "text": "sunnypilot is now in Always Offroad mode. sunnypilot won't start until Always Offroad mode is disabled. Go to \"Settings\" -> \"Device\" to exit Always Offroad mode.",
+    "severity": 1
   }
 }

selfdrive/selfdrived/events.py

@@ -1,9 +1,6 @@
 #!/usr/bin/env python3
-import bisect
 import math
 import os
-from enum import IntEnum
-from collections.abc import Callable
 
 from cereal import log, car
 import cereal.messaging as messaging
@@ -12,6 +9,11 @@ from openpilot.common.git import get_short_branch
 from openpilot.common.realtime import DT_CTRL
 from openpilot.selfdrive.locationd.calibrationd import MIN_SPEED_FILTER
 
+from openpilot.sunnypilot.selfdrive.selfdrived.events_base import EventsBase, Priority, ET, Alert, \
+  NoEntryAlert, SoftDisableAlert, UserSoftDisableAlert, ImmediateDisableAlert, EngagementAlert, NormalPermanentAlert, \
+  StartupAlert, AlertCallbackType
+
+
 AlertSize = log.SelfdriveState.AlertSize
 AlertStatus = log.SelfdriveState.AlertStatus
 VisualAlert = car.CarControl.HUDControl.VisualAlert
@@ -19,201 +21,23 @@ AudibleAlert = car.CarControl.HUDControl.AudibleAlert
 EventName = log.OnroadEvent.EventName
 
 
-# Alert priorities
-class Priority(IntEnum):
-  LOWEST = 0
-  LOWER = 1
-  LOW = 2
-  MID = 3
-  HIGH = 4
-  HIGHEST = 5
-
-
-# Event types
-class ET:
-  ENABLE = 'enable'
-  PRE_ENABLE = 'preEnable'
-  OVERRIDE_LATERAL = 'overrideLateral'
-  OVERRIDE_LONGITUDINAL = 'overrideLongitudinal'
-  NO_ENTRY = 'noEntry'
-  WARNING = 'warning'
-  USER_DISABLE = 'userDisable'
-  SOFT_DISABLE = 'softDisable'
-  IMMEDIATE_DISABLE = 'immediateDisable'
-  PERMANENT = 'permanent'
-
-
 # get event name from enum
 EVENT_NAME = {v: k for k, v in EventName.schema.enumerants.items()}
 
 
-class Events:
+class Events(EventsBase):
   def __init__(self):
-    self.events: list[int] = []
-    self.static_events: list[int] = []
+    super().__init__()
     self.event_counters = dict.fromkeys(EVENTS.keys(), 0)
 
-  @property
-  def names(self) -> list[int]:
-    return self.events
+  def get_events_mapping(self) -> dict[int, dict[str, Alert | AlertCallbackType]]:
+    return EVENTS
 
-  def __len__(self) -> int:
-    return len(self.events)
+  def get_event_name(self, event: int):
+    return EVENT_NAME[event]
 
-  def add(self, event_name: int, static: bool=False) -> None:
-    if static:
-      bisect.insort(self.static_events, event_name)
-    bisect.insort(self.events, event_name)
-
-  def clear(self) -> None:
-    self.event_counters = {k: (v + 1 if k in self.events else 0) for k, v in self.event_counters.items()}
-    self.events = self.static_events.copy()
-
-  def contains(self, event_type: str) -> bool:
-    return any(event_type in EVENTS.get(e, {}) for e in self.events)
-
-  def create_alerts(self, event_types: list[str], callback_args=None):
-    if callback_args is None:
-      callback_args = []
-
-    ret = []
-    for e in self.events:
-      types = EVENTS[e].keys()
-      for et in event_types:
-        if et in types:
-          alert = EVENTS[e][et]
-          if not isinstance(alert, Alert):
-            alert = alert(*callback_args)
-
-          if DT_CTRL * (self.event_counters[e] + 1) >= alert.creation_delay:
-            alert.alert_type = f"{EVENT_NAME[e]}/{et}"
-            alert.event_type = et
-            ret.append(alert)
-    return ret
-
-  def add_from_msg(self, events):
-    for e in events:
-      bisect.insort(self.events, e.name.raw)
-
-  def to_msg(self):
-    ret = []
-    for event_name in self.events:
-      event = log.OnroadEvent.new_message()
-      event.name = event_name
-      for event_type in EVENTS.get(event_name, {}):
-        setattr(event, event_type, True)
-      ret.append(event)
-    return ret
-
-  def has(self, event_name: int) -> bool:
-    return event_name in self.events
-
-  def contains_in_list(self, events_list: list[int]) -> bool:
-    return any(event_name in self.events for event_name in events_list)
-
-  def remove(self, event_name: int, static: bool = False) -> None:
-    if static and event_name in self.static_events:
-      self.static_events.remove(event_name)
-
-    if event_name in self.events:
-      self.event_counters[event_name] = self.event_counters[event_name] + 1
-      self.events.remove(event_name)
-
-  def replace(self, prev_event_name: int, cur_event_name: int, static: bool = False) -> None:
-    self.remove(prev_event_name, static)
-    self.add(cur_event_name, static)
-
-
-class Alert:
-  def __init__(self,
-               alert_text_1: str,
-               alert_text_2: str,
-               alert_status: log.SelfdriveState.AlertStatus,
-               alert_size: log.SelfdriveState.AlertSize,
-               priority: Priority,
-               visual_alert: car.CarControl.HUDControl.VisualAlert,
-               audible_alert: car.CarControl.HUDControl.AudibleAlert,
-               duration: float,
-               creation_delay: float = 0.):
-
-    self.alert_text_1 = alert_text_1
-    self.alert_text_2 = alert_text_2
-    self.alert_status = alert_status
-    self.alert_size = alert_size
-    self.priority = priority
-    self.visual_alert = visual_alert
-    self.audible_alert = audible_alert
-
-    self.duration = int(duration / DT_CTRL)
-
-    self.creation_delay = creation_delay
-
-    self.alert_type = ""
-    self.event_type: str | None = None
-
-  def __str__(self) -> str:
-    return f"{self.alert_text_1}/{self.alert_text_2} {self.priority} {self.visual_alert} {self.audible_alert}"
-
-  def __gt__(self, alert2) -> bool:
-    if not isinstance(alert2, Alert):
-      return False
-    return self.priority > alert2.priority
-
-EmptyAlert = Alert("" , "", AlertStatus.normal, AlertSize.none, Priority.LOWEST,
-                   VisualAlert.none, AudibleAlert.none, 0)
-
-class NoEntryAlert(Alert):
-  def __init__(self, alert_text_2: str,
-               alert_text_1: str = "openpilot Unavailable",
-               visual_alert: car.CarControl.HUDControl.VisualAlert=VisualAlert.none):
-    super().__init__(alert_text_1, alert_text_2, AlertStatus.normal,
-                     AlertSize.mid, Priority.LOW, visual_alert,
-                     AudibleAlert.refuse, 3.)
-
-
-class SoftDisableAlert(Alert):
-  def __init__(self, alert_text_2: str):
-    super().__init__("TAKE CONTROL IMMEDIATELY", alert_text_2,
-                     AlertStatus.userPrompt, AlertSize.full,
-                     Priority.MID, VisualAlert.steerRequired,
-                     AudibleAlert.warningSoft, 2.),
-
-
-# less harsh version of SoftDisable, where the condition is user-triggered
-class UserSoftDisableAlert(SoftDisableAlert):
-  def __init__(self, alert_text_2: str):
-    super().__init__(alert_text_2),
-    self.alert_text_1 = "openpilot will disengage"
-
-
-class ImmediateDisableAlert(Alert):
-  def __init__(self, alert_text_2: str):
-    super().__init__("TAKE CONTROL IMMEDIATELY", alert_text_2,
-                     AlertStatus.critical, AlertSize.full,
-                     Priority.HIGHEST, VisualAlert.steerRequired,
-                     AudibleAlert.warningImmediate, 4.),
-
-
-class EngagementAlert(Alert):
-  def __init__(self, audible_alert: car.CarControl.HUDControl.AudibleAlert):
-    super().__init__("", "",
-                     AlertStatus.normal, AlertSize.none,
-                     Priority.MID, VisualAlert.none,
-                     audible_alert, .2),
-
-
-class NormalPermanentAlert(Alert):
-  def __init__(self, alert_text_1: str, alert_text_2: str = "", duration: float = 0.2, priority: Priority = Priority.LOWER, creation_delay: float = 0.):
-    super().__init__(alert_text_1, alert_text_2,
-                     AlertStatus.normal, AlertSize.mid if len(alert_text_2) else AlertSize.small,
-                     priority, VisualAlert.none, AudibleAlert.none, duration, creation_delay=creation_delay),
-
-
-class StartupAlert(Alert):
-  def __init__(self, alert_text_1: str, alert_text_2: str = "Always keep hands on wheel and eyes on road", alert_status=AlertStatus.normal):
-    super().__init__(alert_text_1, alert_text_2,
-                     alert_status, AlertSize.mid,
-                     Priority.LOWER, VisualAlert.none, AudibleAlert.none, 5.),
+  def get_event_msg_type(self):
+    return log.OnroadEvent
 
 
 # ********** helper functions **********
@@ -225,8 +49,6 @@ def get_display_speed(speed_ms: float, metric: bool) -> str:
 
 # ********** alert callback functions **********
 
-AlertCallbackType = Callable[[car.CarParams, car.CarState, messaging.SubMaster, bool, int, log.ControlsState], Alert]
-
 
 def soft_disable_alert(alert_text_2: str) -> AlertCallbackType:
   def func(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality) -> Alert:
@@ -652,8 +474,11 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
   },
 
   EventName.brakeHold: {
-    ET.USER_DISABLE: EngagementAlert(AudibleAlert.disengage),
-    ET.NO_ENTRY: NoEntryAlert("Brake Hold Active"),
+    ET.WARNING: Alert(
+      "Press Resume to Exit Brake Hold",
+      "",
+      AlertStatus.userPrompt, AlertSize.small,
+      Priority.LOW, VisualAlert.none, AudibleAlert.none, .2),
   },
 
   EventName.parkBrake: {
@@ -969,99 +794,6 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
     ET.WARNING: personality_changed_alert,
   },
 
-  # sunnypilot
-  EventName.lkasEnable: {
-    ET.ENABLE: EngagementAlert(AudibleAlert.engage),
-  },
-
-  EventName.lkasDisable: {
-    ET.USER_DISABLE: EngagementAlert(AudibleAlert.disengage),
-  },
-
-  EventName.manualSteeringRequired: {
-    ET.USER_DISABLE: Alert(
-      "Automatic Lane Centering is OFF",
-      "Manual Steering Required",
-      AlertStatus.normal, AlertSize.mid,
-      Priority.LOW, VisualAlert.none, AudibleAlert.disengage, 1.),
-  },
-
-  EventName.manualLongitudinalRequired: {
-    ET.WARNING: Alert(
-      "Smart/Adaptive Cruise Control: OFF",
-      "Manual Speed Control Required",
-      AlertStatus.normal, AlertSize.mid,
-      Priority.LOW, VisualAlert.none, AudibleAlert.none, 1.),
-  },
-
-  EventName.silentLkasEnable: {
-    ET.ENABLE: EngagementAlert(AudibleAlert.none),
-  },
-
-  EventName.silentLkasDisable: {
-    ET.USER_DISABLE: EngagementAlert(AudibleAlert.none),
-  },
-
-  EventName.silentBrakeHold: {
-    ET.USER_DISABLE: EngagementAlert(AudibleAlert.none),
-    ET.NO_ENTRY: NoEntryAlert("Brake Hold Active"),
-  },
-
-  EventName.silentWrongGear: {
-    ET.WARNING: Alert(
-      "",
-      "",
-      AlertStatus.normal, AlertSize.none,
-      Priority.LOWEST, VisualAlert.none, AudibleAlert.none, 0.),
-    ET.NO_ENTRY: Alert(
-      "Gear not D",
-      "openpilot Unavailable",
-      AlertStatus.normal, AlertSize.mid,
-      Priority.LOW, VisualAlert.none, AudibleAlert.none, 0.),
-  },
-
-  EventName.silentReverseGear: {
-    ET.PERMANENT: Alert(
-      "Reverse\nGear",
-      "",
-      AlertStatus.normal, AlertSize.full,
-      Priority.LOWEST, VisualAlert.none, AudibleAlert.none, .2, creation_delay=0.5),
-    ET.NO_ENTRY: NoEntryAlert("Reverse Gear"),
-  },
-
-  EventName.silentDoorOpen: {
-    ET.WARNING: Alert(
-      "",
-      "",
-      AlertStatus.normal, AlertSize.none,
-      Priority.LOWEST, VisualAlert.none, AudibleAlert.none, 0.),
-    ET.NO_ENTRY: NoEntryAlert("Door Open"),
-  },
-
-  EventName.silentSeatbeltNotLatched: {
-    ET.WARNING: Alert(
-      "",
-      "",
-      AlertStatus.normal, AlertSize.none,
-      Priority.LOWEST, VisualAlert.none, AudibleAlert.none, 0.),
-    ET.NO_ENTRY: NoEntryAlert("Seatbelt Unlatched"),
-  },
-
-  EventName.silentParkBrake: {
-    ET.WARNING: Alert(
-      "",
-      "",
-      AlertStatus.normal, AlertSize.none,
-      Priority.LOWEST, VisualAlert.none, AudibleAlert.none, 0.),
-    ET.NO_ENTRY: NoEntryAlert("Parking Brake Engaged"),
-  },
-
-  EventName.controlsMismatchLateral: {
-    ET.IMMEDIATE_DISABLE: ImmediateDisableAlert("Controls Mismatch: Lateral"),
-    ET.NO_ENTRY: NoEntryAlert("Controls Mismatch: Lateral"),
-  },
-
-
 }
 
 

selfdrive/selfdrived/selfdrived.py

@@ -5,7 +5,7 @@ import threading
 
 import cereal.messaging as messaging
 
-from cereal import car, log
+from cereal import car, log, custom
 from msgq.visionipc import VisionIpcClient, VisionStreamType
 from panda import ALTERNATIVE_EXPERIENCE
 
@@ -24,6 +24,9 @@ from openpilot.selfdrive.controls.lib.latcontrol import MIN_LATERAL_CONTROL_SPEE
 from openpilot.system.version import get_build_metadata
 
 from openpilot.sunnypilot.mads.mads import ModularAssistiveDrivingSystem
+from openpilot.sunnypilot.selfdrive.car.car_specific import CarSpecificEventsSP
+from openpilot.sunnypilot.selfdrive.car.cruise_helpers import CruiseHelper
+from openpilot.sunnypilot.selfdrive.selfdrived.events import EventsSP
 
 REPLAY = "REPLAY" in os.environ
 SIMULATION = "SIMULATION" in os.environ
@@ -43,8 +46,8 @@ SafetyModel = car.CarParams.SafetyModel
 IGNORED_SAFETY_MODES = (SafetyModel.silent, SafetyModel.noOutput)
 
 
-class SelfdriveD:
-  def __init__(self, CP=None):
+class SelfdriveD(CruiseHelper):
+  def __init__(self, CP=None, CP_SP=None):
     self.params = Params()
 
     # Ensure the current branch is cached, otherwise the first cycle lags
@@ -57,6 +60,13 @@ class SelfdriveD:
     else:
       self.CP = CP
 
+    if CP_SP is None:
+      cloudlog.info("selfdrived is waiting for CarParamsSP")
+      self.CP_SP = messaging.log_from_bytes(self.params.get("CarParamsSP", block=True), custom.CarParamsSP)
+      cloudlog.info("selfdrived got CarParamsSP")
+    else:
+      self.CP_SP = CP_SP
+
     self.car_events = CarSpecificEvents(self.CP)
     self.disengage_on_accelerator = not (self.CP.alternativeExperience & ALTERNATIVE_EXPERIENCE.DISABLE_DISENGAGE_ON_GAS)
 
@@ -133,14 +143,22 @@ class SelfdriveD:
     elif self.CP.passive:
       self.events.add(EventName.dashcamMode, static=True)
 
+    self.events_sp = EventsSP()
+    self.events_sp_prev = []
+
     self.mads = ModularAssistiveDrivingSystem(self)
-    sock_services = list(self.pm.sock.keys()) + ['selfdriveStateSP']
+    sock_services = list(self.pm.sock.keys()) + ['selfdriveStateSP', 'onroadEventsSP']
     self.pm = messaging.PubMaster(sock_services)
 
+    self.car_events_sp = CarSpecificEventsSP(self.CP, self.params)
+
+    CruiseHelper.__init__(self, self.CP)
+
   def update_events(self, CS):
     """Compute onroadEvents from carState"""
 
     self.events.clear()
+    self.events_sp.clear()
 
     if self.sm['controlsState'].lateralControlState.which() == 'debugState':
       self.events.add(EventName.joystickDebug)
@@ -177,6 +195,9 @@ class SelfdriveD:
       car_events = self.car_events.update(CS, self.CS_prev, self.sm['carControl']).to_msg()
       self.events.add_from_msg(car_events)
 
+      car_events_sp = self.car_events_sp.update().to_msg()
+      self.events_sp.add_from_msg(car_events_sp)
+
       if self.CP.notCar:
         # wait for everything to init first
         if self.sm.frame > int(5. / DT_CTRL) and self.initialized:
@@ -361,12 +382,16 @@ class SelfdriveD:
       if self.sm['modelV2'].frameDropPerc > 20:
         self.events.add(EventName.modeldLagging)
 
+    CruiseHelper.update(self, CS, self.events_sp, self.experimental_mode)
+
     # decrement personality on distance button press
     if self.CP.openpilotLongitudinalControl:
       if any(not be.pressed and be.type == ButtonType.gapAdjustCruise for be in CS.buttonEvents):
-        self.personality = (self.personality - 1) % 3
-        self.params.put_nonblocking('LongitudinalPersonality', str(self.personality))
-        self.events.add(EventName.personalityChanged)
+        if not self.experimental_mode_switched:
+          self.personality = (self.personality - 1) % 3
+          self.params.put_nonblocking('LongitudinalPersonality', str(self.personality))
+          self.events.add(EventName.personalityChanged)
+        self.experimental_mode_switched = False
 
   def data_sample(self):
     car_state = messaging.recv_one(self.car_state_sock)
@@ -423,9 +448,13 @@ class SelfdriveD:
       clear_event_types.add(ET.NO_ENTRY)
 
     pers = LONGITUDINAL_PERSONALITY_MAP[self.personality]
-    alerts = self.events.create_alerts(self.state_machine.current_alert_types, [self.CP, CS, self.sm, self.is_metric,
-                                                                                self.state_machine.soft_disable_timer, pers])
-    self.AM.add_many(self.sm.frame, alerts)
+    callback_args = [self.CP, CS, self.sm, self.is_metric,
+                     self.state_machine.soft_disable_timer, pers]
+
+    alerts = self.events.create_alerts(self.state_machine.current_alert_types, callback_args)
+    alerts_sp = self.events_sp.create_alerts(self.state_machine.current_alert_types, callback_args)
+
+    self.AM.add_many(self.sm.frame, alerts + alerts_sp)
     self.AM.process_alerts(self.sm.frame, clear_event_types)
 
   def publish_selfdriveState(self, CS):
@@ -446,6 +475,7 @@ class SelfdriveD:
     ss.alertStatus = self.AM.current_alert.alert_status
     ss.alertType = self.AM.current_alert.alert_type
     ss.alertSound = self.AM.current_alert.audible_alert
+    ss.alertHudVisual = self.AM.current_alert.visual_alert
 
     self.pm.send('selfdriveState', ss_msg)
 
@@ -469,13 +499,21 @@ class SelfdriveD:
 
     self.pm.send('selfdriveStateSP', ss_sp_msg)
 
+    # onroadEventsSP - logged every second or on change
+    if (self.sm.frame % int(1. / DT_CTRL) == 0) or (self.events_sp.names != self.events_sp_prev):
+      ce_send_sp = messaging.new_message('onroadEventsSP', len(self.events_sp))
+      ce_send_sp.valid = True
+      ce_send_sp.onroadEventsSP = self.events_sp.to_msg()
+      self.pm.send('onroadEventsSP', ce_send_sp)
+    self.events_sp_prev = self.events_sp.names.copy()
+
   def step(self):
     CS = self.data_sample()
     self.update_events(CS)
     if not self.CP.passive and self.initialized:
       self.enabled, self.active = self.state_machine.update(self.events)
     if not self.CP.notCar:
-      self.mads.update(CS, self.sm)
+      self.mads.update(CS)
     self.update_alerts(CS)
 
     self.publish_selfdriveState(CS)
@@ -495,6 +533,7 @@ class SelfdriveD:
       self.personality = self.read_personality_param()
 
       self.mads.read_params()
+      self.car_events_sp.read_params()
       time.sleep(0.1)
 
   def run(self):