Car: Migrate sunnypilot CarParams to its own cereal (#604)

* sp flags * pass CP_SP to card and car interfaces * CP_SP in radar interface * bump opendbc * use dataclass like old times * bump opendbc * write to params for controls * fix test models * fix * need to use copy instead * fix data type * add service * more * fix * Revert "fix" This reverts commit 74723d7fb277907e174c021af14ecbcdb54f1586. * Revert "fix data type" This reverts commit 02355f44df2e9fca06d565b2e30641ff8d62f1a3. * missed * more * no more lagging * Reapply "fix data type" This reverts commit dbf1b8583fe1843db91c6fb5c96cf1fb3be5ef79. * Reapply "fix" This reverts commit 9cbce9968a6bf245087681d5061a39db90873226. * Revert "Reapply "fix"" This reverts commit 1871919b633c6f94627b2574299740c6c3f6738e. * Revert "Reapply "fix data type"" This reverts commit 5e95752fd5e7d754acdaa83855ec8868ece290f2. * no longer * Revert "no longer" This reverts commit 66ee1ba1513b43b0b07d8344e0dfb13752f6872d. * Reapply "Reapply "fix data type"" This reverts commit 670a3843338490fa7fbf23bbe7bf47c261ded9c8. * Reapply "Reapply "fix"" This reverts commit 42f09f955c84391d01866f3d48717382547f8b85. * only for car params sp * rename * fix more test * no need for process replay * pass stock car params to sp set car params * pass stock car params to sp set car params * deprecate CarParams.sunnypilotFlags to CarParamsSP.flags * missed arg * fix tests * tests fixed * need to pass this too * must generate cp_sp! * fix typing * must be initialized prior can comm callback! * no more cancer (@devtekve) * remove more cancer * Refactor `get_non_essential_params_sp` to simplify arguments (#612) * Refactor 'get_non_essential_params_sp' function calls in tests In both `test_latcontrol.py` and `process_replay.py`, simplified the function calls to 'get_non_essential_params_sp'. Removed an unnecessary call to 'get_non_essential_params'. This change makes the code cleaner and more efficient by reducing redundancy in the function calls. This modification also ensures consistency across different code files. * Refactor get_non_essential_params_sp to take car_params. Simplify parameters by modifying `get_non_essential_params_sp` to use `car_params` as input. Adjust related calls in test files and process replay to match the updated method signature. This improves code clarity and reduces redundancy. * bump opendbc * Refactor parameter handling for `get_params_sp`. Removed unnecessary reassignment of `car_params` in calls to `get_params_sp`, ensuring a cleaner and more streamlined code structure. This change improves code clarity and eliminates redundant operations. All relevant assertions and behavior remain unaffected. * bumping opedbc * bump opendbc --------- Co-authored-by: DevTekVE <devtekve@gmail.com>
2026-08-05 08:16:03 +08:00 · 2025-01-22 14:22:32 -05:00
parent 403d77ff3f
commit 3e7240516e
16 changed files with 125 additions and 37 deletions
@@ -133,7 +133,8 @@ struct OnroadEventSP @0xda96579883444c35 {
  }
 }

-struct CustomReserved4 @0x80ae746ee2596b11 {
+struct CarParamsSP @0x80ae746ee2596b11 {
+  flags @0 :UInt32;  # flags for car specific quirks in sunnypilot
 }

 struct CustomReserved5 @0xa5cd762cd951a455 {
@@ -2569,7 +2569,7 @@ struct Event {
    modelManagerSP @108 :Custom.ModelManagerSP;
    longitudinalPlanSP @109 :Custom.LongitudinalPlanSP;
    onroadEventsSP @110 :List(Custom.OnroadEventSP);
-    customReserved4 @111 :Custom.CustomReserved4;
+    carParamsSP @111 :Custom.CarParamsSP;
    customReserved5 @112 :Custom.CustomReserved5;
    customReserved6 @113 :Custom.CustomReserved6;
    customReserved7 @114 :Custom.CustomReserved7;
@@ -79,6 +79,7 @@ _services: dict[str, tuple] = {
  "selfdriveStateSP": (True, 100., 10),
  "longitudinalPlanSP": (True, 20., 10),
  "onroadEventsSP": (True, 1., 1),
+  "carParamsSP": (True, 0.02, 1),

  # debug
  "uiDebug": (True, 0., 1),
@@ -202,6 +202,9 @@ std::unordered_map<std::string, uint32_t> keys = {

    // --- sunnypilot params --- //
    {"ApiCache_DriveStats", PERSISTENT},
+    {"CarParamsSP", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CarParamsSPCache", CLEAR_ON_MANAGER_START},
+    {"CarParamsSPPersistent", PERSISTENT},
    {"EnableGithubRunner", PERSISTENT | BACKUP},
    {"ModelRunnerTypeCache", CLEAR_ON_ONROAD_TRANSITION},
    {"OffroadMode", CLEAR_ON_MANAGER_START},
@@ -5,7 +5,7 @@ import threading

 import cereal.messaging as messaging

-from cereal import car, log
+from cereal import car, log, custom

 from panda import ALTERNATIVE_EXPERIENCE

@@ -21,6 +21,7 @@ from opendbc.car.interfaces import CarInterfaceBase, RadarInterfaceBase
 from openpilot.selfdrive.pandad import can_capnp_to_list, can_list_to_can_capnp
 from openpilot.selfdrive.car.cruise import VCruiseHelper
 from openpilot.selfdrive.car.car_specific import MockCarState
+from openpilot.selfdrive.car.helpers import convert_to_capnp

 from openpilot.sunnypilot.mads.mads import MadsParams
 from openpilot.sunnypilot.selfdrive.car import interfaces
@@ -66,11 +67,13 @@ class Car:
  CI: CarInterfaceBase
  RI: RadarInterfaceBase
  CP: car.CarParams
+  CP_SP: structs.CarParamsSP
+  CP_SP_capnp: custom.CarParamsSP

  def __init__(self, CI=None, RI=None) -> None:
    self.can_sock = messaging.sub_sock('can', timeout=20)
    self.sm = messaging.SubMaster(['pandaStates', 'carControl', 'onroadEvents'])
-    self.pm = messaging.PubMaster(['sendcan', 'carState', 'carParams', 'carOutput', 'liveTracks'])
+    self.pm = messaging.PubMaster(['sendcan', 'carState', 'carParams', 'carOutput', 'liveTracks'] + ['carParamsSP'])

    self.can_rcv_cum_timeout_counter = 0

@@ -102,14 +105,15 @@ class Car:
          cached_params = _cached_params

      self.CI = get_car(*self.can_callbacks, obd_callback(self.params), experimental_long_allowed, num_pandas, cached_params)
-      interfaces.setup_car_interface_sp(self.CI.CP, self.params)
-      self.RI = get_radar_interface(self.CI.CP)
+      interfaces.setup_car_interface_sp(self.CI.CP, self.CI.CP_SP, self.params)
+      self.RI = get_radar_interface(self.CI.CP, self.CI.CP_SP)
      self.CP = self.CI.CP
+      self.CP_SP = self.CI.CP_SP

      # continue onto next fingerprinting step in pandad
      self.params.put_bool("FirmwareQueryDone", True)
    else:
-      self.CI, self.CP = CI, CI.CP
+      self.CI, self.CP, self.CP_SP = CI, CI.CP, CI.CP_SP
      self.RI = RI

    # set alternative experiences from parameters
@@ -120,7 +124,7 @@ class Car:

    # mads
    MadsParams().set_alternative_experience(self.CP)
-    MadsParams().set_car_specific_params(self.CP)
+    MadsParams().set_car_specific_params(self.CP, self.CP_SP)

    # Dynamic Experimental Control
    self.dynamic_experimental_control = self.params.get_bool("DynamicExperimentalControl")
@@ -167,6 +171,14 @@ class Car:
    self.params.put_nonblocking("CarParamsCache", cp_bytes)
    self.params.put_nonblocking("CarParamsPersistent", cp_bytes)

+    # Write CarParamsSP for controls
+    # convert to pycapnp representation for caching and logging
+    self.CP_SP_capnp = convert_to_capnp(self.CP_SP)
+    cp_sp_bytes = self.CP_SP_capnp.to_bytes()
+    self.params.put("CarParamsSP", cp_sp_bytes)
+    self.params.put_nonblocking("CarParamsSPCache", cp_sp_bytes)
+    self.params.put_nonblocking("CarParamsSPPersistent", cp_sp_bytes)
+
    self.mock_carstate = MockCarState()
    self.v_cruise_helper = VCruiseHelper(self.CP)

@@ -245,14 +257,21 @@ class Car:
      tracks_msg.liveTracks = RD
      self.pm.send('liveTracks', tracks_msg)

+    # carParamsSP - logged every 50 seconds (> 1 per segment)
+    if self.sm.frame % int(50. / DT_CTRL) == 0:
+      cp_sp_send = messaging.new_message('carParamsSP')
+      cp_sp_send.valid = True
+      cp_sp_send.carParamsSP = self.CP_SP_capnp
+      self.pm.send('carParamsSP', cp_sp_send)
+
  def controls_update(self, CS: car.CarState, CC: car.CarControl):
    """control update loop, driven by carControl"""

    if not self.initialized_prev:
      # Initialize CarInterface, once controls are ready
      # TODO: this can make us miss at least a few cycles when doing an ECU knockout
-      self.CI.init(self.CP, *self.can_callbacks)
-      interfaces.initialize_car_interface_sp(self.CP, self.params, *self.can_callbacks)
+      self.CI.init(self.CP, self.CP_SP, *self.can_callbacks)
+      interfaces.initialize_car_interface_sp(self.CP, self.CP_SP, self.params, *self.can_callbacks)
      # signal pandad to switch to car safety mode
      self.params.put_bool_nonblocking("ControlsReady", True)

@@ -0,0 +1,46 @@
+import capnp
+from typing import Any
+
+from cereal import custom
+from opendbc.car import structs
+
+_FIELDS = '__dataclass_fields__'  # copy of dataclasses._FIELDS
+
+
+def is_dataclass(obj):
+  """Similar to dataclasses.is_dataclass without instance type check checking"""
+  return hasattr(obj, _FIELDS)
+
+
+def _asdictref_inner(obj) -> dict[str, Any] | Any:
+  if is_dataclass(obj):
+    ret = {}
+    for field in getattr(obj, _FIELDS):  # similar to dataclasses.fields()
+      ret[field] = _asdictref_inner(getattr(obj, field))
+    return ret
+  elif isinstance(obj, (tuple, list)):
+    return type(obj)(_asdictref_inner(v) for v in obj)
+  else:
+    return obj
+
+
+def asdictref(obj) -> dict[str, Any]:
+  """
+  Similar to dataclasses.asdict without recursive type checking and copy.deepcopy
+  Note that the resulting dict will contain references to the original struct as a result
+  """
+  if not is_dataclass(obj):
+    raise TypeError("asdictref() should be called on dataclass instances")
+
+  return _asdictref_inner(obj)
+
+
+def convert_to_capnp(struct: structs.CarParamsSP) -> capnp.lib.capnp._DynamicStructBuilder:
+  struct_dict = asdictref(struct)
+
+  if isinstance(struct, structs.CarParamsSP):
+    struct_capnp = custom.CarParamsSP.new_message(**struct_dict)
+  else:
+    raise ValueError(f"Unsupported struct type: {type(struct)}")
+
+  return struct_capnp
@@ -40,9 +40,12 @@ class TestCarInterfaces:

    car_params = CarInterface.get_params(car_name, args['fingerprints'], args['car_fw'],
                                         experimental_long=args['experimental_long'], docs=False)
+    car_params_sp = CarInterface.get_params_sp(car_params, car_name, args['fingerprints'], args['car_fw'],
+                                                           experimental_long=args['experimental_long'], docs=False)
    car_params = car_params.as_reader()
-    car_interface = CarInterface(car_params, CarController, CarState)
+    car_interface = CarInterface(car_params, car_params_sp, CarController, CarState)
    assert car_params
+    assert car_params_sp
    assert car_interface

    assert car_params.mass > 1
@@ -1,4 +1,5 @@
 import capnp
+import copy
 import os
 import pytest
 import random
@@ -158,7 +159,9 @@ class TestCarModelBase(unittest.TestCase):

    cls.CarInterface, cls.CarController, cls.CarState, cls.RadarInterface = interfaces[cls.platform]
    cls.CP = cls.CarInterface.get_params(cls.platform,  cls.fingerprint, car_fw, experimental_long, docs=False)
+    cls.CP_SP = cls.CarInterface.get_params_sp(cls.CP, cls.platform,  cls.fingerprint, car_fw, experimental_long, docs=False)
    assert cls.CP
+    assert cls.CP_SP
    assert cls.CP.carFingerprint == cls.platform

    os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
@@ -168,7 +171,7 @@ class TestCarModelBase(unittest.TestCase):
    del cls.can_msgs

  def setUp(self):
-    self.CI = self.CarInterface(self.CP.copy(), self.CarController, self.CarState)
+    self.CI = self.CarInterface(self.CP.copy(), copy.deepcopy(self.CP_SP), self.CarController, self.CarState)
    assert self.CI

    Params().put_bool("OpenpilotEnabledToggle", self.openpilot_enabled)
@@ -217,7 +220,7 @@ class TestCarModelBase(unittest.TestCase):
    self.assertEqual(can_invalid_cnt, 0)

  def test_radar_interface(self):
-    RI = self.RadarInterface(self.CP)
+    RI = self.RadarInterface(self.CP, self.CP_SP)
    assert RI

    # Since OBD port is multiplexed to bus 1 (commonly radar bus) while fingerprinting,
@@ -277,7 +280,7 @@ class TestCarModelBase(unittest.TestCase):
    def test_car_controller(car_control):
      now_nanos = 0
      msgs_sent = 0
-      CI = self.CarInterface(self.CP, self.CarController, self.CarState)
+      CI = self.CarInterface(self.CP, self.CP_SP, self.CarController, self.CarState)
      for _ in range(round(10.0 / DT_CTRL)):  # make sure we hit the slowest messages
        CI.update([])
        _, sendcan = CI.apply(car_control, now_nanos)
@@ -387,7 +390,7 @@ class TestCarModelBase(unittest.TestCase):
    controls_allowed_prev = False
    CS_prev = car.CarState.new_message()
    checks = defaultdict(int)
-    selfdrived = SelfdriveD(CP=self.CP)
+    selfdrived = SelfdriveD(CP=self.CP, CP_SP=self.CP_SP)
    selfdrived.initialized = True
    for idx, can in enumerate(self.can_msgs):
      CS = self.CI.update(can_capnp_to_list((can.as_builder().to_bytes(), ))).as_reader()
@@ -2,7 +2,7 @@
 import math
 from typing import SupportsFloat

-from cereal import car, log
+from cereal import car, log, custom
 import cereal.messaging as messaging
 from openpilot.common.conversions import Conversions as CV
 from openpilot.common.params import Params
@@ -34,11 +34,16 @@ class Controls:
    self.CP = messaging.log_from_bytes(self.params.get("CarParams", block=True), car.CarParams)
    cloudlog.info("controlsd got CarParams")

-    self.CI = get_car_interface(self.CP)
+    cloudlog.info("controlsd is waiting for CarParamsSP")
+    self.CP_SP = messaging.log_from_bytes(self.params.get("CarParamsSP", block=True), custom.CarParamsSP)
+    cloudlog.info("controlsd got CarParamsSP")
+
+    self.CI = get_car_interface(self.CP, self.CP_SP)

    self.sm = messaging.SubMaster(['liveParameters', 'liveTorqueParameters', 'modelV2', 'selfdriveState',
                                   'liveCalibration', 'livePose', 'longitudinalPlan', 'carState', 'carOutput',
-                                   'driverMonitoringState', 'onroadEvents', 'driverAssistance'], poll='selfdriveState')
+                                   'driverMonitoringState', 'onroadEvents', 'driverAssistance'] + ['selfdriveStateSP'],
+                                  poll='selfdriveState')
    self.pm = messaging.PubMaster(['carControl', 'controlsState'])

    self.steer_limited = False
@@ -57,9 +62,6 @@ class Controls:
    elif self.CP.lateralTuning.which() == 'torque':
      self.LaC = LatControlTorque(self.CP, self.CI)

-    data_services = list(self.sm.data.keys()) + ['selfdriveStateSP']
-    self.sm = messaging.SubMaster(data_services, poll='selfdriveState')
-
  def update(self):
    self.sm.update(15)
    if self.sm.updated["liveCalibration"]:
@@ -19,7 +19,8 @@ class TestLatControl:
  def test_saturation(self, car_name, controller):
    CarInterface, CarController, CarState, RadarInterface = interfaces[car_name]
    CP = CarInterface.get_non_essential_params(car_name)
-    CI = CarInterface(CP, CarController, CarState)
+    CP_SP = CarInterface.get_non_essential_params_sp(CP, car_name)
+    CI = CarInterface(CP, CP_SP, CarController, CarState)
    VM = VehicleModel(CP)

    controller = controller(CP.as_reader(), CI)
@@ -5,7 +5,7 @@ import threading

 import cereal.messaging as messaging

-from cereal import car, log
+from cereal import car, log, custom
 from msgq.visionipc import VisionIpcClient, VisionStreamType
 from panda import ALTERNATIVE_EXPERIENCE

@@ -47,7 +47,7 @@ IGNORED_SAFETY_MODES = (SafetyModel.silent, SafetyModel.noOutput)


 class SelfdriveD(CruiseHelper):
-  def __init__(self, CP=None):
+  def __init__(self, CP=None, CP_SP=None):
    self.params = Params()

    # Ensure the current branch is cached, otherwise the first cycle lags
@@ -60,11 +60,18 @@ class SelfdriveD(CruiseHelper):
    else:
      self.CP = CP

+    if CP_SP is None:
+      cloudlog.info("selfdrived is waiting for CarParamsSP")
+      self.CP_SP = messaging.log_from_bytes(self.params.get("CarParamsSP", block=True), custom.CarParamsSP)
+      cloudlog.info("selfdrived got CarParamsSP")
+    else:
+      self.CP_SP = CP_SP
+
    self.car_events = CarSpecificEvents(self.CP)
    self.disengage_on_accelerator = not (self.CP.alternativeExperience & ALTERNATIVE_EXPERIENCE.DISABLE_DISENGAGE_ON_GAS)

    # Setup sockets
-    self.pm = messaging.PubMaster(['selfdriveState', 'onroadEvents'])
+    self.pm = messaging.PubMaster(['selfdriveState', 'onroadEvents'] + ['selfdriveStateSP', 'onroadEventsSP'])

    self.gps_location_service = get_gps_location_service(self.params)
    self.gps_packets = [self.gps_location_service]
@@ -140,8 +147,6 @@ class SelfdriveD(CruiseHelper):
    self.events_sp_prev = []

    self.mads = ModularAssistiveDrivingSystem(self)
-    sock_services = list(self.pm.sock.keys()) + ['selfdriveStateSP', 'onroadEventsSP']
-    self.pm = messaging.PubMaster(sock_services)

    self.car_events_sp = CarSpecificEventsSP(self.CP, self.params)

@@ -23,7 +23,7 @@ from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.common.timeout import Timeout
 from openpilot.common.realtime import DT_CTRL
 from panda.python import ALTERNATIVE_EXPERIENCE
-from openpilot.selfdrive.car.card import can_comm_callbacks
+from openpilot.selfdrive.car.card import can_comm_callbacks, convert_to_capnp
 from openpilot.system.manager.process_config import managed_processes
 from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera_state, available_streams
 from openpilot.selfdrive.test.process_replay.migration import migrate_all
@@ -344,6 +344,7 @@ def get_car_params_callback(rc, pm, msgs, fingerprint):
  if fingerprint:
    CarInterface, _, _, _ = interfaces[fingerprint]
    CP = CarInterface.get_non_essential_params(fingerprint)
+    CP_SP = CarInterface.get_non_essential_params_sp(CP, fingerprint)
  else:
    can = DummySocket()
    sendcan = DummySocket()
@@ -364,12 +365,14 @@ def get_car_params_callback(rc, pm, msgs, fingerprint):
      with car.CarParams.from_bytes(cached_params_raw) as _cached_params:
        cached_params = _cached_params

-    CP = get_car(*can_callbacks, lambda obd: None, Params().get_bool("ExperimentalLongitudinalEnabled"), cached_params=cached_params).CP
+    _CI = get_car(*can_callbacks, lambda obd: None, Params().get_bool("ExperimentalLongitudinalEnabled"), cached_params=cached_params)
+    CP, CP_SP = _CI.CP, _CI.CP_SP

    if not params.get_bool("DisengageOnAccelerator"):
      CP.alternativeExperience |= ALTERNATIVE_EXPERIENCE.DISABLE_DISENGAGE_ON_GAS

  params.put("CarParams", CP.to_bytes())
+  params.put("CarParamsSP", convert_to_capnp(CP_SP).to_bytes())


 def selfdrived_rcv_callback(msg, cfg, frame):
@@ -48,13 +48,13 @@ class MadsParams:
      if pause_lateral_on_brake:
        CP.alternativeExperience |= ALTERNATIVE_EXPERIENCE.DISENGAGE_LATERAL_ON_BRAKE

-  def set_car_specific_params(self, CP):
+  def set_car_specific_params(self, CP, CP_SP):
    if CP.carName == "hyundai":
      # TODO-SP: This should be separated from MADS module for future implementations
      #          Use "HyundaiLongitudinalMainCruiseToggleable" param
      hyundai_cruise_main_toggleable = True
      if hyundai_cruise_main_toggleable:
-        CP.sunnypilotFlags |= HyundaiFlagsSP.LONGITUDINAL_MAIN_CRUISE_TOGGLEABLE.value
+        CP_SP.flags |= HyundaiFlagsSP.LONGITUDINAL_MAIN_CRUISE_TOGGLEABLE.value
        CP.safetyConfigs[-1].safetyParam |= Panda.FLAG_HYUNDAI_LONG_MAIN_CRUISE_TOGGLEABLE

    # MADS is currently not supported in Tesla due to lack of consistent states to engage controls
@@ -57,7 +57,7 @@ class ModularAssistiveDrivingSystem:
    self.events_sp = self.selfdrive.events_sp

    if self.selfdrive.CP.carName == "hyundai":
-      if (self.selfdrive.CP.sunnypilotFlags & HyundaiFlagsSP.HAS_LFA_BUTTON) or \
+      if (self.selfdrive.CP_SP.flags & HyundaiFlagsSP.HAS_LFA_BUTTON) or \
            (self.selfdrive.CP.flags & HyundaiFlags.CANFD):
        self.allow_always = True

@@ -22,20 +22,21 @@ def log_fingerprint(CP: structs.CarParams) -> None:
    sentry.capture_fingerprint(CP.carFingerprint, CP.carName)


-def setup_car_interface_sp(CP: structs.CarParams, params):
+def setup_car_interface_sp(CP: structs.CarParams, CP_SP: structs.CarParamsSP, params):
  if CP.carName == 'hyundai':
    if CP.flags & HyundaiFlags.MANDO_RADAR and CP.radarUnavailable:
      # Having this automatic without a toggle causes a weird process replay diff because
      # somehow it sees fewer logs than intended
      if params.get_bool("HyundaiRadarTracksToggle"):
-        CP.sunnypilotFlags |= HyundaiFlagsSP.ENABLE_RADAR_TRACKS.value
+        CP_SP.flags |= HyundaiFlagsSP.ENABLE_RADAR_TRACKS.value
        if params.get_bool("HyundaiRadarTracks"):
          CP.radarUnavailable = False


-def initialize_car_interface_sp(CP: structs.CarParams, params, can_recv: CanRecvCallable, can_send: CanSendCallable):
+def initialize_car_interface_sp(CP: structs.CarParams, CP_SP: structs.CarParamsSP, params, can_recv: CanRecvCallable,
+                                can_send: CanSendCallable):
  if CP.carName == 'hyundai':
-    if CP.sunnypilotFlags & HyundaiFlagsSP.ENABLE_RADAR_TRACKS:
+    if CP_SP.flags & HyundaiFlagsSP.ENABLE_RADAR_TRACKS:
      can_recv()
      _, fingerprint = can_fingerprint(can_recv)
      radar_unavailable = RADAR_START_ADDR not in fingerprint[1] or Bus.radar not in HYUNDAI_DBC[CP.carFingerprint]