[TICI] ci: point to master-tici for certain submodules (#1350)

* ci: point to master-tici for certain submodules

* new

* new

* minimal

* check

* revert

* try this out

* more

cereal/custom.capnp

@@ -202,7 +202,20 @@ struct CarControlSP @0xa5cd762cd951a455 {
 
   struct Param {
     key @0 :Text;
-    value @1 :Text;
+    type @2 :ParamType;
+    value @3 :Data;
+
+    valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
+  }
+  
+  enum ParamType {
+    string @0;
+    bool @1;
+    int @2;
+    float @3;
+    time @4;
+    json @5;
+    bytes @6;
   }
 }
 

release/check-submodules.sh

@@ -31,13 +31,25 @@ while read hash submodule ref; do
       exit 1
     fi
   else
-    git -C $submodule fetch --depth 100 origin master
-    git -C $submodule branch -r --contains $hash | grep "origin/master"
-    if [ "$?" -eq 0 ]; then
-      echo "$submodule ok"
+    # Check against master-tici for specific submodules, master for others
+    if [ "$submodule" = "opendbc_repo" ] || [ "$submodule" = "panda" ]; then
+      git -C $submodule fetch --depth 100 origin master-tici
+      remote_hash=$(git -C $submodule rev-parse FETCH_HEAD)
+      if git -C $submodule merge-base --is-ancestor $hash $remote_hash; then
+        echo "$submodule ok"
+      else
+        echo "$submodule: $hash is not on master-tici"
+        exit 1
+      fi
     else
-      echo "$submodule: $hash is not on master"
-      exit 1
+      git -C $submodule fetch --depth 100 origin master
+      git -C $submodule branch -r --contains $hash | grep "origin/master"
+      if [ "$?" -eq 0 ]; then
+        echo "$submodule ok"
+      else
+        echo "$submodule: $hash is not on master"
+        exit 1
+      fi
     fi
   fi
 done <<< $(git submodule status --recursive)

sunnypilot/selfdrive/controls/controlsd_ext.py

@@ -73,7 +73,7 @@ class ControlsExt:
     # MADS state
     CC_SP.mads = sm['selfdriveStateSP'].mads
 
-    CC_SP.params = self.param_store.publish()
+    CC_SP.params = self.param_store.param_list
 
     return CC_SP
 

sunnypilot/selfdrive/controls/lib/param_store.py

@@ -4,39 +4,41 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
 This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
-import capnp
-
 from cereal import custom
-
 from opendbc.car import structs
 from openpilot.common.params import Params
 
+from sunnypilot.sunnylink.utils import get_param_as_byte
+
 
 class ParamStore:
   keys: list[str]
-  values: dict[str, str]
+  _params: dict[str, custom.CarControlSP.Param]
 
   def __init__(self, CP: structs.CarParams):
     universal_params: list[str] = []
     brand_params: list[str] = []
 
     self.keys = universal_params + brand_params
-    self.values = {}
-    self.cached_params_list: list[capnp.lib.capnp._DynamicStructBuilder] | None = None
+    self._params = {}
 
     self.frame = 0
 
   def update(self, params: Params) -> None:
-    if self.frame % 300 == 0:
-      old_values = dict(self.values)
-      self.values = {k: params.get(k) or "0" for k in self.keys}
-      if old_values != self.values:
-        self.cached_params_list = None
-
     self.frame += 1
+    if self.frame % 300 != 0:
+      return
 
-  def publish(self) -> list[capnp.lib.capnp._DynamicStructBuilder]:
-    if self.cached_params_list is None:
-      # TODO-SP: Why are we doing a list instead of a dictionary here?
-      self.cached_params_list = [custom.CarControlSP.Param(key=k, value=self.values[k]) for k in self.keys]
-    return self.cached_params_list
+    for key in self.keys:
+      param_type = params.get_type(key).name.lower()  # Using string instead of number because its "loose" dependency, and could change by OP at anytime.
+
+      # Over engineering opportunity: It's possible this conversion is slow, we may check the value as params returns it for cache purposes. Not today.
+      param_value = get_param_as_byte(key, params)
+      if (existing_param := self._params.get(key)) is not None and existing_param.value == param_value:
+        continue
+
+      self._params[key] = custom.CarControlSP.Param(key=key, value=param_value, type=param_type)
+
+  @property
+  def param_list(self) -> list[custom.CarControlSP.Param]:
+    return [v for k,v in self._params.items()]

sunnypilot/sunnylink/utils.py

@@ -60,9 +60,9 @@ def get_api_token():
   print(f"API Token: {token}")
 
 
-def get_param_as_byte(param_name: str) -> bytes | None:
+def get_param_as_byte(param_name: str, params=None) -> bytes | None:
   """Get a parameter as bytes. Returns None if the parameter does not exist."""
-  params = Params()
+  params = params or Params() # Use existing Params instance if provided
   param = params.get(param_name)
   if param is None:
     return None
@@ -85,6 +85,17 @@ def save_param_from_base64_encoded_string(param_name: str, base64_encoded_data:
   if is_compressed:
     value = gzip.decompress(value)
 
+  # We convert to string anything that isn't bytes first. We later transform further.
+  param_value = _convert_param_to_type(value, param_type)
+  params.put(param_name, param_value)
+
+
+def _convert_param_to_type(value: bytes, param_type: ParamKeyType) -> bytes | str | int | float | bool | dict | None:
+  """
+  Convert a byte value to the specified param type. Used internally when getting a Param to convert it to the right type.
+  If this method looks familiar, it's because on SP we have a similar one in openpilot/sunnypilot/car/__init__.py.
+  """
+
   # We convert to string anything that isn't bytes first. We later transform further.
   if param_type != ParamKeyType.BYTES:
     value = value.decode('utf-8')  # type: ignore
@@ -101,4 +112,5 @@ def save_param_from_base64_encoded_string(param_name: str, base64_encoded_data:
     value = str(value)  # type: ignore
   elif param_type == ParamKeyType.JSON:
     value = json.loads(value)
-  params.put(param_name, value)
+
+  return value

uv.lock

@@ -912,7 +912,7 @@ dependencies = [
     { name = "yapf" },
 ]
 wheels = [
-    { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl", hash = "sha256:fbf0ea9be67e65cd45d38ff930e3d49f705dd76c9ddbd1e1482e3f87b61efcef" },
+    { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl", hash = "sha256:d0afaf3b005e35e14b929d5491d2d5b64562d0c1cd5093ba969fb63908670dd4" },
 ]
 
 [package.metadata]