Merge branch 'master' into sync-priv-20231020

# Conflicts:
#	selfdrive/car/toyota/carstate.py
#	selfdrive/controls/lib/longitudinal_planner.py

selfdrive/test/longitudinal_maneuvers/plant.py

@@ -6,7 +6,7 @@ from cereal import log
 import cereal.messaging as messaging
 from openpilot.common.realtime import Ratekeeper, DT_MDL
 from openpilot.selfdrive.controls.lib.longcontrol import LongCtrlState
-from openpilot.selfdrive.modeld.constants import T_IDXS
+from openpilot.selfdrive.modeld.constants import ModelConstants
 from openpilot.selfdrive.controls.lib.longitudinal_planner import LongitudinalPlanner
 from openpilot.selfdrive.controls.radard import _LEAD_ACCEL_TAU
 
@@ -100,13 +100,13 @@ class Plant:
     # this is to ensure lead policy is effective when model
     # does not predict slowdown in e2e mode
     position = log.XYZTData.new_message()
-    position.x = [float(x) for x in (self.speed + 0.5) * np.array(T_IDXS)]
+    position.x = [float(x) for x in (self.speed + 0.5) * np.array(ModelConstants.T_IDXS)]
     model.modelV2.position = position
     velocity = log.XYZTData.new_message()
-    velocity.x = [float(x) for x in (self.speed + 0.5) * np.ones_like(T_IDXS)]
+    velocity.x = [float(x) for x in (self.speed + 0.5) * np.ones_like(ModelConstants.T_IDXS)]
     model.modelV2.velocity = velocity
     acceleration = log.XYZTData.new_message()
-    acceleration.x = [float(x) for x in np.zeros_like(T_IDXS)]
+    acceleration.x = [float(x) for x in np.zeros_like(ModelConstants.T_IDXS)]
     model.modelV2.acceleration = acceleration
 
     control.controlsState.longControlState = LongCtrlState.pid if self.enabled else LongCtrlState.off

selfdrive/test/process_replay/compare_logs.py

@@ -90,6 +90,26 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=Non
   return diff
 
 
+def format_process_diff(diff):
+  diff_short, diff_long = "", ""
+
+  if isinstance(diff, str):
+    diff_short += f"        {diff}\n"
+    diff_long += f"\t{diff}\n"
+  else:
+    cnt: Dict[str, int] = {}
+    for d in diff:
+      diff_long += f"\t{str(d)}\n"
+
+      k = str(d[1])
+      cnt[k] = 1 if k not in cnt else cnt[k] + 1
+
+    for k, v in sorted(cnt.items()):
+      diff_short += f"        {k}: {v}\n"
+
+  return diff_short, diff_long
+
+
 def format_diff(results, log_paths, ref_commit):
   diff_short, diff_long = "", ""
   diff_long += f"***** tested against commit {ref_commit} *****\n"
@@ -105,25 +125,17 @@ def format_diff(results, log_paths, ref_commit):
       diff_long += f"\tnew: {log_paths[segment][proc]['new']}\n\n"
 
       diff_short += f"    {proc}\n"
-      if isinstance(diff, str):
+
+      if isinstance(diff, str) or len(diff):
         diff_short += f"        ref: {log_paths[segment][proc]['ref']}\n"
         diff_short += f"        new: {log_paths[segment][proc]['new']}\n\n"
-        diff_short += f"        {diff}\n"
         failed = True
-      elif len(diff):
-        diff_short += f"        ref: {log_paths[segment][proc]['ref']}\n"
-        diff_short += f"        new: {log_paths[segment][proc]['new']}\n\n"
 
-        cnt: Dict[str, int] = {}
-        for d in diff:
-          diff_long += f"\t{str(d)}\n"
+        proc_diff_short, proc_diff_long = format_process_diff(diff)
 
-          k = str(d[1])
-          cnt[k] = 1 if k not in cnt else cnt[k] + 1
+        diff_long += proc_diff_long
+        diff_short += proc_diff_short
 
-        for k, v in sorted(cnt.items()):
-          diff_short += f"        {k}: {v}\n"
-        failed = True
   return diff_short, diff_long, failed
 
 

selfdrive/test/process_replay/model_replay_ref_commit

@@ -1 +1 @@
-f851c7e7f90eff828a59444d20fac5df8cd7ae0c
+0e0f55cf3bb2cf79b44adf190e6387a83deb6646

selfdrive/test/test_onroad.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import bz2
 import math
 import json
 import os
@@ -36,7 +37,7 @@ PROCS = {
   "selfdrive.locationd.paramsd": 9.0,
   "./sensord": 7.0,
   "selfdrive.controls.radard": 4.5,
-  "selfdrive.modeld.modeld": 8.0,
+  "selfdrive.modeld.modeld": 13.0,
   "selfdrive.modeld.dmonitoringmodeld": 8.0,
   "selfdrive.modeld.navmodeld": 1.0,
   "selfdrive.thermald.thermald": 3.87,
@@ -161,6 +162,7 @@ class TestOnroad(unittest.TestCase):
 
     # use the second segment by default as it's the first full segment
     cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog")))
+    cls.log_path = cls.segments[1]
 
   @cached_property
   def service_msgs(self):
@@ -191,6 +193,26 @@ class TestOnroad(unittest.TestCase):
     big_logs = [f for f, n in cnt.most_common(3) if n / sum(cnt.values()) > 30.]
     self.assertEqual(len(big_logs), 0, f"Log spam: {big_logs}")
 
+  def test_log_sizes(self):
+    for f in self.log_path.iterdir():
+      assert f.is_file()
+
+      sz = f.stat().st_size / 1e6
+      if f.name in ("qlog", "rlog"):
+        with open(f, 'rb') as ff:
+          sz = len(bz2.compress(ff.read())) / 1e6
+
+      if f.name == "qcamera.ts":
+        assert 2.15 < sz < 2.35
+      elif f.name == "qlog":
+        assert 0.7 < sz < 1.0
+      elif f.name == "rlog":
+        assert 5 < sz < 50
+      elif f.name.endswith('.hevc'):
+        assert 70 < sz < 77
+      else:
+        raise NotImplementedError
+
   def test_ui_timings(self):
     result = "\n"
     result += "------------------------------------------------\n"