Move HITL tests to UP (#39138)

openpilot/common/hardware/usb.py

@@ -7,6 +7,7 @@ CHESTNUT_ROM_USB_IDS = ((0x174C, 0x2464), (0x174C, 0x2463))
 CHESTNUT_USB_PRODUCT = f"custom {CHESTNUT_FW_VERSION}-CLEAN"
 USB_DEVICES_PATH = Path("/sys/bus/usb/devices")
 TYPEC_CC_ORIENTATION_PATH = Path("/sys/class/power_supply/usb/typec_cc_orientation")
+UDC_PATH = Path("/sys/class/udc")
 PRIMARY_USB_CONTROLLER = "a600000.ssusb"
 
 
@@ -40,6 +41,15 @@ def cable_connected() -> bool:
   return read_int(TYPEC_CC_ORIENTATION_PATH) != 0
 
 
+def usb_host_connected() -> bool:
+  """Whether a USB host, e.g. a PC with adb or the UP in CI, is on the USB-C port, also while it has suspended this device.
+  A chestnut is a USB device, so then it isn't on that port."""
+  try:
+    return any(read(p) not in (None, "not attached") for p in UDC_PATH.glob("*/state"))
+  except OSError:
+    return False
+
+
 def usb_devices() -> list[Path]:
   try:
     devices = (d for d in USB_DEVICES_PATH.glob("*") if (d / "idVendor").exists())

openpilot/selfdrive/modeld/helpers.py

@@ -39,7 +39,8 @@ def chestnut_present() -> bool:
 
 def chestnut_compiled() -> bool:
   path = modeld_pkl_path(chestnut=True)
-  return path.is_file() and all(
+  # the big model is only built for releases, in a dev checkout it's a git LFS pointer
+  return path.is_file() and path.stat().st_size > 1024 and all(
     (MODELS_DIR / f'big_driving_warp_{size}_tinygrad.pkl').is_file() for size in ('1344x760', '1928x1208'))
 
 def wait_for_chestnut(timeout: float = 10.) -> None:

openpilot/selfdrive/modeld/modeld.py

@@ -33,7 +33,7 @@ from msgq.visionipc import VisionIpcClient, VisionBuf
 from opendbc.car.car_helpers import get_demo_car_params
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
-from openpilot.common.hardware.usb import cable_connected
+from openpilot.common.hardware.usb import cable_connected, usb_host_connected
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.realtime import config_realtime_process, set_core_affinity, DT_MDL
 from openpilot.common.transformations.camera import DEVICE_CAMERAS
@@ -310,7 +310,7 @@ def main(demo=False):
 
   from openpilot.starpilot.models.manager import resolve_runtime, requested_runtime_id
   # a Jetson, PC or phone on the port looks like a chestnut to cable_connected(): the cable counts with Jetlink off only
-  chestnut_available = chestnut_present() or (cable_connected() and not jetlink_adapter.link_on())
+  chestnut_available = chestnut_present() or (cable_connected() and not jetlink_adapter.link_on() and not usb_host_connected())
   recovery_small_only = os.getenv("STARPILOT_MODELD_RECOVERY_SMALL_ONLY") == "1"
   selection = resolve_runtime(chestnut_available=chestnut_available and not recovery_small_only, randomize=not recovery_small_only)
   requested_model_id = requested_runtime_id(chestnut_available)

openpilot/selfdrive/ui/ui_state.py

@@ -20,7 +20,7 @@ from openpilot.selfdrive.ui.lib.prime_state import PrimeState
 from openpilot.selfdrive.selfdrived.alertmanager import OFFROAD_ALERTS
 from openpilot.system.ui.lib.application import gui_app
 from openpilot.common.hardware import HARDWARE, PC
-from openpilot.common.hardware.usb import cable_connected, get_usb_state, is_chestnut_usb_id
+from openpilot.common.hardware.usb import cable_connected, get_usb_state, is_chestnut_usb_id, usb_host_connected
 from openpilot.starpilot.models.manager import ModelManager
 from openpilot.starpilot.models.jetlink_status import JetlinkStatusFeed
 from openpilot.tools.replay.display_clock import BOOT_TIME_SERVICES, DisplayClockReader
@@ -390,7 +390,7 @@ class UIState:
         # are eligible. Live transitions retain their stricter receive boundary.
         self.started_frame = max(0, self.sm.frame - 1) if replay else self.sm.frame
         self.started_time = time.monotonic()
-        self.chestnut_present = self.sm["deviceState"].chestnutPresent or (self.chestnut_compiled and self.usb_connected)
+        self.chestnut_present = self.sm["deviceState"].chestnutPresent or (self.chestnut_compiled and self.usb_connected and not usb_host_connected())
 
       for callback in self._offroad_transition_callbacks:
         gui_app.measure_frame_phase("transition", callback)

openpilot/system/camerad/test/test_camerad.py

@@ -14,28 +14,26 @@ from openpilot.selfdrive.test.helpers import collect_logs, log_collector, proces
 
 TEST_TIMESPAN = 10
 CAMERAS = ('narrowRoadCameraState', 'cabinCameraState', 'wideRoadCameraState')
-EXPOSURE_STABLE_COUNT = 3
-EXPOSURE_RANGE = (0.15, 0.35)
+EXPOSURE_STABLE_FRAMES = 20
+# auto exposure should settle within this fraction of camerad's (scene dependent) target grey
+EXPOSURE_TOLERANCE = 0.2
 MAX_TEST_TIME = 25
 
 
 def _numpy_rgb2gray(im):
   return np.clip(im[:,:,2] * 0.114 + im[:,:,1] * 0.587 + im[:,:,0] * 0.299, 0, 255).astype(np.uint8)
 
-def _exposure_stats(im):
+def _image_stats(im):
   h, w = im.shape[:2]
-  gray = _numpy_rgb2gray(im[h//10:9*h//10, w//10:9*w//10])
-  return float(np.median(gray) / 255.), float(np.mean(gray) / 255.)
+  gray = _numpy_rgb2gray(im[h//10:9*h//10, w//10:9*w//10]) / 255.
+  return float(np.mean(gray)), float(np.std(gray))
 
-def _in_range(median, mean):
-  lo, hi = EXPOSURE_RANGE
-  return lo < median < hi and lo < mean < hi
-
-def _exposure_stable(results):
-  return all(
-    len(v) >= EXPOSURE_STABLE_COUNT and all(_in_range(*s) for s in v[-EXPOSURE_STABLE_COUNT:])
-    for v in results.values()
-  )
+def _converged(ts, cam):
+  measured, target = ts[cam]['measuredGreyFraction'], ts[cam]['targetGreyFraction']
+  # in a scene too dark (or bright) to reach the target, auto exposure settles at its limit instead
+  ev = ts[cam]['exposureValPercent']
+  saturated = ((ev > 99.9) & (measured < target)) | ((ev < 0.1) & (measured > target))
+  return (np.abs(measured - target) < EXPOSURE_TOLERANCE * target) | saturated
 
 
 def run_and_log(procs, services, duration):
@@ -46,31 +44,34 @@ def _camera_session():
   """Single camerad session that collects logs and exposure data.
      Runs until exposure stabilizes (min TEST_TIMESPAN seconds for enough log data)."""
   with processes_context(["camerad"]), log_collector(CAMERAS) as (raw_logs, lock):
-    exposure = {cam: [] for cam in CAMERAS}
+    images = {cam: [] for cam in CAMERAS}
     start = time.monotonic()
     while time.monotonic() - start < MAX_TEST_TIME:
       rpic, dpic = get_snapshots(frame="narrowRoadCameraState", front_frame="cabinCameraState")
       wpic, _ = get_snapshots(frame="wideRoadCameraState")
       for cam, img in zip(CAMERAS, [rpic, dpic, wpic], strict=True):
-        exposure[cam].append(_exposure_stats(img))
+        images[cam].append(_image_stats(img))
 
-      if time.monotonic() - start >= TEST_TIMESPAN and _exposure_stable(exposure):
-        break
-
-    elapsed = time.monotonic() - start
+      if time.monotonic() - start >= TEST_TIMESPAN:
+        with lock:
+          ts = msgs_to_time_series(raw_logs)
+        if all(len(ts[cam]['t']) >= EXPOSURE_STABLE_FRAMES and
+               _converged(ts, cam)[-EXPOSURE_STABLE_FRAMES:].all() for cam in CAMERAS):
+          break
 
   with lock:
     ts = msgs_to_time_series(raw_logs)
 
   for cam in CAMERAS:
-    expected_frames = SERVICE_LIST[cam].frequency * elapsed
+    # measured from the first frame, camerad takes a few seconds to start
+    expected_frames = SERVICE_LIST[cam].frequency * (ts[cam]['t'][-1] - ts[cam]['t'][0]) + 1
     cnt = len(ts[cam]['t'])
     assert expected_frames*0.8 < cnt < expected_frames*1.2, f"unexpected frame count {cam}: {expected_frames=}, got {cnt}"
 
     dts = np.abs(np.diff([ts[cam]['timestampSof']/1e6]) - 1000/SERVICE_LIST[cam].frequency)
     assert (dts < 1.0).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
 
-  return ts, exposure
+  return ts, images
 
 class TestCamerad(OpenpilotTestCase):
   COMMA_HARDWARE_TEST = True
@@ -78,28 +79,22 @@ class TestCamerad(OpenpilotTestCase):
   @classmethod
   def setUpClass(cls):
     super().setUpClass()
-    cls.logs, cls.exposure_data = _camera_session()
+    cls.logs, cls.images = _camera_session()
 
   @parameterized.expand(CAMERAS, names=("cam",))
   def test_camera_exposure(self, cam):
-    lo, hi = EXPOSURE_RANGE
-    checks = self.exposure_data[cam]
-    assert len(checks) >= EXPOSURE_STABLE_COUNT, f"{cam}: only got {len(checks)} samples"
+    # auto exposure converges to its target
+    converged = _converged(self.logs, cam)
+    assert len(converged) >= EXPOSURE_STABLE_FRAMES and converged[-EXPOSURE_STABLE_FRAMES:].all(), \
+      f"{cam}: exposure didn't converge, {converged.sum()}/{len(converged)} frames converged"
 
-    # check that exposure converges into the valid range
-    passed = sum(_in_range(med, mean) for med, mean in checks)
-    assert passed >= EXPOSURE_STABLE_COUNT, \
-      f"{cam}: only {passed}/{len(checks)} checks in range. " + \
-      " | ".join(f"#{i+1}: med={m:.4f} mean={u:.4f}" for i, (m, u) in enumerate(checks))
-
-    # check that exposure is stable once converged (no regressions)
-    in_range = False
-    for i, (median, mean) in enumerate(checks):
-      ok = _in_range(median, mean)
-      if in_range and not ok:
-        self.fail(f"{cam}: exposure regressed on sample {i+1} " +
-                    f"(median={median:.4f}, mean={mean:.4f}, expected: ({lo}, {hi}))")
-      in_range = ok
+  @parameterized.expand(CAMERAS, names=("cam",))
+  def test_camera_image(self, cam):
+    # the image isn't black, saturated, or flat
+    mean, std = self.images[cam][-1]
+    assert 0.02 < mean < 0.9, f"{cam}: bad image brightness {mean=:.3f}"
+    # a dead sensor or pipeline is flat to within noise, the rack scenes have little contrast, e.g. std=0.016
+    assert std > 0.005, f"{cam}: flat image {std=:.3f}"
 
   def test_frame_skips(self):
     for c in CAMERAS: