From 61bc1470013cbbca88f238f09a29d1b62472fd09 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 2 Nov 2026 22:38:34 +0000 Subject: [PATCH] Move HITL tests to UP (#39138) --- openpilot/common/hardware/usb.py | 10 +++ openpilot/selfdrive/modeld/helpers.py | 3 +- openpilot/selfdrive/modeld/modeld.py | 4 +- openpilot/selfdrive/ui/ui_state.py | 4 +- openpilot/system/camerad/test/test_camerad.py | 75 +++++++++---------- 5 files changed, 51 insertions(+), 45 deletions(-) diff --git a/openpilot/common/hardware/usb.py b/openpilot/common/hardware/usb.py index 55d970a245..6658b2d4b4 100644 --- a/openpilot/common/hardware/usb.py +++ b/openpilot/common/hardware/usb.py @@ -7,6 +7,7 @@ CHESTNUT_ROM_USB_IDS = ((0x174C, 0x2464), (0x174C, 0x2463)) CHESTNUT_USB_PRODUCT = f"custom {CHESTNUT_FW_VERSION}-CLEAN" USB_DEVICES_PATH = Path("/sys/bus/usb/devices") TYPEC_CC_ORIENTATION_PATH = Path("/sys/class/power_supply/usb/typec_cc_orientation") +UDC_PATH = Path("/sys/class/udc") PRIMARY_USB_CONTROLLER = "a600000.ssusb" @@ -40,6 +41,15 @@ def cable_connected() -> bool: return read_int(TYPEC_CC_ORIENTATION_PATH) != 0 +def usb_host_connected() -> bool: + """Whether a USB host, e.g. a PC with adb or the UP in CI, is on the USB-C port, also while it has suspended this device. + A chestnut is a USB device, so then it isn't on that port.""" + try: + return any(read(p) not in (None, "not attached") for p in UDC_PATH.glob("*/state")) + except OSError: + return False + + def usb_devices() -> list[Path]: try: devices = (d for d in USB_DEVICES_PATH.glob("*") if (d / "idVendor").exists()) diff --git a/openpilot/selfdrive/modeld/helpers.py b/openpilot/selfdrive/modeld/helpers.py index eb15083db1..5ec423b510 100644 --- a/openpilot/selfdrive/modeld/helpers.py +++ b/openpilot/selfdrive/modeld/helpers.py @@ -39,7 +39,8 @@ def chestnut_present() -> bool: def chestnut_compiled() -> bool: path = modeld_pkl_path(chestnut=True) - return path.is_file() and all( + # the big model is only built for releases, in a dev checkout it's a git LFS pointer + return path.is_file() and path.stat().st_size > 1024 and all( (MODELS_DIR / f'big_driving_warp_{size}_tinygrad.pkl').is_file() for size in ('1344x760', '1928x1208')) def wait_for_chestnut(timeout: float = 10.) -> None: diff --git a/openpilot/selfdrive/modeld/modeld.py b/openpilot/selfdrive/modeld/modeld.py index 07320b82e1..0754baf747 100755 --- a/openpilot/selfdrive/modeld/modeld.py +++ b/openpilot/selfdrive/modeld/modeld.py @@ -33,7 +33,7 @@ from msgq.visionipc import VisionIpcClient, VisionBuf from opendbc.car.car_helpers import get_demo_car_params from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params -from openpilot.common.hardware.usb import cable_connected +from openpilot.common.hardware.usb import cable_connected, usb_host_connected from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.realtime import config_realtime_process, set_core_affinity, DT_MDL from openpilot.common.transformations.camera import DEVICE_CAMERAS @@ -310,7 +310,7 @@ def main(demo=False): from openpilot.starpilot.models.manager import resolve_runtime, requested_runtime_id # a Jetson, PC or phone on the port looks like a chestnut to cable_connected(): the cable counts with Jetlink off only - chestnut_available = chestnut_present() or (cable_connected() and not jetlink_adapter.link_on()) + chestnut_available = chestnut_present() or (cable_connected() and not jetlink_adapter.link_on() and not usb_host_connected()) recovery_small_only = os.getenv("STARPILOT_MODELD_RECOVERY_SMALL_ONLY") == "1" selection = resolve_runtime(chestnut_available=chestnut_available and not recovery_small_only, randomize=not recovery_small_only) requested_model_id = requested_runtime_id(chestnut_available) diff --git a/openpilot/selfdrive/ui/ui_state.py b/openpilot/selfdrive/ui/ui_state.py index 0b8a774230..0cecc62cac 100644 --- a/openpilot/selfdrive/ui/ui_state.py +++ b/openpilot/selfdrive/ui/ui_state.py @@ -20,7 +20,7 @@ from openpilot.selfdrive.ui.lib.prime_state import PrimeState from openpilot.selfdrive.selfdrived.alertmanager import OFFROAD_ALERTS from openpilot.system.ui.lib.application import gui_app from openpilot.common.hardware import HARDWARE, PC -from openpilot.common.hardware.usb import cable_connected, get_usb_state, is_chestnut_usb_id +from openpilot.common.hardware.usb import cable_connected, get_usb_state, is_chestnut_usb_id, usb_host_connected from openpilot.starpilot.models.manager import ModelManager from openpilot.starpilot.models.jetlink_status import JetlinkStatusFeed from openpilot.tools.replay.display_clock import BOOT_TIME_SERVICES, DisplayClockReader @@ -390,7 +390,7 @@ class UIState: # are eligible. Live transitions retain their stricter receive boundary. self.started_frame = max(0, self.sm.frame - 1) if replay else self.sm.frame self.started_time = time.monotonic() - self.chestnut_present = self.sm["deviceState"].chestnutPresent or (self.chestnut_compiled and self.usb_connected) + self.chestnut_present = self.sm["deviceState"].chestnutPresent or (self.chestnut_compiled and self.usb_connected and not usb_host_connected()) for callback in self._offroad_transition_callbacks: gui_app.measure_frame_phase("transition", callback) diff --git a/openpilot/system/camerad/test/test_camerad.py b/openpilot/system/camerad/test/test_camerad.py index 410ea9fdb3..8a6db6fcbe 100755 --- a/openpilot/system/camerad/test/test_camerad.py +++ b/openpilot/system/camerad/test/test_camerad.py @@ -14,28 +14,26 @@ from openpilot.selfdrive.test.helpers import collect_logs, log_collector, proces TEST_TIMESPAN = 10 CAMERAS = ('narrowRoadCameraState', 'cabinCameraState', 'wideRoadCameraState') -EXPOSURE_STABLE_COUNT = 3 -EXPOSURE_RANGE = (0.15, 0.35) +EXPOSURE_STABLE_FRAMES = 20 +# auto exposure should settle within this fraction of camerad's (scene dependent) target grey +EXPOSURE_TOLERANCE = 0.2 MAX_TEST_TIME = 25 def _numpy_rgb2gray(im): return np.clip(im[:,:,2] * 0.114 + im[:,:,1] * 0.587 + im[:,:,0] * 0.299, 0, 255).astype(np.uint8) -def _exposure_stats(im): +def _image_stats(im): h, w = im.shape[:2] - gray = _numpy_rgb2gray(im[h//10:9*h//10, w//10:9*w//10]) - return float(np.median(gray) / 255.), float(np.mean(gray) / 255.) + gray = _numpy_rgb2gray(im[h//10:9*h//10, w//10:9*w//10]) / 255. + return float(np.mean(gray)), float(np.std(gray)) -def _in_range(median, mean): - lo, hi = EXPOSURE_RANGE - return lo < median < hi and lo < mean < hi - -def _exposure_stable(results): - return all( - len(v) >= EXPOSURE_STABLE_COUNT and all(_in_range(*s) for s in v[-EXPOSURE_STABLE_COUNT:]) - for v in results.values() - ) +def _converged(ts, cam): + measured, target = ts[cam]['measuredGreyFraction'], ts[cam]['targetGreyFraction'] + # in a scene too dark (or bright) to reach the target, auto exposure settles at its limit instead + ev = ts[cam]['exposureValPercent'] + saturated = ((ev > 99.9) & (measured < target)) | ((ev < 0.1) & (measured > target)) + return (np.abs(measured - target) < EXPOSURE_TOLERANCE * target) | saturated def run_and_log(procs, services, duration): @@ -46,31 +44,34 @@ def _camera_session(): """Single camerad session that collects logs and exposure data. Runs until exposure stabilizes (min TEST_TIMESPAN seconds for enough log data).""" with processes_context(["camerad"]), log_collector(CAMERAS) as (raw_logs, lock): - exposure = {cam: [] for cam in CAMERAS} + images = {cam: [] for cam in CAMERAS} start = time.monotonic() while time.monotonic() - start < MAX_TEST_TIME: rpic, dpic = get_snapshots(frame="narrowRoadCameraState", front_frame="cabinCameraState") wpic, _ = get_snapshots(frame="wideRoadCameraState") for cam, img in zip(CAMERAS, [rpic, dpic, wpic], strict=True): - exposure[cam].append(_exposure_stats(img)) + images[cam].append(_image_stats(img)) - if time.monotonic() - start >= TEST_TIMESPAN and _exposure_stable(exposure): - break - - elapsed = time.monotonic() - start + if time.monotonic() - start >= TEST_TIMESPAN: + with lock: + ts = msgs_to_time_series(raw_logs) + if all(len(ts[cam]['t']) >= EXPOSURE_STABLE_FRAMES and + _converged(ts, cam)[-EXPOSURE_STABLE_FRAMES:].all() for cam in CAMERAS): + break with lock: ts = msgs_to_time_series(raw_logs) for cam in CAMERAS: - expected_frames = SERVICE_LIST[cam].frequency * elapsed + # measured from the first frame, camerad takes a few seconds to start + expected_frames = SERVICE_LIST[cam].frequency * (ts[cam]['t'][-1] - ts[cam]['t'][0]) + 1 cnt = len(ts[cam]['t']) assert expected_frames*0.8 < cnt < expected_frames*1.2, f"unexpected frame count {cam}: {expected_frames=}, got {cnt}" dts = np.abs(np.diff([ts[cam]['timestampSof']/1e6]) - 1000/SERVICE_LIST[cam].frequency) assert (dts < 1.0).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}" - return ts, exposure + return ts, images class TestCamerad(OpenpilotTestCase): COMMA_HARDWARE_TEST = True @@ -78,28 +79,22 @@ class TestCamerad(OpenpilotTestCase): @classmethod def setUpClass(cls): super().setUpClass() - cls.logs, cls.exposure_data = _camera_session() + cls.logs, cls.images = _camera_session() @parameterized.expand(CAMERAS, names=("cam",)) def test_camera_exposure(self, cam): - lo, hi = EXPOSURE_RANGE - checks = self.exposure_data[cam] - assert len(checks) >= EXPOSURE_STABLE_COUNT, f"{cam}: only got {len(checks)} samples" + # auto exposure converges to its target + converged = _converged(self.logs, cam) + assert len(converged) >= EXPOSURE_STABLE_FRAMES and converged[-EXPOSURE_STABLE_FRAMES:].all(), \ + f"{cam}: exposure didn't converge, {converged.sum()}/{len(converged)} frames converged" - # check that exposure converges into the valid range - passed = sum(_in_range(med, mean) for med, mean in checks) - assert passed >= EXPOSURE_STABLE_COUNT, \ - f"{cam}: only {passed}/{len(checks)} checks in range. " + \ - " | ".join(f"#{i+1}: med={m:.4f} mean={u:.4f}" for i, (m, u) in enumerate(checks)) - - # check that exposure is stable once converged (no regressions) - in_range = False - for i, (median, mean) in enumerate(checks): - ok = _in_range(median, mean) - if in_range and not ok: - self.fail(f"{cam}: exposure regressed on sample {i+1} " + - f"(median={median:.4f}, mean={mean:.4f}, expected: ({lo}, {hi}))") - in_range = ok + @parameterized.expand(CAMERAS, names=("cam",)) + def test_camera_image(self, cam): + # the image isn't black, saturated, or flat + mean, std = self.images[cam][-1] + assert 0.02 < mean < 0.9, f"{cam}: bad image brightness {mean=:.3f}" + # a dead sensor or pipeline is flat to within noise, the rack scenes have little contrast, e.g. std=0.016 + assert std > 0.005, f"{cam}: flat image {std=:.3f}" def test_frame_skips(self): for c in CAMERAS: