From e757bfb584a04afa11735ad86d019e3eebe40da5 Mon Sep 17 00:00:00 2001 From: whoisdomi Date: Tue, 15 Sep 2026 04:59:22 -0500 Subject: [PATCH] shmedium --- selfdrive/modeld/modeld.py | 16 +++++--- selfdrive/modeld/tests/test_usbgpu_helpers.py | 31 ++++++++++++++-- .../tests/test_big_model_engagement.py | 37 +++++++++++++++++++ 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index efb021a6f6..ea3ec0834c 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -106,10 +106,13 @@ EXTERNAL_GPU_POWER_WAIT_TIMEOUT_SECONDS = 60.0 EXTERNAL_GPU_POWER_LOG_INTERVAL_SECONDS = 10.0 LAT_SMOOTH_BP = [2.0, 8.0] -# The background big-model loader must leave modeld's SCHED_FIFO core. Thread affinity is -# inherited from config_realtime_process(7, 54), so without this the loader would sit behind -# the 20 Hz publish loop on core 7 (shared with dmonitoringmodeld) and barely run at all. -BIG_MODEL_LOADER_CORES = {6} +# The background big-model loader must leave modeld's SCHED_FIFO core, because thread +# affinity is inherited from config_realtime_process(7, 54) and the loader would otherwise +# sit behind the 20 Hz publish loop. Every other core is spoken for as well -- 4 is +# card/controlsd, 5 is plannerd/radard/selfdrived, 6 is camerad (system/camerad/main.cc), +# 7 is modeld and dmonitoringmodeld -- so the loader floats across the little cores, which +# run the non-realtime locationd/UI work, instead of pinning on top of one critical process. +BIG_MODEL_LOADER_CORES = {0, 1, 2, 3} # A healthy load takes ~30 s once vehicle power is stable. If the AMD device never comes up # the tinygrad calls can block indefinitely, so give up rather than reporting "loading" @@ -1462,14 +1465,17 @@ def main(demo=False): elif big_loader is not None and not big_loader.in_progress: loaded_big_model, big_load_error = big_loader.take() big_loader = None - params.put_bool("UsbGpuLoading", False) if loaded_big_model is not None: big_model = loaded_big_model + # Raise pending before clearing loading: selfdrived polls these separately at 100 Hz + # and reads "neither loading nor pending" as a failed load. params.put_bool("UsbGpuPending", True) + params.put_bool("UsbGpuLoading", False) cloudlog.warning("big model ready; waiting for the driver to disengage before using it") else: params.put_bool("UsbGpuPending", False) params.put_bool("UsbGpuActive", False) + params.put_bool("UsbGpuLoading", False) cloudlog.error(f"big model unavailable, staying on the small model: {big_load_error}") if big_model is not None and model is not big_model and _big_model_swap_allowed( diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index 7d395dbff5..d9066e3092 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -389,12 +389,35 @@ def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs(): "chestnutState polling must be gated on the background loader being idle" -def test_background_big_model_loader_runs_off_modelds_realtime_core(): - # modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there - # would be starved behind the 20 Hz publish loop. - assert 7 not in modeld.BIG_MODEL_LOADER_CORES +def test_background_big_model_loader_avoids_the_realtime_cores(): + """The loader must not share a core with modeld or the camera pipeline. + + modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there is + starved behind the 20 Hz publish loop. Putting it on camerad's core instead stalls frame + delivery: measured p95 on roadCameraState went 50.8 ms -> 108.9 ms for the whole load. + """ assert modeld.BIG_MODEL_LOADER_CORES + reserved = { + 7: "modeld (config_realtime_process(7, 54)) and dmonitoringmodeld", + 6: "camerad (system/camerad/main.cc set_core_affinity({6}))", + 5: "plannerd, radard, selfdrived, starpilot_process", + 4: "card and controlsd", + } + for core, owner in reserved.items(): + assert core not in modeld.BIG_MODEL_LOADER_CORES, f"core {core} belongs to {owner}" + + # camerad pins itself in C++, which is easy to miss when auditing Python callers. + import re + from pathlib import Path + + camerad_main = Path(modeld.__file__).parents[2] / "system" / "camerad" / "main.cc" + pinned = re.search(r"set_core_affinity\(\{([0-9,\s]+)\}\)", camerad_main.read_text(encoding="utf-8")) + assert pinned, "could not find camerad's core affinity" + camerad_cores = {int(c) for c in pinned.group(1).split(",") if c.strip()} + assert not (camerad_cores & modeld.BIG_MODEL_LOADER_CORES), \ + f"the loader must not share camerad's cores {sorted(camerad_cores)}" + @pytest.mark.parametrize("failure", [ dict(model_error=RuntimeError("artifact is corrupt")), diff --git a/selfdrive/selfdrived/tests/test_big_model_engagement.py b/selfdrive/selfdrived/tests/test_big_model_engagement.py index f25dd022de..e175481f28 100644 --- a/selfdrive/selfdrived/tests/test_big_model_engagement.py +++ b/selfdrive/selfdrived/tests/test_big_model_engagement.py @@ -25,6 +25,43 @@ def test_big_model_failure_still_disengages(): assert ET.SOFT_DISABLE in EVENTS[EventName.bigModelFailed] +def test_pending_is_raised_before_loading_is_cleared(): + """selfdrived polls UsbGpuLoading and UsbGpuPending separately at 100 Hz. + + Clearing loading first leaves a window where neither is set, which reads as a failed load: + captured as exactly one frame of bigModelFailed at the moment the load completed, on two + drives (50.77 s and 62.98 s). + """ + import ast + from pathlib import Path + + from openpilot.selfdrive.modeld import modeld + + source = (Path(modeld.__file__).with_name("modeld.py")).read_text(encoding="utf-8") + main_fn = next(n for n in ast.parse(source).body + if isinstance(n, ast.FunctionDef) and n.name == "main") + collect = next( + node for node in ast.walk(main_fn) + if isinstance(node, ast.If) and "UsbGpuPending" in ast.dump(node) + and "take" in ast.dump(node) + ) + # The success branch is the `if loaded_big_model is not None:` inside the collect block. + success = next(n for n in ast.walk(collect) + if isinstance(n, ast.If) and "loaded_big_model" in ast.dump(n.test)) + writes = [ + node.args[0].value + for node in ast.walk(ast.Module(body=list(success.body), type_ignores=[])) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr == "put_bool" and node.args + and isinstance(node.args[0], ast.Constant) + ] + pending_writes = [i for i, k in enumerate(writes) if k == "UsbGpuPending"] + loading_writes = [i for i, k in enumerate(writes) if k == "UsbGpuLoading"] + assert pending_writes and loading_writes + assert min(pending_writes) < min(loading_writes), \ + "UsbGpuPending must be raised before UsbGpuLoading is cleared" + + def test_loading_banner_is_brief_and_hands_over_to_the_icon(): # The load can run for minutes; the banner announces the handover to the small model and # then gets out of the way, leaving the blinking eGPU icon as the "still loading" cue.