shmedium

selfdrive/modeld/modeld.py

@@ -106,10 +106,13 @@ EXTERNAL_GPU_POWER_WAIT_TIMEOUT_SECONDS = 60.0
 EXTERNAL_GPU_POWER_LOG_INTERVAL_SECONDS = 10.0
 LAT_SMOOTH_BP = [2.0, 8.0]
 
-# The background big-model loader must leave modeld's SCHED_FIFO core. Thread affinity is
-# inherited from config_realtime_process(7, 54), so without this the loader would sit behind
-# the 20 Hz publish loop on core 7 (shared with dmonitoringmodeld) and barely run at all.
-BIG_MODEL_LOADER_CORES = {6}
+# The background big-model loader must leave modeld's SCHED_FIFO core, because thread
+# affinity is inherited from config_realtime_process(7, 54) and the loader would otherwise
+# sit behind the 20 Hz publish loop. Every other core is spoken for as well -- 4 is
+# card/controlsd, 5 is plannerd/radard/selfdrived, 6 is camerad (system/camerad/main.cc),
+# 7 is modeld and dmonitoringmodeld -- so the loader floats across the little cores, which
+# run the non-realtime locationd/UI work, instead of pinning on top of one critical process.
+BIG_MODEL_LOADER_CORES = {0, 1, 2, 3}
 
 # A healthy load takes ~30 s once vehicle power is stable. If the AMD device never comes up
 # the tinygrad calls can block indefinitely, so give up rather than reporting "loading"
@@ -1462,14 +1465,17 @@ def main(demo=False):
     elif big_loader is not None and not big_loader.in_progress:
       loaded_big_model, big_load_error = big_loader.take()
       big_loader = None
-      params.put_bool("UsbGpuLoading", False)
       if loaded_big_model is not None:
         big_model = loaded_big_model
+        # Raise pending before clearing loading: selfdrived polls these separately at 100 Hz
+        # and reads "neither loading nor pending" as a failed load.
         params.put_bool("UsbGpuPending", True)
+        params.put_bool("UsbGpuLoading", False)
         cloudlog.warning("big model ready; waiting for the driver to disengage before using it")
       else:
         params.put_bool("UsbGpuPending", False)
         params.put_bool("UsbGpuActive", False)
+        params.put_bool("UsbGpuLoading", False)
         cloudlog.error(f"big model unavailable, staying on the small model: {big_load_error}")
 
     if big_model is not None and model is not big_model and _big_model_swap_allowed(

selfdrive/modeld/tests/test_usbgpu_helpers.py

@@ -389,12 +389,35 @@ def test_chestnut_telemetry_is_suppressed_while_a_background_load_runs():
     "chestnutState polling must be gated on the background loader being idle"
 
 
-def test_background_big_model_loader_runs_off_modelds_realtime_core():
-  # modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there
-  # would be starved behind the 20 Hz publish loop.
-  assert 7 not in modeld.BIG_MODEL_LOADER_CORES
+def test_background_big_model_loader_avoids_the_realtime_cores():
+  """The loader must not share a core with modeld or the camera pipeline.
+
+  modeld is SCHED_FIFO on core 7 and threads inherit its affinity, so a loader left there is
+  starved behind the 20 Hz publish loop. Putting it on camerad's core instead stalls frame
+  delivery: measured p95 on roadCameraState went 50.8 ms -> 108.9 ms for the whole load.
+  """
   assert modeld.BIG_MODEL_LOADER_CORES
 
+  reserved = {
+    7: "modeld (config_realtime_process(7, 54)) and dmonitoringmodeld",
+    6: "camerad (system/camerad/main.cc set_core_affinity({6}))",
+    5: "plannerd, radard, selfdrived, starpilot_process",
+    4: "card and controlsd",
+  }
+  for core, owner in reserved.items():
+    assert core not in modeld.BIG_MODEL_LOADER_CORES, f"core {core} belongs to {owner}"
+
+  # camerad pins itself in C++, which is easy to miss when auditing Python callers.
+  import re
+  from pathlib import Path
+
+  camerad_main = Path(modeld.__file__).parents[2] / "system" / "camerad" / "main.cc"
+  pinned = re.search(r"set_core_affinity\(\{([0-9,\s]+)\}\)", camerad_main.read_text(encoding="utf-8"))
+  assert pinned, "could not find camerad's core affinity"
+  camerad_cores = {int(c) for c in pinned.group(1).split(",") if c.strip()}
+  assert not (camerad_cores & modeld.BIG_MODEL_LOADER_CORES), \
+    f"the loader must not share camerad's cores {sorted(camerad_cores)}"
+
 
 @pytest.mark.parametrize("failure", [
   dict(model_error=RuntimeError("artifact is corrupt")),

selfdrive/selfdrived/tests/test_big_model_engagement.py

@@ -25,6 +25,43 @@ def test_big_model_failure_still_disengages():
   assert ET.SOFT_DISABLE in EVENTS[EventName.bigModelFailed]
 
 
+def test_pending_is_raised_before_loading_is_cleared():
+  """selfdrived polls UsbGpuLoading and UsbGpuPending separately at 100 Hz.
+
+  Clearing loading first leaves a window where neither is set, which reads as a failed load:
+  captured as exactly one frame of bigModelFailed at the moment the load completed, on two
+  drives (50.77 s and 62.98 s).
+  """
+  import ast
+  from pathlib import Path
+
+  from openpilot.selfdrive.modeld import modeld
+
+  source = (Path(modeld.__file__).with_name("modeld.py")).read_text(encoding="utf-8")
+  main_fn = next(n for n in ast.parse(source).body
+                 if isinstance(n, ast.FunctionDef) and n.name == "main")
+  collect = next(
+    node for node in ast.walk(main_fn)
+    if isinstance(node, ast.If) and "UsbGpuPending" in ast.dump(node)
+    and "take" in ast.dump(node)
+  )
+  # The success branch is the `if loaded_big_model is not None:` inside the collect block.
+  success = next(n for n in ast.walk(collect)
+                 if isinstance(n, ast.If) and "loaded_big_model" in ast.dump(n.test))
+  writes = [
+    node.args[0].value
+    for node in ast.walk(ast.Module(body=list(success.body), type_ignores=[]))
+    if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
+    and node.func.attr == "put_bool" and node.args
+    and isinstance(node.args[0], ast.Constant)
+  ]
+  pending_writes = [i for i, k in enumerate(writes) if k == "UsbGpuPending"]
+  loading_writes = [i for i, k in enumerate(writes) if k == "UsbGpuLoading"]
+  assert pending_writes and loading_writes
+  assert min(pending_writes) < min(loading_writes), \
+    "UsbGpuPending must be raised before UsbGpuLoading is cleared"
+
+
 def test_loading_banner_is_brief_and_hands_over_to_the_icon():
   # The load can run for minutes; the banner announces the handover to the small model and
   # then gets out of the way, leaving the blinking eGPU icon as the "still loading" cue.