diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 5ba4b9e79..7c6985d9c 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -65,6 +65,8 @@ def _model_smooth_seconds(params, key, default): return round(min(max(value, SMOOTH_SECONDS_STEP), 2.0) / SMOOTH_SECONDS_STEP) * SMOOTH_SECONDS_STEP MIN_LAT_CONTROL_SPEED = 0.3 BIG_MODEL_TIMEOUT = 60 +BIG_MODEL_LOAD_WAIT_TIMEOUT_MS = 30000 +BIG_MODEL_RUN_WAIT_TIMEOUT_MS = 3000 def _get_param_str(params: Params, key: str, default: str = "") -> str: @@ -539,7 +541,10 @@ def main(demo=False): params.put_bool("UsbGpuActive", False) params.put_bool("UsbGpuLoading", external_gpu_requested) if external_gpu_requested: - os.environ["HCQDEV_WAIT_TIMEOUT_MS"] = "3000" + # Loading the large artifact competes with the rest of on-road startup. + # Keep the short watchdog for inference, but allow tinygrad's normal wait + # while model weights are being streamed into VRAM. + os.environ["HCQDEV_WAIT_TIMEOUT_MS"] = str(BIG_MODEL_LOAD_WAIT_TIMEOUT_MS) from tinygrad.helpers import DEV device_config = tinygrad_dev_config(True, TICI) DEV.value = device_config @@ -590,6 +595,7 @@ def main(demo=False): loader = threading.Thread(target=load_big_model, name="big_model_loader", daemon=True) loader.start() loader.join(BIG_MODEL_TIMEOUT) + os.environ["HCQDEV_WAIT_TIMEOUT_MS"] = str(BIG_MODEL_RUN_WAIT_TIMEOUT_MS) if loader.is_alive(): cloudlog.error(f"external GPU model load timed out after {BIG_MODEL_TIMEOUT}s") model = big_model diff --git a/selfdrive/modeld/tests/test_usbgpu_helpers.py b/selfdrive/modeld/tests/test_usbgpu_helpers.py index d19447e2d..ab4128abd 100644 --- a/selfdrive/modeld/tests/test_usbgpu_helpers.py +++ b/selfdrive/modeld/tests/test_usbgpu_helpers.py @@ -14,6 +14,11 @@ def test_external_gpu_keeps_the_native_device_available(): assert tinygrad_dev_config(True, tici=False) == "CPU:LLVM;USB+AMD:LLVM" +def test_external_gpu_uses_a_longer_load_watchdog(): + assert modeld.BIG_MODEL_LOAD_WAIT_TIMEOUT_MS == 30000 + assert modeld.BIG_MODEL_RUN_WAIT_TIMEOUT_MS == 3000 + + def test_out_of_band_artifact_round_trip(): artifact = {"weights": np.arange(32, dtype=np.float32), "metadata": {"version": 1}} stream = io.BytesIO()