diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index 30a31aae27..8b2322f94e 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -7,7 +7,7 @@ from openpilot.common.file_chunker import chunk_file, get_chunk_targets, get_exi from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE, DM_INPUT_SIZE from openpilot.selfdrive.modeld.constants import ModelConstants -from openpilot.selfdrive.modeld.helpers import TG_INPUT_DEVICES_PATH, usbgpu_present, modeld_pkl_path +from openpilot.selfdrive.modeld.helpers import TG_INPUT_DEVICES_PATH, big_model_source_available, usbgpu_present, modeld_pkl_path CAMERA_CONFIGS = [ @@ -44,6 +44,10 @@ tg_devices = { # which device to put jit inputs to at runtime } USBGPU = usbgpu_present() +if USBGPU and not big_model_source_available(): + print("Big model source unavailable, skipping big model build") + USBGPU = False + if USBGPU: usbgpu_tg_flags = f'DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV={tg_backend} FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2' # the USB+AMD GPU takes an exclusive flock; serialize all targets that touch it diff --git a/openpilot/selfdrive/modeld/helpers.py b/openpilot/selfdrive/modeld/helpers.py index 37ab0b26d7..ffd59eae06 100644 --- a/openpilot/selfdrive/modeld/helpers.py +++ b/openpilot/selfdrive/modeld/helpers.py @@ -21,6 +21,10 @@ def modeld_pkl_path(usbgpu: bool): prefix = 'big_' if usbgpu else '' return MODELS_DIR / f'{prefix}driving_tinygrad.pkl' +def big_model_source_available() -> bool: + model_path = MODELS_DIR / 'big_driving_supercombo.onnx' + return model_path.is_file() and model_path.stat().st_size >= 1024 + def dump_oob(obj, f): with tempfile.TemporaryFile(dir=".") as tmp: def buffer_callback(pb: pickle.PickleBuffer): diff --git a/openpilot/system/hardware/hardwared.py b/openpilot/system/hardware/hardwared.py index a423d8f97d..2817ef3942 100755 --- a/openpilot/system/hardware/hardwared.py +++ b/openpilot/system/hardware/hardwared.py @@ -16,6 +16,7 @@ from openpilot.common.utils import strip_deprecated_keys from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.params import Params from openpilot.common.realtime import DT_HW +from openpilot.selfdrive.modeld.helpers import usbgpu_compiled from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.common.hardware import HARDWARE, COMMA_HARDWARE, PC from openpilot.common.basedir import BASEDIR @@ -237,8 +238,7 @@ def hardware_thread(end_event, hw_queue) -> None: fan_controller = FanController(int(1./DT_HW)) chestnut = Chestnut() - big_model_available = os.path.isfile(os.path.join(BASEDIR, "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx")) or \ - os.path.isfile(os.path.join(BASEDIR, "openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl.chunkmanifest")) + big_model_available = usbgpu_compiled() while not end_event.is_set(): sm.update(PANDA_STATES_TIMEOUT)