From b4c3e4d4d9b841a40f5ae43d5759bb2dc1bdc37b Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Fri, 2 May 2025 16:25:23 +0200 Subject: [PATCH] models: Remove local model compilation for modeldv2 (#868) * Remove supercombo model ONNX file. Deleted the large `supercombo.onnx` model file from the repository. This cleanup reduces repository size and dependency on unused or outdated files for this version. * Disable tinygrad model compilation on macos temporarily * Remove unused dmonitoring model file. Deleted the ONNX model for dmonitoring as it is no longer required. This eliminates unnecessary assets and reduces repository size. * Removing the model also from the snpe build, we have them, prebuilt --- selfdrive/modeld/SConscript | 10 +- sunnypilot/modeld/SConscript | 12 -- sunnypilot/modeld/models/supercombo.onnx | 3 - sunnypilot/modeld_v2/SConscript | 18 -- sunnypilot/modeld_v2/dmonitoringmodeld | 4 - sunnypilot/modeld_v2/dmonitoringmodeld.py | 192 ------------------ .../models/dmonitoring_model.current | 2 - .../modeld_v2/models/dmonitoring_model.onnx | 3 - sunnypilot/modeld_v2/models/supercombo.onnx | 3 - 9 files changed, 6 insertions(+), 241 deletions(-) delete mode 100644 sunnypilot/modeld/models/supercombo.onnx delete mode 100755 sunnypilot/modeld_v2/dmonitoringmodeld delete mode 100755 sunnypilot/modeld_v2/dmonitoringmodeld.py delete mode 100644 sunnypilot/modeld_v2/models/dmonitoring_model.current delete mode 100644 sunnypilot/modeld_v2/models/dmonitoring_model.onnx delete mode 100644 sunnypilot/modeld_v2/models/supercombo.onnx diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index cecebfa18..0c04ec282 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -47,8 +47,10 @@ elif arch == 'Darwin': else: device_string = 'LLVM=1 LLVMOPT=1 BEAM=0 IMAGE=0' -for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']: - fn = File(f"models/{model_name}").abspath - cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl' - lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd) +# TODO-SP: after 15.4 it's not possible to compile models locally on mac https://discord.com/channels/469524606043160576/1362735424644055230 +if arch != 'Darwin': + for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']: + fn = File(f"models/{model_name}").abspath + cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl' + lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd) diff --git a/sunnypilot/modeld/SConscript b/sunnypilot/modeld/SConscript index 1a54fe1e3..bebd2de95 100644 --- a/sunnypilot/modeld/SConscript +++ b/sunnypilot/modeld/SConscript @@ -52,15 +52,3 @@ commonmodel_lib = lenv.Library('commonmodel', common_src) lenvCython.Program('runners/runmodel_pyx.so', 'runners/runmodel_pyx.pyx', LIBS=cython_libs, FRAMEWORKS=frameworks) lenvCython.Program('runners/snpemodel_pyx.so', 'runners/snpemodel_pyx.pyx', LIBS=[snpemodel_lib, snpe_lib, *cython_libs], FRAMEWORKS=frameworks, RPATH=snpe_rpath) lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks) - -tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath)] - -# Get model metadata -fn = File("models/supercombo").abspath -cmd = f'python3 {Dir("#sunnypilot/modeld").abspath}/get_model_metadata.py {fn}.onnx' -lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd) - -if arch == "larch64": - thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'OpenCL', 'dl']) - thneedmodel_lib = env.Library('thneedmodel', ['runners/thneedmodel.cc']) - lenvCython.Program('runners/thneedmodel_pyx.so', 'runners/thneedmodel_pyx.pyx', LIBS=envCython["LIBS"]+[thneedmodel_lib, thneed_lib, gpucommon, common, 'dl', 'OpenCL']) diff --git a/sunnypilot/modeld/models/supercombo.onnx b/sunnypilot/modeld/models/supercombo.onnx deleted file mode 100644 index 8b7126c44..000000000 --- a/sunnypilot/modeld/models/supercombo.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fb2018c74cdd9e5cb070ec7bed7f8581fabd55e39057d0a03aaffd2e42408154 -size 62486347 diff --git a/sunnypilot/modeld_v2/SConscript b/sunnypilot/modeld_v2/SConscript index 1a173dcde..4ade1469f 100644 --- a/sunnypilot/modeld_v2/SConscript +++ b/sunnypilot/modeld_v2/SConscript @@ -29,22 +29,4 @@ for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transfor cython_libs = envCython["LIBS"] + libs commonmodel_lib = lenv.Library('commonmodel', common_src) lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks) -tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x] - -# Get model metadata -fn = File("models/supercombo").abspath -cmd = f'python3 {Dir("#sunnypilot/modeld_v2").abspath}/get_model_metadata.py {fn}.onnx' -lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd) - -# Compile tinygrad model -pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"' -if arch == 'larch64': - device_string = 'QCOM=1' -else: - device_string = 'CLANG=1 IMAGE=0' - -for model_name in ['supercombo', 'dmonitoring_model']: - fn = File(f"models/{model_name}").abspath - cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl' - lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd) diff --git a/sunnypilot/modeld_v2/dmonitoringmodeld b/sunnypilot/modeld_v2/dmonitoringmodeld deleted file mode 100755 index 90b43800f..000000000 --- a/sunnypilot/modeld_v2/dmonitoringmodeld +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -exec "$DIR/dmonitoringmodeld.py" "$@" diff --git a/sunnypilot/modeld_v2/dmonitoringmodeld.py b/sunnypilot/modeld_v2/dmonitoringmodeld.py deleted file mode 100755 index 4469618c8..000000000 --- a/sunnypilot/modeld_v2/dmonitoringmodeld.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -import os -from openpilot.system.hardware import TICI -if TICI: - from tinygrad.tensor import Tensor - from tinygrad.dtype import dtypes - from openpilot.sunnypilot.modeld_v2.runners.tinygrad_helpers import qcom_tensor_from_opencl_address - os.environ['QCOM'] = '1' -else: - from openpilot.sunnypilot.modeld_v2.runners.ort_helpers import make_onnx_cpu_runner -import math -import time -import pickle -import ctypes -import numpy as np -from pathlib import Path -from setproctitle import setproctitle - -from cereal import messaging -from cereal.messaging import PubMaster, SubMaster -from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf -from openpilot.common.swaglog import cloudlog -from openpilot.common.realtime import config_realtime_process -from openpilot.common.transformations.model import dmonitoringmodel_intrinsics, DM_INPUT_SIZE -from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye -from openpilot.sunnypilot.modeld_v2.models.commonmodel_pyx import CLContext, MonitoringModelFrame -from openpilot.sunnypilot.modeld_v2.parse_model_outputs import sigmoid -from openpilot.system import sentry - -MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE -CALIB_LEN = 3 -FEATURE_LEN = 512 -OUTPUT_SIZE = 84 + FEATURE_LEN - -PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld" -SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') -MODEL_PATH = Path(__file__).parent / 'models/dmonitoring_model.onnx' -MODEL_PKL_PATH = Path(__file__).parent / 'models/dmonitoring_model_tinygrad.pkl' - - -class DriverStateResult(ctypes.Structure): - _fields_ = [ - ("face_orientation", ctypes.c_float*3), - ("face_position", ctypes.c_float*3), - ("face_orientation_std", ctypes.c_float*3), - ("face_position_std", ctypes.c_float*3), - ("face_prob", ctypes.c_float), - ("_unused_a", ctypes.c_float*8), - ("left_eye_prob", ctypes.c_float), - ("_unused_b", ctypes.c_float*8), - ("right_eye_prob", ctypes.c_float), - ("left_blink_prob", ctypes.c_float), - ("right_blink_prob", ctypes.c_float), - ("sunglasses_prob", ctypes.c_float), - ("occluded_prob", ctypes.c_float), - ("ready_prob", ctypes.c_float*4), - ("not_ready_prob", ctypes.c_float*2)] - - -class DMonitoringModelResult(ctypes.Structure): - _fields_ = [ - ("driver_state_lhd", DriverStateResult), - ("driver_state_rhd", DriverStateResult), - ("poor_vision_prob", ctypes.c_float), - ("wheel_on_right_prob", ctypes.c_float), - ("features", ctypes.c_float*FEATURE_LEN)] - - -class ModelState: - inputs: dict[str, np.ndarray] - output: np.ndarray - - def __init__(self, cl_ctx): - assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float) - - self.frame = MonitoringModelFrame(cl_ctx) - self.numpy_inputs = { - 'calib': np.zeros((1, CALIB_LEN), dtype=np.float32), - } - - if TICI: - self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()} - with open(MODEL_PKL_PATH, "rb") as f: - self.model_run = pickle.load(f) - else: - self.onnx_cpu_runner = make_onnx_cpu_runner(MODEL_PATH) - - def run(self, buf: VisionBuf, calib: np.ndarray, transform: np.ndarray) -> tuple[np.ndarray, float]: - self.numpy_inputs['calib'][0,:] = calib - - t1 = time.perf_counter() - - input_img_cl = self.frame.prepare(buf, transform.flatten()) - if TICI: - # The imgs tensors are backed by opencl memory, only need init once - if 'input_img' not in self.tensor_inputs: - self.tensor_inputs['input_img'] = qcom_tensor_from_opencl_address(input_img_cl.mem_address, (1, MODEL_WIDTH*MODEL_HEIGHT), dtype=dtypes.uint8) - else: - self.numpy_inputs['input_img'] = self.frame.buffer_from_cl(input_img_cl).reshape((1, MODEL_WIDTH*MODEL_HEIGHT)) - - if TICI: - output = self.model_run(**self.tensor_inputs).numpy().flatten() - else: - output = self.onnx_cpu_runner.run(None, self.numpy_inputs)[0].flatten() - - t2 = time.perf_counter() - return output, t2 - t1 - - -def fill_driver_state(msg, ds_result: DriverStateResult): - msg.faceOrientation = list(ds_result.face_orientation) - msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std] - msg.facePosition = list(ds_result.face_position[:2]) - msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]] - msg.faceProb = float(sigmoid(ds_result.face_prob)) - msg.leftEyeProb = float(sigmoid(ds_result.left_eye_prob)) - msg.rightEyeProb = float(sigmoid(ds_result.right_eye_prob)) - msg.leftBlinkProb = float(sigmoid(ds_result.left_blink_prob)) - msg.rightBlinkProb = float(sigmoid(ds_result.right_blink_prob)) - msg.sunglassesProb = float(sigmoid(ds_result.sunglasses_prob)) - msg.occludedProb = float(sigmoid(ds_result.occluded_prob)) - msg.readyProb = [float(sigmoid(x)) for x in ds_result.ready_prob] - msg.notReadyProb = [float(sigmoid(x)) for x in ds_result.not_ready_prob] - - -def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: int, execution_time: float, gpu_execution_time: float): - model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(DMonitoringModelResult)).contents - msg = messaging.new_message('driverStateV2', valid=True) - ds = msg.driverStateV2 - ds.frameId = frame_id - ds.modelExecutionTime = execution_time - ds.gpuExecutionTime = gpu_execution_time - ds.poorVisionProb = float(sigmoid(model_result.poor_vision_prob)) - ds.wheelOnRightProb = float(sigmoid(model_result.wheel_on_right_prob)) - ds.rawPredictions = model_output.tobytes() if SEND_RAW_PRED else b'' - fill_driver_state(ds.leftDriverData, model_result.driver_state_lhd) - fill_driver_state(ds.rightDriverData, model_result.driver_state_rhd) - return msg - - -def main(): - setproctitle(PROCESS_NAME) - config_realtime_process([0, 1, 2, 3], 5) - - sentry.set_tag("daemon", PROCESS_NAME) - cloudlog.bind(daemon=PROCESS_NAME) - - cl_context = CLContext() - model = ModelState(cl_context) - cloudlog.warning("models loaded, dmonitoringmodeld starting") - - cloudlog.warning("connecting to driver stream") - vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context) - while not vipc_client.connect(False): - time.sleep(0.1) - assert vipc_client.is_connected() - cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}") - - sm = SubMaster(["liveCalibration"]) - pm = PubMaster(["driverStateV2"]) - - calib = np.zeros(CALIB_LEN, dtype=np.float32) - model_transform = None - - while True: - buf = vipc_client.recv() - if buf is None: - continue - - if model_transform is None: - cam = _os_fisheye if buf.width == _os_fisheye.width else _ar_ox_fisheye - model_transform = np.linalg.inv(np.dot(dmonitoringmodel_intrinsics, np.linalg.inv(cam.intrinsics))).astype(np.float32) - - sm.update(0) - if sm.updated["liveCalibration"]: - calib[:] = np.array(sm["liveCalibration"].rpyCalib) - - t1 = time.perf_counter() - model_output, gpu_execution_time = model.run(buf, calib, model_transform) - t2 = time.perf_counter() - - pm.send("driverStateV2", get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time)) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - cloudlog.warning(f"child {PROCESS_NAME} got SIGINT") - except Exception: - sentry.capture_exception() - raise diff --git a/sunnypilot/modeld_v2/models/dmonitoring_model.current b/sunnypilot/modeld_v2/models/dmonitoring_model.current deleted file mode 100644 index 121871ef2..000000000 --- a/sunnypilot/modeld_v2/models/dmonitoring_model.current +++ /dev/null @@ -1,2 +0,0 @@ -fa69be01-b430-4504-9d72-7dcb058eb6dd -d9fb22d1c4fa3ca3d201dbc8edf1d0f0918e53e6 diff --git a/sunnypilot/modeld_v2/models/dmonitoring_model.onnx b/sunnypilot/modeld_v2/models/dmonitoring_model.onnx deleted file mode 100644 index dcc727510..000000000 --- a/sunnypilot/modeld_v2/models/dmonitoring_model.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:50efe6451a3fb3fa04b6bb0e846544533329bd46ecefe9e657e91214dee2aaeb -size 7196502 diff --git a/sunnypilot/modeld_v2/models/supercombo.onnx b/sunnypilot/modeld_v2/models/supercombo.onnx deleted file mode 100644 index 4c9f79557..000000000 --- a/sunnypilot/modeld_v2/models/supercombo.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d21daa542227ecc5972da45df4e26f018ba113c0461f270e367d57e3ad89221a -size 51461700