mirror of
https://github.com/infiniteCable2/openpilot.git
synced 2026-08-05 16:26:10 +08:00
models: Remove local model compilation for modeldv2 (#868)
* Remove supercombo model ONNX file. Deleted the large `supercombo.onnx` model file from the repository. This cleanup reduces repository size and dependency on unused or outdated files for this version. * Disable tinygrad model compilation on macos temporarily * Remove unused dmonitoring model file. Deleted the ONNX model for dmonitoring as it is no longer required. This eliminates unnecessary assets and reduces repository size. * Removing the model also from the snpe build, we have them, prebuilt
This commit is contained in:
DevTekVE
@@ -47,8 +47,10 @@ elif arch == 'Darwin':
|
||||
else:
|
||||
device_string = 'LLVM=1 LLVMOPT=1 BEAM=0 IMAGE=0'
|
||||
|
||||
for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
|
||||
fn = File(f"models/{model_name}").abspath
|
||||
cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl'
|
||||
lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
|
||||
# TODO-SP: after 15.4 it's not possible to compile models locally on mac https://discord.com/channels/469524606043160576/1362735424644055230
|
||||
if arch != 'Darwin':
|
||||
for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
|
||||
fn = File(f"models/{model_name}").abspath
|
||||
cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl'
|
||||
lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
|
||||
|
||||
|
||||
@@ -52,15 +52,3 @@ commonmodel_lib = lenv.Library('commonmodel', common_src)
|
||||
lenvCython.Program('runners/runmodel_pyx.so', 'runners/runmodel_pyx.pyx', LIBS=cython_libs, FRAMEWORKS=frameworks)
|
||||
lenvCython.Program('runners/snpemodel_pyx.so', 'runners/snpemodel_pyx.pyx', LIBS=[snpemodel_lib, snpe_lib, *cython_libs], FRAMEWORKS=frameworks, RPATH=snpe_rpath)
|
||||
lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks)
|
||||
|
||||
tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath)]
|
||||
|
||||
# Get model metadata
|
||||
fn = File("models/supercombo").abspath
|
||||
cmd = f'python3 {Dir("#sunnypilot/modeld").abspath}/get_model_metadata.py {fn}.onnx'
|
||||
lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
|
||||
|
||||
if arch == "larch64":
|
||||
thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'OpenCL', 'dl'])
|
||||
thneedmodel_lib = env.Library('thneedmodel', ['runners/thneedmodel.cc'])
|
||||
lenvCython.Program('runners/thneedmodel_pyx.so', 'runners/thneedmodel_pyx.pyx', LIBS=envCython["LIBS"]+[thneedmodel_lib, thneed_lib, gpucommon, common, 'dl', 'OpenCL'])
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fb2018c74cdd9e5cb070ec7bed7f8581fabd55e39057d0a03aaffd2e42408154
|
||||
size 62486347
|
||||
@@ -29,22 +29,4 @@ for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transfor
|
||||
cython_libs = envCython["LIBS"] + libs
|
||||
commonmodel_lib = lenv.Library('commonmodel', common_src)
|
||||
lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks)
|
||||
tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath) if 'pycache' not in x]
|
||||
|
||||
# Get model metadata
|
||||
fn = File("models/supercombo").abspath
|
||||
cmd = f'python3 {Dir("#sunnypilot/modeld_v2").abspath}/get_model_metadata.py {fn}.onnx'
|
||||
lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
|
||||
|
||||
# Compile tinygrad model
|
||||
pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"'
|
||||
if arch == 'larch64':
|
||||
device_string = 'QCOM=1'
|
||||
else:
|
||||
device_string = 'CLANG=1 IMAGE=0'
|
||||
|
||||
for model_name in ['supercombo', 'dmonitoring_model']:
|
||||
fn = File(f"models/{model_name}").abspath
|
||||
cmd = f'{pythonpath_string} {device_string} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {fn}_tinygrad.pkl'
|
||||
lenv.Command(fn + "_tinygrad.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
|
||||
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
exec "$DIR/dmonitoringmodeld.py" "$@"
|
||||
@@ -1,192 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
from openpilot.system.hardware import TICI
|
||||
if TICI:
|
||||
from tinygrad.tensor import Tensor
|
||||
from tinygrad.dtype import dtypes
|
||||
from openpilot.sunnypilot.modeld_v2.runners.tinygrad_helpers import qcom_tensor_from_opencl_address
|
||||
os.environ['QCOM'] = '1'
|
||||
else:
|
||||
from openpilot.sunnypilot.modeld_v2.runners.ort_helpers import make_onnx_cpu_runner
|
||||
import math
|
||||
import time
|
||||
import pickle
|
||||
import ctypes
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from setproctitle import setproctitle
|
||||
|
||||
from cereal import messaging
|
||||
from cereal.messaging import PubMaster, SubMaster
|
||||
from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.common.realtime import config_realtime_process
|
||||
from openpilot.common.transformations.model import dmonitoringmodel_intrinsics, DM_INPUT_SIZE
|
||||
from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye
|
||||
from openpilot.sunnypilot.modeld_v2.models.commonmodel_pyx import CLContext, MonitoringModelFrame
|
||||
from openpilot.sunnypilot.modeld_v2.parse_model_outputs import sigmoid
|
||||
from openpilot.system import sentry
|
||||
|
||||
MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE
|
||||
CALIB_LEN = 3
|
||||
FEATURE_LEN = 512
|
||||
OUTPUT_SIZE = 84 + FEATURE_LEN
|
||||
|
||||
PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld"
|
||||
SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
|
||||
MODEL_PATH = Path(__file__).parent / 'models/dmonitoring_model.onnx'
|
||||
MODEL_PKL_PATH = Path(__file__).parent / 'models/dmonitoring_model_tinygrad.pkl'
|
||||
|
||||
|
||||
class DriverStateResult(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("face_orientation", ctypes.c_float*3),
|
||||
("face_position", ctypes.c_float*3),
|
||||
("face_orientation_std", ctypes.c_float*3),
|
||||
("face_position_std", ctypes.c_float*3),
|
||||
("face_prob", ctypes.c_float),
|
||||
("_unused_a", ctypes.c_float*8),
|
||||
("left_eye_prob", ctypes.c_float),
|
||||
("_unused_b", ctypes.c_float*8),
|
||||
("right_eye_prob", ctypes.c_float),
|
||||
("left_blink_prob", ctypes.c_float),
|
||||
("right_blink_prob", ctypes.c_float),
|
||||
("sunglasses_prob", ctypes.c_float),
|
||||
("occluded_prob", ctypes.c_float),
|
||||
("ready_prob", ctypes.c_float*4),
|
||||
("not_ready_prob", ctypes.c_float*2)]
|
||||
|
||||
|
||||
class DMonitoringModelResult(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("driver_state_lhd", DriverStateResult),
|
||||
("driver_state_rhd", DriverStateResult),
|
||||
("poor_vision_prob", ctypes.c_float),
|
||||
("wheel_on_right_prob", ctypes.c_float),
|
||||
("features", ctypes.c_float*FEATURE_LEN)]
|
||||
|
||||
|
||||
class ModelState:
|
||||
inputs: dict[str, np.ndarray]
|
||||
output: np.ndarray
|
||||
|
||||
def __init__(self, cl_ctx):
|
||||
assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float)
|
||||
|
||||
self.frame = MonitoringModelFrame(cl_ctx)
|
||||
self.numpy_inputs = {
|
||||
'calib': np.zeros((1, CALIB_LEN), dtype=np.float32),
|
||||
}
|
||||
|
||||
if TICI:
|
||||
self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()}
|
||||
with open(MODEL_PKL_PATH, "rb") as f:
|
||||
self.model_run = pickle.load(f)
|
||||
else:
|
||||
self.onnx_cpu_runner = make_onnx_cpu_runner(MODEL_PATH)
|
||||
|
||||
def run(self, buf: VisionBuf, calib: np.ndarray, transform: np.ndarray) -> tuple[np.ndarray, float]:
|
||||
self.numpy_inputs['calib'][0,:] = calib
|
||||
|
||||
t1 = time.perf_counter()
|
||||
|
||||
input_img_cl = self.frame.prepare(buf, transform.flatten())
|
||||
if TICI:
|
||||
# The imgs tensors are backed by opencl memory, only need init once
|
||||
if 'input_img' not in self.tensor_inputs:
|
||||
self.tensor_inputs['input_img'] = qcom_tensor_from_opencl_address(input_img_cl.mem_address, (1, MODEL_WIDTH*MODEL_HEIGHT), dtype=dtypes.uint8)
|
||||
else:
|
||||
self.numpy_inputs['input_img'] = self.frame.buffer_from_cl(input_img_cl).reshape((1, MODEL_WIDTH*MODEL_HEIGHT))
|
||||
|
||||
if TICI:
|
||||
output = self.model_run(**self.tensor_inputs).numpy().flatten()
|
||||
else:
|
||||
output = self.onnx_cpu_runner.run(None, self.numpy_inputs)[0].flatten()
|
||||
|
||||
t2 = time.perf_counter()
|
||||
return output, t2 - t1
|
||||
|
||||
|
||||
def fill_driver_state(msg, ds_result: DriverStateResult):
|
||||
msg.faceOrientation = list(ds_result.face_orientation)
|
||||
msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std]
|
||||
msg.facePosition = list(ds_result.face_position[:2])
|
||||
msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]]
|
||||
msg.faceProb = float(sigmoid(ds_result.face_prob))
|
||||
msg.leftEyeProb = float(sigmoid(ds_result.left_eye_prob))
|
||||
msg.rightEyeProb = float(sigmoid(ds_result.right_eye_prob))
|
||||
msg.leftBlinkProb = float(sigmoid(ds_result.left_blink_prob))
|
||||
msg.rightBlinkProb = float(sigmoid(ds_result.right_blink_prob))
|
||||
msg.sunglassesProb = float(sigmoid(ds_result.sunglasses_prob))
|
||||
msg.occludedProb = float(sigmoid(ds_result.occluded_prob))
|
||||
msg.readyProb = [float(sigmoid(x)) for x in ds_result.ready_prob]
|
||||
msg.notReadyProb = [float(sigmoid(x)) for x in ds_result.not_ready_prob]
|
||||
|
||||
|
||||
def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: int, execution_time: float, gpu_execution_time: float):
|
||||
model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(DMonitoringModelResult)).contents
|
||||
msg = messaging.new_message('driverStateV2', valid=True)
|
||||
ds = msg.driverStateV2
|
||||
ds.frameId = frame_id
|
||||
ds.modelExecutionTime = execution_time
|
||||
ds.gpuExecutionTime = gpu_execution_time
|
||||
ds.poorVisionProb = float(sigmoid(model_result.poor_vision_prob))
|
||||
ds.wheelOnRightProb = float(sigmoid(model_result.wheel_on_right_prob))
|
||||
ds.rawPredictions = model_output.tobytes() if SEND_RAW_PRED else b''
|
||||
fill_driver_state(ds.leftDriverData, model_result.driver_state_lhd)
|
||||
fill_driver_state(ds.rightDriverData, model_result.driver_state_rhd)
|
||||
return msg
|
||||
|
||||
|
||||
def main():
|
||||
setproctitle(PROCESS_NAME)
|
||||
config_realtime_process([0, 1, 2, 3], 5)
|
||||
|
||||
sentry.set_tag("daemon", PROCESS_NAME)
|
||||
cloudlog.bind(daemon=PROCESS_NAME)
|
||||
|
||||
cl_context = CLContext()
|
||||
model = ModelState(cl_context)
|
||||
cloudlog.warning("models loaded, dmonitoringmodeld starting")
|
||||
|
||||
cloudlog.warning("connecting to driver stream")
|
||||
vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context)
|
||||
while not vipc_client.connect(False):
|
||||
time.sleep(0.1)
|
||||
assert vipc_client.is_connected()
|
||||
cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}")
|
||||
|
||||
sm = SubMaster(["liveCalibration"])
|
||||
pm = PubMaster(["driverStateV2"])
|
||||
|
||||
calib = np.zeros(CALIB_LEN, dtype=np.float32)
|
||||
model_transform = None
|
||||
|
||||
while True:
|
||||
buf = vipc_client.recv()
|
||||
if buf is None:
|
||||
continue
|
||||
|
||||
if model_transform is None:
|
||||
cam = _os_fisheye if buf.width == _os_fisheye.width else _ar_ox_fisheye
|
||||
model_transform = np.linalg.inv(np.dot(dmonitoringmodel_intrinsics, np.linalg.inv(cam.intrinsics))).astype(np.float32)
|
||||
|
||||
sm.update(0)
|
||||
if sm.updated["liveCalibration"]:
|
||||
calib[:] = np.array(sm["liveCalibration"].rpyCalib)
|
||||
|
||||
t1 = time.perf_counter()
|
||||
model_output, gpu_execution_time = model.run(buf, calib, model_transform)
|
||||
t2 = time.perf_counter()
|
||||
|
||||
pm.send("driverStateV2", get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
cloudlog.warning(f"child {PROCESS_NAME} got SIGINT")
|
||||
except Exception:
|
||||
sentry.capture_exception()
|
||||
raise
|
||||
@@ -1,2 +0,0 @@
|
||||
fa69be01-b430-4504-9d72-7dcb058eb6dd
|
||||
d9fb22d1c4fa3ca3d201dbc8edf1d0f0918e53e6
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:50efe6451a3fb3fa04b6bb0e846544533329bd46ecefe9e657e91214dee2aaeb
|
||||
size 7196502
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d21daa542227ecc5972da45df4e26f018ba113c0461f270e367d57e3ad89221a
|
||||
size 51461700
|
||||
Reference in New Issue
Block a user