#!/usr/bin/env python3 import os import base64 from openpilot.starpilot.models.runtime_bridge.runtime import prepare as prepare_model_runtime prepare_model_runtime('monitoring') from openpilot.selfdrive.modeld.helpers import MODELS_DIR, load_oob from tinygrad.tensor import Tensor import time import pickle import numpy as np from openpilot.cereal import messaging from openpilot.cereal.messaging import PubMaster, SubMaster from openpilot.cereal.visionipc import VisionStreamType from msgq.visionipc import VisionIpcClient, VisionBuf from openpilot.common.swaglog import cloudlog from openpilot.common.realtime import config_realtime_process from openpilot.common.transformations.model import dmonitoringmodel_intrinsics from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid, safe_exp SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') MODEL_PKL_PATH = MODELS_DIR / 'dmonitoring_model_tinygrad.pkl' class ModelState: inputs: dict[str, np.ndarray] output: np.ndarray def __init__(self, cam_w: int, cam_h: int): jits = load_oob(MODEL_PKL_PATH) self.DEV = jits['input_specs']['input_img'][2] self.input_shapes = jits['metadata']['input_shapes'] self.output_slices = pickle.loads(base64.b64decode(jits['metadata']['metadata']['output_slices'])) self.numpy_inputs = { 'calib': np.zeros(self.input_shapes['calib'], dtype=np.float32), } self.warp_inputs_np = {'transform': np.zeros((3,3), dtype=np.float32)} self.warp_inputs = {k: Tensor(v, device='NPY') for k,v in self.warp_inputs_np.items()} self.frame_buf_params = get_nv12_info(cam_w, cam_h) self.tensor_inputs = {k: Tensor(v, device=self.DEV).realize() for k,v in self.numpy_inputs.items()} self.calib_host = Tensor(self.numpy_inputs['calib'], device='NPY')._buffer() self._blob_cache : dict[int, Tensor] = {} self.model_run = jits['run'] self.outputs = {name: Tensor(np.zeros(shape, dtype=dtype), device=device).realize() for name, (shape, dtype, device) in jits['output_specs'].items()} warp_path = MODELS_DIR / f'dm_warp_{cam_w}x{cam_h}_tinygrad.pkl' from openpilot.starpilot.models.runtime_bridge.runtime import selected_legacy, load_warp if selected_legacy(): self.image_warp = load_warp(warp_path) else: with warp_path.open('rb') as f: self.image_warp = pickle.load(f)['run'] def run(self, buf: VisionBuf, calib: np.ndarray, transform: np.ndarray) -> tuple[np.ndarray, float]: self.numpy_inputs['calib'][0,:] = calib t1 = time.perf_counter() self.tensor_inputs['calib']._buffer().copy_from(self.calib_host) ptr = np.frombuffer(buf.data, dtype=np.uint8).ctypes.data # There is a ringbuffer of imgs, just cache tensors pointing to all of them if ptr not in self._blob_cache: self._blob_cache[ptr] = Tensor.from_blob(ptr, (self.frame_buf_params[3],), dtype='uint8', device=self.DEV) self.warp_inputs_np['transform'][:] = transform[:] self.tensor_inputs['input_img'] = self.image_warp(input_frame=self._blob_cache[ptr], M_inv=self.warp_inputs['transform']) self.model_run(output_buffers=self.outputs, **self.tensor_inputs) output = self.outputs['outputs'].numpy().astype(np.float32).reshape(-1) t2 = time.perf_counter() return output, t2 - t1 def slice_outputs(model_outputs, output_slices): return {k: model_outputs[np.newaxis, v] for k,v in output_slices.items()} def parse_model_output(model_output): parsed = {} parsed['wheel_on_right'] = sigmoid(model_output['wheel_on_right']) for ds_suffix in ['lhd', 'rhd']: face_descs = model_output[f'face_descs_{ds_suffix}'] parsed[f'face_descs_{ds_suffix}'] = face_descs[:, :-6] parsed[f'face_descs_{ds_suffix}_std'] = safe_exp(face_descs[:, -6:]) for key in ['face_prob', 'left_eye_prob', 'right_eye_prob','left_blink_prob', 'right_blink_prob', 'sunglasses_prob', 'using_phone_prob', 'sleep_prob']: parsed[f'{key}_{ds_suffix}'] = sigmoid(model_output[f'{key}_{ds_suffix}']) return parsed def fill_driver_data(msg, model_output, ds_suffix): msg.faceOrientation = model_output[f'face_descs_{ds_suffix}'][0, :3].tolist() msg.faceOrientationStd = model_output[f'face_descs_{ds_suffix}_std'][0, :3].tolist() msg.facePosition = model_output[f'face_descs_{ds_suffix}'][0, 3:5].tolist() msg.facePositionStd = model_output[f'face_descs_{ds_suffix}_std'][0, 3:5].tolist() msg.faceProb = model_output[f'face_prob_{ds_suffix}'][0, 0].item() msg.leftEyeProb = model_output[f'left_eye_prob_{ds_suffix}'][0, 0].item() msg.rightEyeProb = model_output[f'right_eye_prob_{ds_suffix}'][0, 0].item() msg.leftBlinkProb = model_output[f'left_blink_prob_{ds_suffix}'][0, 0].item() msg.rightBlinkProb = model_output[f'right_blink_prob_{ds_suffix}'][0, 0].item() msg.sunglassesProb = model_output[f'sunglasses_prob_{ds_suffix}'][0, 0].item() msg.phoneProb = model_output[f'using_phone_prob_{ds_suffix}'][0, 0].item() msg.sleepProb = model_output[f'sleep_prob_{ds_suffix}'][0, 0].item() def get_driverstate_packet(model_output, frame_id: int, location_ts: int, exec_time: float, gpu_exec_time: float): msg = messaging.new_message('driverStateV2', valid=True) ds = msg.driverStateV2 ds.frameId = frame_id ds.modelExecutionTime = exec_time ds.gpuExecutionTime = gpu_exec_time ds.rawPredictions = model_output['raw_pred'] ds.wheelOnRightProb = model_output['wheel_on_right'][0, 0].item() fill_driver_data(ds.leftDriverData, model_output, 'lhd') fill_driver_data(ds.rightDriverData, model_output, 'rhd') return msg def main(): config_realtime_process(7, 5) cloudlog.warning("connecting to cabin stream") vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_CABIN, True) while not vipc_client.connect(False): time.sleep(0.1) assert vipc_client.is_connected() cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}") model = ModelState(vipc_client.width, vipc_client.height) cloudlog.warning("models loaded, dmonitoringmodeld starting") sm = SubMaster(["extrinsicsCalibration"]) pm = PubMaster(["driverStateV2"]) calib = np.zeros(model.numpy_inputs['calib'].size, dtype=np.float32) model_transform = None while True: buf = vipc_client.recv() if buf is None: continue if model_transform is None: cam = _os_fisheye if buf.width == _os_fisheye.width else _ar_ox_fisheye model_transform = np.linalg.inv(np.dot(dmonitoringmodel_intrinsics, np.linalg.inv(cam.intrinsics))).astype(np.float32) sm.update(0) if sm.updated["extrinsicsCalibration"]: calib[:] = np.array(sm["extrinsicsCalibration"].rpyCalib) t1 = time.perf_counter() model_output, gpu_execution_time = model.run(buf, calib, model_transform) t2 = time.perf_counter() raw_pred = model_output.tobytes() if SEND_RAW_PRED else b'' model_output = slice_outputs(model_output, model.output_slices) model_output = parse_model_output(model_output) model_output['raw_pred'] = raw_pred msg = get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time) pm.send("driverStateV2", msg) if __name__ == "__main__": try: main() except KeyboardInterrupt: cloudlog.warning("got SIGINT")