diff --git a/cereal/libcereal.a b/cereal/libcereal.a index 5b1a1c69e..ff25759b1 100644 Binary files a/cereal/libcereal.a and b/cereal/libcereal.a differ diff --git a/cereal/log.capnp b/cereal/log.capnp index 917cd972f..e2f3bf004 100644 --- a/cereal/log.capnp +++ b/cereal/log.capnp @@ -2161,12 +2161,14 @@ struct DriverStateV2 { facePosition @2 :List(Float32); facePositionStd @3 :List(Float32); faceProb @4 :Float32; - leftEyeProb @5 :Float32; - rightEyeProb @6 :Float32; - leftBlinkProb @7 :Float32; - rightBlinkProb @8 :Float32; - sunglassesProb @9 :Float32; + eyesVisibleProb @14 :Float32; + eyesClosedProb @15 :Float32; phoneProb @13 :Float32; + leftEyeProbDEPRECATED @5 :Float32; + rightEyeProbDEPRECATED @6 :Float32; + leftBlinkProbDEPRECATED @7 :Float32; + rightBlinkProbDEPRECATED @8 :Float32; + sunglassesProbDEPRECATED @9 :Float32; notReadyProbDEPRECATED @12 :List(Float32); occludedProbDEPRECATED @10 :Float32; readyProbDEPRECATED @11 :List(Float32); diff --git a/panda/board/obj/body_h7.bin.signed b/panda/board/obj/body_h7.bin.signed index 1598fe1a3..b0264cbf9 100644 Binary files a/panda/board/obj/body_h7.bin.signed and b/panda/board/obj/body_h7.bin.signed differ diff --git a/panda/board/obj/body_h7/bootstub.elf b/panda/board/obj/body_h7/bootstub.elf index e508479ae..e5af3b154 100755 Binary files a/panda/board/obj/body_h7/bootstub.elf and b/panda/board/obj/body_h7/bootstub.elf differ diff --git a/panda/board/obj/body_h7/main.bin b/panda/board/obj/body_h7/main.bin index eccee6913..7da3f0729 100755 Binary files a/panda/board/obj/body_h7/main.bin and b/panda/board/obj/body_h7/main.bin differ diff --git a/panda/board/obj/body_h7/main.elf b/panda/board/obj/body_h7/main.elf index 3b5eecba5..b5d8c5ce7 100755 Binary files a/panda/board/obj/body_h7/main.elf and b/panda/board/obj/body_h7/main.elf differ diff --git a/panda/board/obj/bootstub.body_h7.bin b/panda/board/obj/bootstub.body_h7.bin index 520fd4635..5c1da6da9 100755 Binary files a/panda/board/obj/bootstub.body_h7.bin and b/panda/board/obj/bootstub.body_h7.bin differ diff --git a/panda/board/obj/bootstub.panda.bin b/panda/board/obj/bootstub.panda.bin index 3b33e5f63..c9fb03d73 100755 Binary files a/panda/board/obj/bootstub.panda.bin and b/panda/board/obj/bootstub.panda.bin differ diff --git a/panda/board/obj/bootstub.panda_h7.bin b/panda/board/obj/bootstub.panda_h7.bin index 5d32cf951..5f8987813 100755 Binary files a/panda/board/obj/bootstub.panda_h7.bin and b/panda/board/obj/bootstub.panda_h7.bin differ diff --git a/panda/board/obj/bootstub.panda_h7_remote.bin b/panda/board/obj/bootstub.panda_h7_remote.bin index 5d32cf951..5f8987813 100755 Binary files a/panda/board/obj/bootstub.panda_h7_remote.bin and b/panda/board/obj/bootstub.panda_h7_remote.bin differ diff --git a/panda/board/obj/bootstub.panda_jungle_h7.bin b/panda/board/obj/bootstub.panda_jungle_h7.bin index bd12b00b7..2246b1e1a 100755 Binary files a/panda/board/obj/bootstub.panda_jungle_h7.bin and b/panda/board/obj/bootstub.panda_jungle_h7.bin differ diff --git a/panda/board/obj/bootstub.panda_remote.bin b/panda/board/obj/bootstub.panda_remote.bin index 3b33e5f63..c9fb03d73 100755 Binary files a/panda/board/obj/bootstub.panda_remote.bin and b/panda/board/obj/bootstub.panda_remote.bin differ diff --git a/panda/board/obj/gitversion.h b/panda/board/obj/gitversion.h index ceb48434c..f8e38b81b 100644 --- a/panda/board/obj/gitversion.h +++ b/panda/board/obj/gitversion.h @@ -1,2 +1,2 @@ extern const uint8_t gitversion[19]; -const uint8_t gitversion[19] = "DEV-ba4cca62-DEBUG"; +const uint8_t gitversion[19] = "DEV-bad1a01c-DEBUG"; diff --git a/panda/board/obj/panda.bin.signed b/panda/board/obj/panda.bin.signed index 3b66392f0..6ca756c34 100644 Binary files a/panda/board/obj/panda.bin.signed and b/panda/board/obj/panda.bin.signed differ diff --git a/panda/board/obj/panda/bootstub.elf b/panda/board/obj/panda/bootstub.elf index c2cfaa35d..1e624bc64 100755 Binary files a/panda/board/obj/panda/bootstub.elf and b/panda/board/obj/panda/bootstub.elf differ diff --git a/panda/board/obj/panda/main.bin b/panda/board/obj/panda/main.bin index 3b585fbac..5774ec54f 100755 Binary files a/panda/board/obj/panda/main.bin and b/panda/board/obj/panda/main.bin differ diff --git a/panda/board/obj/panda/main.elf b/panda/board/obj/panda/main.elf index 8e1ed0414..d76855968 100755 Binary files a/panda/board/obj/panda/main.elf and b/panda/board/obj/panda/main.elf differ diff --git a/panda/board/obj/panda_h7.bin.signed b/panda/board/obj/panda_h7.bin.signed index 6ef5e23d9..b7b394fc6 100644 Binary files a/panda/board/obj/panda_h7.bin.signed and b/panda/board/obj/panda_h7.bin.signed differ diff --git a/panda/board/obj/panda_h7/bootstub.elf b/panda/board/obj/panda_h7/bootstub.elf index 6a58f5d46..0a97fa35b 100755 Binary files a/panda/board/obj/panda_h7/bootstub.elf and b/panda/board/obj/panda_h7/bootstub.elf differ diff --git a/panda/board/obj/panda_h7/main.bin b/panda/board/obj/panda_h7/main.bin index d7512eac6..ccd913026 100755 Binary files a/panda/board/obj/panda_h7/main.bin and b/panda/board/obj/panda_h7/main.bin differ diff --git a/panda/board/obj/panda_h7/main.elf b/panda/board/obj/panda_h7/main.elf index 0c3dae6df..bb349deba 100755 Binary files a/panda/board/obj/panda_h7/main.elf and b/panda/board/obj/panda_h7/main.elf differ diff --git a/panda/board/obj/panda_h7_remote.bin.signed b/panda/board/obj/panda_h7_remote.bin.signed index 093edc785..7cc90093b 100644 Binary files a/panda/board/obj/panda_h7_remote.bin.signed and b/panda/board/obj/panda_h7_remote.bin.signed differ diff --git a/panda/board/obj/panda_h7_remote/bootstub.elf b/panda/board/obj/panda_h7_remote/bootstub.elf index c5834216d..328692bca 100755 Binary files a/panda/board/obj/panda_h7_remote/bootstub.elf and b/panda/board/obj/panda_h7_remote/bootstub.elf differ diff --git a/panda/board/obj/panda_h7_remote/main.bin b/panda/board/obj/panda_h7_remote/main.bin index e639e59b7..e50ca809c 100755 Binary files a/panda/board/obj/panda_h7_remote/main.bin and b/panda/board/obj/panda_h7_remote/main.bin differ diff --git a/panda/board/obj/panda_h7_remote/main.elf b/panda/board/obj/panda_h7_remote/main.elf index 97739f540..ab1d48bcb 100755 Binary files a/panda/board/obj/panda_h7_remote/main.elf and b/panda/board/obj/panda_h7_remote/main.elf differ diff --git a/panda/board/obj/panda_jungle_h7.bin.signed b/panda/board/obj/panda_jungle_h7.bin.signed index 68ebbc1dd..08ac4f9f1 100644 Binary files a/panda/board/obj/panda_jungle_h7.bin.signed and b/panda/board/obj/panda_jungle_h7.bin.signed differ diff --git a/panda/board/obj/panda_jungle_h7/bootstub.elf b/panda/board/obj/panda_jungle_h7/bootstub.elf index 568f60580..80425a46c 100755 Binary files a/panda/board/obj/panda_jungle_h7/bootstub.elf and b/panda/board/obj/panda_jungle_h7/bootstub.elf differ diff --git a/panda/board/obj/panda_jungle_h7/main.bin b/panda/board/obj/panda_jungle_h7/main.bin index a73555fde..3d89ad40e 100755 Binary files a/panda/board/obj/panda_jungle_h7/main.bin and b/panda/board/obj/panda_jungle_h7/main.bin differ diff --git a/panda/board/obj/panda_jungle_h7/main.elf b/panda/board/obj/panda_jungle_h7/main.elf index f51098d6f..49f012b3c 100755 Binary files a/panda/board/obj/panda_jungle_h7/main.elf and b/panda/board/obj/panda_jungle_h7/main.elf differ diff --git a/panda/board/obj/panda_remote.bin.signed b/panda/board/obj/panda_remote.bin.signed index fb33773cd..e0b2828c9 100644 Binary files a/panda/board/obj/panda_remote.bin.signed and b/panda/board/obj/panda_remote.bin.signed differ diff --git a/panda/board/obj/panda_remote/bootstub.elf b/panda/board/obj/panda_remote/bootstub.elf index 4a617368e..54e711fc9 100755 Binary files a/panda/board/obj/panda_remote/bootstub.elf and b/panda/board/obj/panda_remote/bootstub.elf differ diff --git a/panda/board/obj/panda_remote/main.bin b/panda/board/obj/panda_remote/main.bin index 616fdcb96..3d680416a 100755 Binary files a/panda/board/obj/panda_remote/main.bin and b/panda/board/obj/panda_remote/main.bin differ diff --git a/panda/board/obj/panda_remote/main.elf b/panda/board/obj/panda_remote/main.elf index d1791d3d1..23ce9c258 100755 Binary files a/panda/board/obj/panda_remote/main.elf and b/panda/board/obj/panda_remote/main.elf differ diff --git a/panda/board/obj/version b/panda/board/obj/version index 60c411947..7a108e37d 100644 --- a/panda/board/obj/version +++ b/panda/board/obj/version @@ -1 +1 @@ -DEV-ba4cca62-DEBUG \ No newline at end of file +DEV-bad1a01c-DEBUG \ No newline at end of file diff --git a/selfdrive/assets/icons_mici/onroad/glasses.png b/selfdrive/assets/icons_mici/onroad/glasses.png deleted file mode 100644 index e1b080ab4..000000000 Binary files a/selfdrive/assets/icons_mici/onroad/glasses.png and /dev/null differ diff --git a/selfdrive/modeld/models/dmonitoring_model.onnx b/selfdrive/modeld/models/dmonitoring_model.onnx index 7d59ce24e..1cc4613bc 100644 Binary files a/selfdrive/modeld/models/dmonitoring_model.onnx and b/selfdrive/modeld/models/dmonitoring_model.onnx differ diff --git a/selfdrive/modeld/models/dmonitoring_model_metadata.pkl b/selfdrive/modeld/models/dmonitoring_model_metadata.pkl index 71e4b289c..a007b081a 100644 Binary files a/selfdrive/modeld/models/dmonitoring_model_metadata.pkl and b/selfdrive/modeld/models/dmonitoring_model_metadata.pkl differ diff --git a/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl b/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl index 299065b59..2cdadbc47 100644 Binary files a/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl and b/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl differ diff --git a/selfdrive/monitoring/helpers.py b/selfdrive/monitoring/helpers.py index 18df6f74f..1f1f69749 100644 --- a/selfdrive/monitoring/helpers.py +++ b/selfdrive/monitoring/helpers.py @@ -32,9 +32,8 @@ class DRIVER_MONITOR_SETTINGS: self._DISTRACTED_PROMPT_TIME_TILL_TERMINAL = 6. self._FACE_THRESHOLD = 0.7 - self._EYE_THRESHOLD = 0.65 - self._SG_THRESHOLD = 0.9 - self._BLINK_THRESHOLD = 0.865 + self._EYE_THRESHOLD = 0.5 + self._BLINK_THRESHOLD = 0.5 self._PHONE_THRESH = 0.5 self._POSE_PITCH_THRESHOLD = 0.3133 @@ -110,12 +109,6 @@ class DriverProb: self.prob_offseter = RunningStatFilter(raw_priors=raw_priors, max_trackable=max_trackable) self.prob_calibrated = False -class DriverBlink: - def __init__(self): - self.left = 0. - self.right = 0. - - # model output refers to center of undistorted+leveled image EFL = 598.0 # focal length in K cam = DEVICE_CAMERAS[("tici", "ar0231")] # corrected image has same size as raw @@ -149,7 +142,7 @@ class DriverMonitoring: wheelpos_filter_raw_priors = (self.settings._WHEELPOS_DATA_AVG, self.settings._WHEELPOS_DATA_VAR, 2) self.wheelpos = DriverProb(raw_priors=wheelpos_filter_raw_priors, max_trackable=self.settings._WHEELPOS_MAX_COUNT) self.pose = DriverPose(settings=self.settings) - self.blink = DriverBlink() + self.blink_prob = 0. self.phone_prob = 0. self.always_on = always_on @@ -252,7 +245,7 @@ class DriverMonitoring: if pitch_error > pitch_threshold or yaw_error > yaw_threshold: distracted_types.append(DistractedType.DISTRACTED_POSE) - if (self.blink.left + self.blink.right)*0.5 > self.settings._BLINK_THRESHOLD: + if self.blink_prob > self.settings._BLINK_THRESHOLD: distracted_types.append(DistractedType.DISTRACTED_BLINK) if self.phone_prob > self.settings._PHONE_THRESH: @@ -292,10 +285,7 @@ class DriverMonitoring: self.pose.yaw_std = driver_data.faceOrientationStd[1] model_std_max = max(self.pose.pitch_std, self.pose.yaw_std) self.pose.low_std = model_std_max < self.settings._POSESTD_THRESHOLD - self.blink.left = driver_data.leftBlinkProb * (driver_data.leftEyeProb > self.settings._EYE_THRESHOLD) \ - * (driver_data.sunglassesProb < self.settings._SG_THRESHOLD) - self.blink.right = driver_data.rightBlinkProb * (driver_data.rightEyeProb > self.settings._EYE_THRESHOLD) \ - * (driver_data.sunglassesProb < self.settings._SG_THRESHOLD) + self.blink_prob = driver_data.eyesClosedProb * (driver_data.eyesVisibleProb > self.settings._EYE_THRESHOLD) self.phone_prob = driver_data.phoneProb self.distracted_types = self._get_distracted_types() diff --git a/selfdrive/monitoring/test_monitoring.py b/selfdrive/monitoring/test_monitoring.py index 4a3347874..b0a555709 100644 --- a/selfdrive/monitoring/test_monitoring.py +++ b/selfdrive/monitoring/test_monitoring.py @@ -19,10 +19,8 @@ def make_msg(face_detected, distracted=False, model_uncertain=False): ds.leftDriverData.faceOrientation = [0., 0., 0.] ds.leftDriverData.facePosition = [0., 0.] ds.leftDriverData.faceProb = 1. * face_detected - ds.leftDriverData.leftEyeProb = 1. - ds.leftDriverData.rightEyeProb = 1. - ds.leftDriverData.leftBlinkProb = 1. * distracted - ds.leftDriverData.rightBlinkProb = 1. * distracted + ds.leftDriverData.eyesVisibleProb = 1. + ds.leftDriverData.eyesClosedProb = 1. * distracted ds.leftDriverData.faceOrientationStd = [1.*model_uncertain, 1.*model_uncertain, 1.*model_uncertain] ds.leftDriverData.facePositionStd = [1.*model_uncertain, 1.*model_uncertain] # TODO: test both separately when e2e is used diff --git a/selfdrive/test/process_replay/model_replay.py b/selfdrive/test/process_replay/model_replay.py index 9ba599bac..8c29f6540 100755 --- a/selfdrive/test/process_replay/model_replay.py +++ b/selfdrive/test/process_replay/model_replay.py @@ -76,7 +76,7 @@ def generate_report(proposed, master, tmp, commit): (lambda x: get_idx_if_non_empty(x.wheelOnRightProb), "wheelOnRightProb"), (lambda x: get_idx_if_non_empty(x.leftDriverData.faceProb), "leftDriverData.faceProb"), (lambda x: get_idx_if_non_empty(x.leftDriverData.faceOrientation, 0), "leftDriverData.faceOrientation0"), - (lambda x: get_idx_if_non_empty(x.leftDriverData.leftBlinkProb), "leftDriverData.leftBlinkProb"), + (lambda x: get_idx_if_non_empty(x.leftDriverData.eyesClosedProb), "leftDriverData.eyesClosedProb"), (lambda x: get_idx_if_non_empty(x.leftDriverData.phoneProb), "leftDriverData.phoneProb"), (lambda x: get_idx_if_non_empty(x.rightDriverData.faceProb), "rightDriverData.faceProb"), ], "driverStateV2") diff --git a/selfdrive/ui/mici/onroad/driver_camera_dialog.py b/selfdrive/ui/mici/onroad/driver_camera_dialog.py index bab3d6e6f..afacb5323 100644 --- a/selfdrive/ui/mici/onroad/driver_camera_dialog.py +++ b/selfdrive/ui/mici/onroad/driver_camera_dialog.py @@ -42,8 +42,6 @@ class DriverCameraDialog(NavWidget): self._eye_fill_texture = None self._eye_orange_texture = None self._eye_size = 74 - self._glasses_texture = None - self._glasses_size = 171 self._load_eye_textures() @@ -157,8 +155,6 @@ class DriverCameraDialog(NavWidget): self._eye_fill_texture = gui_app.texture("icons_mici/onroad/eye_fill.png", self._eye_size, self._eye_size) if self._eye_orange_texture is None: self._eye_orange_texture = gui_app.texture("icons_mici/onroad/eye_orange.png", self._eye_size, self._eye_size) - if self._glasses_texture is None: - self._glasses_texture = gui_app.texture("icons_mici/onroad/glasses.png", self._glasses_size, self._glasses_size) def _draw_face_detection(self, rect: rl.Rectangle): dm_state = ui_state.sm["driverMonitoringState"] @@ -205,31 +201,21 @@ class DriverCameraDialog(NavWidget): eye_offset_x = 10 eye_offset_y = 10 eye_spacing = self._eye_size + 15 + eyes_prob = driver_data.eyesVisibleProb left_eye_x = rect.x + eye_offset_x left_eye_y = rect.y + eye_offset_y - left_eye_prob = driver_data.leftEyeProb right_eye_x = rect.x + eye_offset_x + eye_spacing right_eye_y = rect.y + eye_offset_y - right_eye_prob = driver_data.rightEyeProb # Draw eyes with opacity based on probability - for eye_x, eye_y, eye_prob in [(left_eye_x, left_eye_y, left_eye_prob), (right_eye_x, right_eye_y, right_eye_prob)]: - fill_opacity = eye_prob - orange_opacity = 1.0 - eye_prob - + fill_opacity = eyes_prob + orange_opacity = 1.0 - eyes_prob + for eye_x, eye_y in [(left_eye_x, left_eye_y), (right_eye_x, right_eye_y)]: rl.draw_texture_v(self._eye_orange_texture, (eye_x, eye_y), rl.Color(255, 255, 255, int(255 * orange_opacity))) rl.draw_texture_v(self._eye_fill_texture, (eye_x, eye_y), rl.Color(255, 255, 255, int(255 * fill_opacity))) - # Draw sunglasses indicator based on sunglasses probability - # Position glasses centered between the two eyes at top left - glasses_x = rect.x + eye_offset_x - 4 - glasses_y = rect.y - glasses_pos = rl.Vector2(glasses_x, glasses_y) - glasses_prob = driver_data.sunglassesProb - rl.draw_texture_v(self._glasses_texture, glasses_pos, rl.Color(70, 80, 161, int(255 * glasses_prob))) - if __name__ == "__main__": gui_app.init_window("Driver Camera View (mici)") diff --git a/starpilot/tinygrad_modeld/dmonitoringmodeld.py b/starpilot/tinygrad_modeld/dmonitoringmodeld.py index 3b767577a..d317f0ad2 100644 --- a/starpilot/tinygrad_modeld/dmonitoringmodeld.py +++ b/starpilot/tinygrad_modeld/dmonitoringmodeld.py @@ -78,7 +78,7 @@ def parse_model_output(model_output): face_descs = model_output[f'face_descs_{ds_suffix}'] parsed[f'face_descs_{ds_suffix}'] = face_descs[:, :-6] parsed[f'face_descs_{ds_suffix}_std'] = safe_exp(face_descs[:, -6:]) - for key in ['face_prob', 'left_eye_prob', 'right_eye_prob', 'left_blink_prob', 'right_blink_prob', 'sunglasses_prob', 'using_phone_prob']: + for key in ['face_prob', 'eyes_visible_prob', 'eyes_closed_prob', 'using_phone_prob']: parsed[f'{key}_{ds_suffix}'] = sigmoid(model_output[f'{key}_{ds_suffix}']) return parsed @@ -89,11 +89,8 @@ def fill_driver_data(msg, model_output, ds_suffix): msg.facePosition = model_output[f'face_descs_{ds_suffix}'][0, 3:5].tolist() msg.facePositionStd = model_output[f'face_descs_{ds_suffix}_std'][0, 3:5].tolist() msg.faceProb = model_output[f'face_prob_{ds_suffix}'][0, 0].item() - msg.leftEyeProb = model_output[f'left_eye_prob_{ds_suffix}'][0, 0].item() - msg.rightEyeProb = model_output[f'right_eye_prob_{ds_suffix}'][0, 0].item() - msg.leftBlinkProb = model_output[f'left_blink_prob_{ds_suffix}'][0, 0].item() - msg.rightBlinkProb = model_output[f'right_blink_prob_{ds_suffix}'][0, 0].item() - msg.sunglassesProb = model_output[f'sunglasses_prob_{ds_suffix}'][0, 0].item() + msg.eyesVisibleProb = model_output[f'eyes_visible_prob_{ds_suffix}'][0, 0].item() + msg.eyesClosedProb = model_output[f'eyes_closed_prob_{ds_suffix}'][0, 0].item() msg.phoneProb = model_output[f'using_phone_prob_{ds_suffix}'][0, 0].item() diff --git a/starpilot/tinygrad_modeld/models/dmonitoring_model_metadata.pkl b/starpilot/tinygrad_modeld/models/dmonitoring_model_metadata.pkl index 71e4b289c..a007b081a 100644 Binary files a/starpilot/tinygrad_modeld/models/dmonitoring_model_metadata.pkl and b/starpilot/tinygrad_modeld/models/dmonitoring_model_metadata.pkl differ