#include "selfdrive/ui/qt/onroad/annotated_camera.h" #include #include #include #include "common/swaglog.h" #include "selfdrive/ui/qt/util.h" // Window that shows camera view and variety of info drawn on top AnnotatedCameraWidget::AnnotatedCameraWidget(VisionStreamType type, QWidget *parent) : fps_filter(UI_FREQ, 3, 1. / UI_FREQ), CameraWidget("camerad", type, parent) { pm = std::make_unique(std::vector{"uiDebug"}); main_layout = new QVBoxLayout(this); main_layout->setMargin(UI_BORDER_SIZE); main_layout->setSpacing(0); experimental_btn = new ExperimentalButton(this); main_layout->addWidget(experimental_btn, 0, Qt::AlignTop | Qt::AlignRight); personality_btn = new DrivingPersonalityButton(this); personality_btn->setVisible(false); screen_recorder = new ScreenRecorder(this); screen_recorder->setVisible(false); } bool AnnotatedCameraWidget::handleHudPress(const QPoint &pos) { return hud.handleNavigationPress(pos); } bool AnnotatedCameraWidget::handleHudRelease(const QPoint &pos) { return hud.handleNavigationRelease(pos); } void AnnotatedCameraWidget::updateState(const UIState &s, const StarPilotUIState &fs) { // update engageability/experimental mode button experimental_btn->updateState(s, fs); dmon.updateState(s); const SubMaster &sm = *(s.sm); const cereal::CarState::Reader &carState = sm["carState"].getCarState(); const bool hide_steering_wheel = starpilot_toggles.value("hide_steering_wheel").toBool(); experimental_btn->setVisible(!hide_steering_wheel); const QPoint experimental_button_position = hide_steering_wheel ? QPoint(width() - UI_BORDER_SIZE - btn_size, UI_BORDER_SIZE) : QPoint(experimental_btn->x(), experimental_btn->y()); starpilot_nvg->experimentalButtonPosition = experimental_button_position; bool onroad_distance_btn_enabled = starpilot_nvg->dmIconPosition != QPoint(0, 0) && !starpilot_nvg->hideBottomIcons && starpilot_toggles.value("onroad_distance_button").toBool(); personality_btn->setVisible(onroad_distance_btn_enabled); if (onroad_distance_btn_enabled) { personality_btn->move(starpilot_nvg->rightHandDM ? width() - UI_BORDER_SIZE - personality_btn->width() - (UI_BORDER_SIZE / 2) : UI_BORDER_SIZE, starpilot_nvg->dmIconPosition.y() - personality_btn->height() / 2); personality_btn->updateState(s, fs); } dmon.onroad_distance_btn_enabled = onroad_distance_btn_enabled; const QPoint screen_recorder_position = hide_steering_wheel ? experimental_button_position : QPoint(experimental_button_position.x() - UI_BORDER_SIZE - btn_size, experimental_button_position.y()); screen_recorder->move(screen_recorder_position); screen_recorder->setVisible(starpilot_nvg->standstillDuration == 0 && !(starpilot_nvg->signalStyle == "static" && carState.getRightBlinker()) && starpilot_toggles.value("screen_recorder").toBool()); } void AnnotatedCameraWidget::initializeGL() { CameraWidget::initializeGL(); qInfo() << "OpenGL version:" << QString((const char*)glGetString(GL_VERSION)); qInfo() << "OpenGL vendor:" << QString((const char*)glGetString(GL_VENDOR)); qInfo() << "OpenGL renderer:" << QString((const char*)glGetString(GL_RENDERER)); qInfo() << "OpenGL language version:" << QString((const char*)glGetString(GL_SHADING_LANGUAGE_VERSION)); prev_draw_t = millis_since_boot(); setBackgroundColor(bg_colors[STATUS_DISENGAGED]); } mat4 AnnotatedCameraWidget::calcFrameMatrix() { // Project point at "infinity" to compute x and y offsets // to ensure this ends up in the middle of the screen // for narrow come and a little lower for wide cam. // TODO: use proper perspective transform? // Select intrinsic matrix and calibration based on camera type auto *s = uiState(); bool wide_cam = active_stream_type == VISION_STREAM_WIDE_ROAD; const auto &intrinsic_matrix = wide_cam ? ECAM_INTRINSIC_MATRIX : FCAM_INTRINSIC_MATRIX; const auto &calibration = wide_cam ? s->scene.view_from_wide_calib : s->scene.view_from_calib; // Compute the calibration transformation matrix const auto calib_transform = intrinsic_matrix * calibration; float zoom = wide_cam ? 2.0 : 1.1; Eigen::Vector3f inf(1000., 0., 0.); auto Kep = calib_transform * inf; int w = width(), h = height(); float center_x = intrinsic_matrix(0, 2); float center_y = intrinsic_matrix(1, 2); float max_x_offset = center_x * zoom - w / 2 - 5; float max_y_offset = center_y * zoom - h / 2 - 5; float x_offset = std::clamp((Kep.x() / Kep.z() - center_x) * zoom, -max_x_offset, max_x_offset); float y_offset = std::clamp((Kep.y() / Kep.z() - center_y) * zoom, -max_y_offset, max_y_offset); // Apply transformation such that video pixel coordinates match video // 1) Put (0, 0) in the middle of the video // 2) Apply same scaling as video // 3) Put (0, 0) in top left corner of video Eigen::Matrix3f video_transform =(Eigen::Matrix3f() << zoom, 0.0f, (w / 2 - x_offset) - (center_x * zoom), 0.0f, zoom, (h / 2 - y_offset) - (center_y * zoom), 0.0f, 0.0f, 1.0f).finished(); model.setTransform(video_transform * calib_transform); float zx = zoom * 2 * center_x / w; float zy = zoom * 2 * center_y / h; return mat4{{ zx, 0.0, 0.0, -x_offset / w * 2, 0.0, zy, 0.0, y_offset / h * 2, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, }}; } void AnnotatedCameraWidget::paintGL() { } void AnnotatedCameraWidget::paintEvent(QPaintEvent *event) { UIState *s = uiState(); SubMaster &sm = *(s->sm); const double start_draw_t = millis_since_boot(); QPainter painter(this); // draw camera frame { std::lock_guard lk(frame_lock); if (frames.empty()) { if (skip_frame_count > 0) { skip_frame_count--; qDebug() << "skipping frame, not ready"; return; } } else { // skip drawing up to this many frames if we're // missing camera frames. this smooths out the // transitions from the narrow and wide cameras skip_frame_count = 5; } // Wide or narrow cam dependent on speed bool has_wide_cam = available_streams.count(VISION_STREAM_WIDE_ROAD); if (has_wide_cam) { float v_ego = sm["carState"].getCarState().getVEgo(); if ((v_ego < 10) || available_streams.size() == 1) { wide_cam_requested = true; } else if (v_ego > 15) { wide_cam_requested = false; } wide_cam_requested = wide_cam_requested && sm["selfdriveState"].getSelfdriveState().getExperimentalMode() && starpilot_toggles.value("camera_view").toInt() == 0; } int camera_view = starpilot_toggles.value("camera_view").toInt(); CameraWidget::setStreamType(camera_view == 1 ? VISION_STREAM_DRIVER : ((camera_view == 3 && has_wide_cam) || wide_cam_requested) ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD); CameraWidget::setFrameId(sm["modelV2"].getModelV2().getFrameId()); painter.beginNativePainting(); CameraWidget::paintGL(); painter.endNativePainting(); } painter.setRenderHint(QPainter::Antialiasing); painter.setPen(Qt::NoPen); dmon.starpilot_nvg = starpilot_nvg; hud.starpilot_nvg = starpilot_nvg; model.starpilot_nvg = starpilot_nvg; experimental_btn->starpilot_scene = starpilot_scene; model.starpilot_scene = starpilot_scene; dmon.starpilot_toggles = starpilot_toggles; experimental_btn->starpilot_toggles = starpilot_toggles; hud.starpilot_toggles = starpilot_toggles; model.starpilot_toggles = starpilot_toggles; model.draw(painter, rect()); dmon.draw(painter, rect()); hud.updateState(*s); hud.draw(painter, rect()); starpilot_nvg->paintStarPilotWidgets(painter, *s); double cur_draw_t = millis_since_boot(); double dt = cur_draw_t - prev_draw_t; fps = fps_filter.update(1. / dt * 1000); if (fps < 15) { LOGW("slow frame rate: %.2f fps", fps); } prev_draw_t = cur_draw_t; // publish debug msg MessageBuilder msg; auto m = msg.initEvent().initUiDebug(); m.setDrawTimeMillis(cur_draw_t - start_draw_t); pm->send("uiDebug", msg); } void AnnotatedCameraWidget::showEvent(QShowEvent *event) { CameraWidget::showEvent(event); ui_update_params(uiState()); prev_draw_t = millis_since_boot(); }