mawei/dragonpilot
1
0
Fork 0
mirror of https://github.com/dragonpilot/dragonpilot.git synced 2026-06-20 13:32:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

ui: skip texture frame copy (#24700)

Browse Source 
* ui_blit working

* simpler and working

* more believable that it's real

* working on device

* build on pc

* use hardware pc

* reduce cpu usage

* yuv conversion to EGL

* move everything to cameraview

* some cleanup

* more cleanup

* init array

* init images with std::map

* dont destroy images

* do destroy images

Co-authored-by: Comma Device <device@comma.ai>
This commit is contained in:
Joost Wooning
2022-06-13 12:27:47 +02:00
committed by GitHub
parent 724b322909
commit 1221aef233
4 changed files with 84 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selfdrive/test/test_onroad.py
+1 -1
View File
@@ -27,7 +27,7 @@ PROCS = {
"./camerad": 16.5,
"./locationd": 9.1,
"selfdrive.controls.plannerd": 11.7,
"./_ui": 26.4,
"./_ui": 19.2,
"selfdrive.locationd.paramsd": 9.0,
"./_sensord": 6.17,
"selfdrive.controls.radard": 4.5,
selfdrive/ui/SConscript
+3
View File
@@ -5,6 +5,9 @@ Import('qt_env', 'arch', 'common', 'messaging', 'visionipc',
base_libs = [common, messaging, cereal, visionipc, transformations, 'zmq',
'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"]
if arch == 'larch64':
base_libs.append('EGL')
maps = arch in ['larch64', 'x86_64']
if maps and arch == 'x86_64':
selfdrive/ui/qt/widgets/cameraview.cc
+64 -8
View File
@@ -26,6 +26,18 @@ const char frame_vertex_shader[] =
" vTexCoord = aTexCoord;\n"
"}\n";
#ifdef QCOM2
const char frame_fragment_shader[] =
"#version 300 es\n"
"#extension GL_OES_EGL_image_external_essl3 : enable\n"
"precision mediump float;\n"
"uniform samplerExternalOES uTexture;\n"
"in vec2 vTexCoord;\n"
"out vec4 colorOut;\n"
"void main() {\n"
" colorOut = texture(uTexture, vTexCoord);\n"
"}\n";
#else
const char frame_fragment_shader[] =
#ifdef __APPLE__
"#version 330 core\n"
@@ -45,6 +57,7 @@ const char frame_fragment_shader[] =
" float b = y + 1.772 * uv.x;\n"
" colorOut = vec4(r, g, b, 1.0);\n"
"}\n";
#endif
const mat4 device_transform = {{
1.0, 0.0, 0.0, 0.0,
@@ -99,7 +112,7 @@ CameraViewWidget::~CameraViewWidget() {
glDeleteVertexArrays(1, &frame_vao);
glDeleteBuffers(1, &frame_vbo);
glDeleteBuffers(1, &frame_ibo);
glDeleteBuffers(3, textures);
glDeleteBuffers(2, textures);
}
doneCurrent();
}
@@ -143,10 +156,15 @@ void CameraViewWidget::initializeGL() {
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
glGenTextures(3, textures);
glUseProgram(program->programId());
#ifdef QCOM2
glUniform1i(program->uniformLocation("uTexture"), 0);
#else
glGenTextures(2, textures);
glUniform1i(program->uniformLocation("uTextureY"), 0);
glUniform1i(program->uniformLocation("uTextureUV"), 1);
#endif
}
void CameraViewWidget::showEvent(QShowEvent *event) {
@@ -207,29 +225,33 @@ void CameraViewWidget::paintGL() {
for (frame_idx = 0; frame_idx < frames.size() - 1; frame_idx++) {
if (frames[frame_idx].first == draw_frame_id) break;
}
VisionBuf *frame = frames[frame_idx].second;
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glPixelStorei(GL_UNPACK_ROW_LENGTH, stream_stride);
glViewport(0, 0, width(), height());
glBindVertexArray(frame_vao);
glUseProgram(program->programId());
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
VisionBuf *frame = frames[frame_idx].second;
#ifdef QCOM2
glActiveTexture(GL_TEXTURE0);
glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, egl_images[frame->idx]);
assert(glGetError() == GL_NO_ERROR);
#else
glPixelStorei(GL_UNPACK_ROW_LENGTH, stream_stride);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, textures[0]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, stream_width, stream_height, GL_RED, GL_UNSIGNED_BYTE, frame->y);
assert(glGetError() == GL_NO_ERROR);
glPixelStorei(GL_UNPACK_ROW_LENGTH, stream_stride/2);
glActiveTexture(GL_TEXTURE0 + 1);
glBindTexture(GL_TEXTURE_2D, textures[1]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, stream_width/2, stream_height/2, GL_RG, GL_UNSIGNED_BYTE, frame->uv);
assert(glGetError() == GL_NO_ERROR);
#endif
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
assert(glGetError() == GL_NO_ERROR);
glEnableVertexAttribArray(0);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, (const void *)0);
glDisableVertexAttribArray(0);
@@ -247,6 +269,32 @@ void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) {
stream_height = vipc_client->buffers[0].height;
stream_stride = vipc_client->buffers[0].stride;
#ifdef QCOM2
egl_display = eglGetCurrentDisplay();
for (auto &pair : egl_images) {
eglDestroyImageKHR(egl_display, pair.second);
}
egl_images.clear();
for (int i = 0; i < vipc_client->num_buffers; i++) { // import buffers into OpenGL
int fd = dup(vipc_client->buffers[i].fd); // eglDestroyImageKHR will close, so duplicate
EGLint img_attrs[] = {
EGL_WIDTH, (int)vipc_client->buffers[i].width,
EGL_HEIGHT, (int)vipc_client->buffers[i].height,
EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_NV12,
EGL_DMA_BUF_PLANE0_FD_EXT, fd,
EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
EGL_DMA_BUF_PLANE0_PITCH_EXT, (int)vipc_client->buffers[i].stride,
EGL_DMA_BUF_PLANE1_FD_EXT, fd,
EGL_DMA_BUF_PLANE1_OFFSET_EXT, (int)vipc_client->buffers[i].uv_offset,
EGL_DMA_BUF_PLANE1_PITCH_EXT, (int)vipc_client->buffers[i].stride,
EGL_NONE
};
egl_images[i] = eglCreateImageKHR(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, 0, img_attrs);
assert(eglGetError() == EGL_SUCCESS);
}
#else
glBindTexture(GL_TEXTURE_2D, textures[0]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
@@ -262,6 +310,7 @@ void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) {
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, stream_width/2, stream_height/2, 0, GL_RG, GL_UNSIGNED_BYTE, nullptr);
assert(glGetError() == GL_NO_ERROR);
#endif
updateFrameMat(width(), height());
}
@@ -297,4 +346,11 @@ void CameraViewWidget::vipcThread() {
emit vipcThreadFrameReceived(buf, meta_main.frame_id);
}
}
#ifdef QCOM2
for (auto &pair : egl_images) {
eglDestroyImageKHR(egl_display, pair.second);
}
egl_images.clear();
#endif
}
selfdrive/ui/qt/widgets/cameraview.h
+16 -1
View File
@@ -6,6 +6,16 @@
#include <QOpenGLShaderProgram>
#include <QOpenGLWidget>
#include <QThread>
#ifdef QCOM2
#define EGL_EGLEXT_PROTOTYPES
#define EGL_NO_X11
#define GL_TEXTURE_EXTERNAL_OES 0x8D65
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <drm/drm_fourcc.h>
#endif
#include "cereal/visionipc/visionipc_client.h"
#include "selfdrive/camerad/cameras/camera_common.h"
#include "selfdrive/ui/ui.h"
@@ -41,11 +51,16 @@ protected:
bool zoomed_view;
GLuint frame_vao, frame_vbo, frame_ibo;
GLuint textures[3];
GLuint textures[2];
mat4 frame_mat;
std::unique_ptr<QOpenGLShaderProgram> program;
QColor bg = QColor("#000000");
#ifdef QCOM2
EGLDisplay egl_display;
std::map<int, EGLImageKHR> egl_images;
#endif
std::string stream_name;
int stream_width = 0;
int stream_height = 0;