Split cereal into cereal/msgq (#32631)

* squash

* fix doc

* compile device

* compile device

* Update ref

* add msgq to precommit exclusions

* No service ports

* fix compile

* address comments

* More comments

* Delete logger

* Update opendbc

* Linting

* bump msgq

selfdrive/controls/controlsd.py

@@ -8,7 +8,7 @@ from typing import SupportsFloat
 import cereal.messaging as messaging
 
 from cereal import car, log
-from cereal.visionipc import VisionIpcClient, VisionStreamType
+from msgq.visionipc import VisionIpcClient, VisionStreamType
 
 
 from openpilot.common.conversions import Conversions as CV

selfdrive/locationd/SConscript

@@ -1,6 +1,6 @@
-Import('env', 'arch', 'common', 'cereal', 'messaging', 'rednose', 'transformations')
+Import('env', 'arch', 'common', 'cereal', 'messaging', 'rednose', 'transformations', 'socketmaster')
 
-loc_libs = [cereal, messaging, 'zmq', common, 'capnp', 'kj', 'pthread', 'dl']
+loc_libs = [cereal, socketmaster, messaging, 'zmq', common, 'capnp', 'kj', 'pthread', 'dl']
 
 # build ekf models
 rednose_gen_dir = 'models/generated'

selfdrive/modeld/dmonitoringmodeld.py

@@ -9,7 +9,7 @@ from pathlib import Path
 
 from cereal import messaging
 from cereal.messaging import PubMaster, SubMaster
-from cereal.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
+from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_realtime_priority

selfdrive/modeld/modeld.py

@@ -8,7 +8,7 @@ from cereal import car, log
 from pathlib import Path
 from setproctitle import setproctitle
 from cereal.messaging import PubMaster, SubMaster
-from cereal.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
+from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
 from openpilot.common.filter_simple import FirstOrderFilter

selfdrive/modeld/models/commonmodel.pxd

@@ -1,6 +1,6 @@
 # distutils: language = c++
 
-from cereal.visionipc.visionipc cimport cl_device_id, cl_context, cl_mem
+from msgq.visionipc.visionipc cimport cl_device_id, cl_context, cl_mem
 
 cdef extern from "common/mat.h":
   cdef struct mat3:

selfdrive/modeld/models/commonmodel_pyx.pxd

@@ -1,7 +1,7 @@
 # distutils: language = c++
 
-from cereal.visionipc.visionipc cimport cl_mem
-from cereal.visionipc.visionipc_pyx cimport CLContext as BaseCLContext
+from msgq.visionipc.visionipc cimport cl_mem
+from msgq.visionipc.visionipc_pyx cimport CLContext as BaseCLContext
 
 cdef class CLContext(BaseCLContext):
   pass

selfdrive/modeld/models/commonmodel_pyx.pyx

@@ -5,8 +5,8 @@ import numpy as np
 cimport numpy as cnp
 from libc.string cimport memcpy
 
-from cereal.visionipc.visionipc cimport cl_mem
-from cereal.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext
+from msgq.visionipc.visionipc cimport cl_mem
+from msgq.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext
 from .commonmodel cimport CL_DEVICE_TYPE_DEFAULT, cl_get_device_id, cl_create_context
 from .commonmodel cimport mat3, sigmoid as cppSigmoid, ModelFrame as cppModelFrame
 

selfdrive/modeld/runners/snpemodel.pxd

@@ -2,7 +2,7 @@
 
 from libcpp.string cimport string
 
-from cereal.visionipc.visionipc cimport cl_context
+from msgq.visionipc.visionipc cimport cl_context
 
 cdef extern from "selfdrive/modeld/runners/snpemodel.h":
   cdef cppclass SNPEModel:

selfdrive/modeld/runners/thneedmodel.pxd

@@ -2,7 +2,7 @@
 
 from libcpp.string cimport string
 
-from cereal.visionipc.visionipc cimport cl_context
+from msgq.visionipc.visionipc cimport cl_context
 
 cdef extern from "selfdrive/modeld/runners/thneedmodel.h":
   cdef cppclass ThneedModel:

selfdrive/modeld/tests/test_modeld.py

@@ -2,7 +2,7 @@ import numpy as np
 import random
 
 import cereal.messaging as messaging
-from cereal.visionipc import VisionIpcServer, VisionStreamType
+from msgq.visionipc import VisionIpcServer, VisionStreamType
 from openpilot.common.transformations.camera import DEVICE_CAMERAS
 from openpilot.common.realtime import DT_MDL
 from openpilot.selfdrive.car.car_helpers import write_car_param

selfdrive/navd/SConscript

@@ -1,7 +1,7 @@
-Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transformations')
+Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transformations', 'socketmaster')
 
 map_env = qt_env.Clone()
-libs = ['qt_widgets', 'qt_util', 'QMapLibre', common, messaging, cereal, visionipc, transformations,
+libs = ['qt_widgets', 'qt_util', 'QMapLibre', common, socketmaster, messaging, cereal, visionipc, transformations,
         'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread', 'json11'] + map_env["LIBS"]
 if arch == 'larch64':
   libs.append(':libEGL_mesa.so.0')

selfdrive/navd/map_renderer.h

@@ -12,7 +12,7 @@
 #include <QOpenGLFunctions>
 #include <QOpenGLFramebufferObject>
 
-#include "cereal/visionipc/visionipc_server.h"
+#include "msgq/visionipc/visionipc_server.h"
 #include "cereal/messaging/messaging.h"
 
 

selfdrive/navd/tests/test_map_renderer.py

@@ -8,7 +8,7 @@ import http.server
 import cereal.messaging as messaging
 
 from typing import Any
-from cereal.visionipc import VisionIpcClient, VisionStreamType
+from msgq.visionipc import VisionIpcClient, VisionStreamType
 from openpilot.common.mock.generators import LLK_DECIMATION, LOCATION1, LOCATION2, generate_liveLocationKalman
 from openpilot.selfdrive.test.helpers import with_processes
 

selfdrive/pandad/SConscript

@@ -1,6 +1,6 @@
-Import('env', 'envCython', 'common', 'cereal', 'messaging')
+Import('env', 'envCython', 'common', 'cereal', 'messaging', 'socketmaster')
 
-libs = ['usb-1.0', common, cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj']
+libs = ['usb-1.0', common, cereal, messaging, socketmaster, 'pthread', 'zmq', 'capnp', 'kj']
 panda = env.Library('panda', ['panda.cc', 'panda_comms.cc', 'spi.cc'])
 
 env.Program('pandad', ['main.cc', 'pandad.cc'], LIBS=[panda] + libs)

selfdrive/test/process_replay/process_replay.py

@@ -16,7 +16,7 @@ import capnp
 import cereal.messaging as messaging
 from cereal import car
 from cereal.services import SERVICE_LIST
-from cereal.visionipc import VisionIpcServer, get_endpoint_name as vipc_get_endpoint_name
+from msgq.visionipc import VisionIpcServer, get_endpoint_name as vipc_get_endpoint_name
 from openpilot.common.params import Params
 from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.common.timeout import Timeout

selfdrive/test/process_replay/vision_meta.py

@@ -1,5 +1,5 @@
 from collections import namedtuple
-from cereal.visionipc import VisionStreamType
+from msgq.visionipc import VisionStreamType
 from openpilot.common.realtime import DT_MDL, DT_DMON
 from openpilot.common.transformations.camera import DEVICE_CAMERAS
 

selfdrive/ui/SConscript

@@ -1,9 +1,9 @@
 import os
 import json
 Import('qt_env', 'arch', 'common', 'messaging', 'visionipc',
-       'cereal', 'transformations')
+       'cereal', 'transformations', 'socketmaster')
 
-base_libs = [common, messaging, cereal, visionipc, transformations, 'zmq',
+base_libs = [common, socketmaster, messaging, cereal, visionipc, transformations, 'zmq',
              'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"]
 
 if arch == 'larch64':

selfdrive/ui/qt/widgets/cameraview.h

@@ -22,7 +22,7 @@
 #include <drm/drm_fourcc.h>
 #endif
 
-#include "cereal/visionipc/visionipc_client.h"
+#include "msgq/visionipc/visionipc_client.h"
 #include "system/camerad/cameras/camera_common.h"
 #include "selfdrive/ui/ui.h"
 

selfdrive/ui/tests/test_ui/run.py

@@ -10,7 +10,7 @@ import pywinctl
 import time
 
 from cereal import messaging, car, log
-from cereal.visionipc import VisionIpcServer, VisionStreamType
+from msgq.visionipc import VisionIpcServer, VisionStreamType
 
 from cereal.messaging import SubMaster, PubMaster
 from openpilot.common.mock import mock_messages