openpilot v0.9.3 release

date: 2023-06-16T05:25:00
master commit: 8704c1ff952b5c85a44f50143bbd1a4f7b4887e2

system/hardware/hw.h

@@ -19,7 +19,7 @@ inline std::string log_root() {
   return Hardware::PC() ? util::getenv("HOME") + "/.comma/media/0/realdata" : "/data/media/0/realdata";
 }
 inline std::string params() {
-  return Hardware::PC() ? util::getenv("HOME") + "/.comma/params" : "/data/params";
+  return Hardware::PC() ? util::getenv("PARAMS_ROOT", util::getenv("HOME") + "/.comma/params") : "/data/params";
 }
 inline std::string rsa_file() {
   return Hardware::PC() ? util::getenv("HOME") + "/.comma/persist/comma/id_rsa" : "/persist/comma/id_rsa";

system/hardware/tici/hardware.h

@@ -77,6 +77,7 @@ public:
   static std::map<std::string, std::string> get_init_logs() {
     std::map<std::string, std::string> ret = {
       {"/BUILD", util::read_file("/BUILD")},
+      {"lsblk", util::check_output("lsblk -o NAME,SIZE,STATE,VENDOR,MODEL,REV,SERIAL")},
     };
 
     std::string bs = util::check_output("abctl --boot_slot");

system/loggerd/encoder/encoder.cc

@@ -5,9 +5,7 @@ VideoEncoder::~VideoEncoder() {}
 
 void VideoEncoder::publisher_init() {
   // publish
-  service_name = this->type == DriverCam ? "driverEncodeData" :
-    (this->type == WideRoadCam ? "wideRoadEncodeData" :
-    (this->in_width == this->out_width ? "roadEncodeData" : "qRoadEncodeData"));
+  service_name = this->publish_name;
   pm.reset(new PubMaster({service_name}));
 }
 
@@ -38,45 +36,5 @@ void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t
   auto words = new kj::Array<capnp::word>(capnp::messageToFlatArray(msg));
   auto bytes = words->asBytes();
   e->pm->send(e->service_name, bytes.begin(), bytes.size());
-  if (e->write) {
-    e->to_write.push(words);
-  } else {
-    delete words;
-  }
-}
-
-// TODO: writing should be moved to loggerd
-void VideoEncoder::write_handler(VideoEncoder *e, const char *path) {
-  VideoWriter writer(path, e->filename, e->codec != cereal::EncodeIndex::Type::FULL_H_E_V_C, e->out_width, e->out_height, e->fps, e->codec);
-
-  bool first = true;
-  kj::Array<capnp::word>* out_buf;
-  while ((out_buf = e->to_write.pop())) {
-    capnp::FlatArrayMessageReader cmsg(*out_buf);
-    cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
-    auto edata = (e->type == DriverCam) ? event.getDriverEncodeData() :
-      ((e->type == WideRoadCam) ? event.getWideRoadEncodeData() :
-      (e->in_width == e->out_width ? event.getRoadEncodeData() : event.getQRoadEncodeData()));
-    auto idx = edata.getIdx();
-    auto flags = idx.getFlags();
-
-    if (first) {
-      assert(flags & V4L2_BUF_FLAG_KEYFRAME);
-      auto header = edata.getHeader();
-      writer.write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false);
-      first = false;
-    }
-
-    // dangerous cast from const, but should be fine
-    auto data = edata.getData();
-    if (data.size() > 0) {
-      writer.write((uint8_t *)data.begin(), data.size(), idx.getTimestampEof()/1000, false, flags & V4L2_BUF_FLAG_KEYFRAME);
-    }
-
-    // free the data
-    delete out_buf;
-  }
-
-  // VideoWriter is freed on out of scope
+  delete words;
 }

system/loggerd/encoder/encoder.h

@@ -7,7 +7,6 @@
 #include "cereal/messaging/messaging.h"
 #include "cereal/visionipc/visionipc.h"
 #include "common/queue.h"
-#include "system/loggerd/video_writer.h"
 #include "system/camerad/cameras/camera_common.h"
 
 #define V4L2_BUF_FLAG_KEYFRAME 8
@@ -15,9 +14,11 @@
 class VideoEncoder {
 public:
   VideoEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
-              int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write)
+              int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height,
+              const char* publish_name)
   : filename(filename), type(type), in_width(in_width), in_height(in_height), fps(fps),
-    bitrate(bitrate), codec(codec), out_width(out_width), out_height(out_height), write(write) { }
+    bitrate(bitrate), codec(codec), out_width(out_width), out_height(out_height),
+    publish_name(publish_name) { }
   virtual ~VideoEncoder();
   virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
   virtual void encoder_open(const char* path) = 0;
@@ -26,21 +27,10 @@ public:
   void publisher_init();
   static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
 
-  void writer_open(const char* path) {
-    if (this->write) write_handler_thread = std::thread(VideoEncoder::write_handler, this, path);
-  }
-
-  void writer_close() {
-    if (this->write) {
-      to_write.push(NULL);
-      write_handler_thread.join();
-    }
-    assert(to_write.empty());
-  }
 
 protected:
-  bool write;
   const char* filename;
+  const char* publish_name;
   int in_width, in_height;
   int out_width, out_height, fps;
   int bitrate;
@@ -54,9 +44,4 @@ private:
   // publishing
   std::unique_ptr<PubMaster> pm;
   const char *service_name;
-
-  // writing support
-  static void write_handler(VideoEncoder *e, const char *path);
-  std::thread write_handler_thread;
-  SafeQueue<kj::Array<capnp::word>* > to_write;
 };

system/loggerd/encoder/ffmpeg_encoder.cc

@@ -60,7 +60,6 @@ void FfmpegEncoder::encoder_open(const char* path) {
   int err = avcodec_open2(this->codec_ctx, codec, NULL);
   assert(err >= 0);
 
-  writer_open(path);
   is_open = true;
   segment_num++;
   counter = 0;
@@ -69,7 +68,6 @@ void FfmpegEncoder::encoder_open(const char* path) {
 void FfmpegEncoder::encoder_close() {
   if (!is_open) return;
 
-  writer_close();
   avcodec_free_context(&codec_ctx);
   is_open = false;
 }

system/loggerd/encoder/ffmpeg_encoder.h

@@ -17,8 +17,10 @@ extern "C" {
 class FfmpegEncoder : public VideoEncoder {
  public:
   FfmpegEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
-                int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) :
-                VideoEncoder(filename, type, in_width, in_height, fps, bitrate, cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS, out_width, out_height, write) { encoder_init(); }
+
+                int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height,
+                const char* publish_name) :
+                VideoEncoder(filename, type, in_width, in_height, fps, bitrate, cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS, out_width, out_height, publish_name) { encoder_init(); }
   ~FfmpegEncoder();
   void encoder_init();
   int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);

system/loggerd/encoder/v4l_encoder.cc

@@ -255,7 +255,6 @@ void V4LEncoder::encoder_init() {
 
 void V4LEncoder::encoder_open(const char* path) {
   dequeue_handler_thread = std::thread(V4LEncoder::dequeue_handler, this);
-  writer_open(path);
   this->is_open = true;
   this->counter = 0;
 }
@@ -288,7 +287,6 @@ void V4LEncoder::encoder_close() {
     // join waits for V4L2_QCOM_BUF_FLAG_EOS
     dequeue_handler_thread.join();
     assert(extras.empty());
-    writer_close();
   }
   this->is_open = false;
 }

system/loggerd/encoder/v4l_encoder.h

@@ -9,8 +9,8 @@
 class V4LEncoder : public VideoEncoder {
 public:
   V4LEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
-             int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) :
-             VideoEncoder(filename, type, in_width, in_height, fps, bitrate, codec, out_width, out_height, write) { encoder_init(); }
+             int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, const char* publish_name) :
+             VideoEncoder(filename, type, in_width, in_height, fps, bitrate, codec, out_width, out_height, publish_name) { encoder_init(); }
   ~V4LEncoder();
   void encoder_init();
   int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);

system/loggerd/encoderd.cc

@@ -35,7 +35,7 @@ bool sync_encoders(EncoderdState *s, CameraType cam_type, uint32_t frame_id) {
 
 
 void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
-  util::set_thread_name(cam_info.filename);
+  util::set_thread_name(cam_info.thread_name);
 
   std::vector<Encoder *> encoders;
   VisionIpcClient vipc_client = VisionIpcClient("camerad", cam_info.stream_type, false);
@@ -50,20 +50,15 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
     // init encoders
     if (encoders.empty()) {
       VisionBuf buf_info = vipc_client.buffers[0];
-      LOGW("encoder %s init %dx%d", cam_info.filename, buf_info.width, buf_info.height);
+      LOGW("encoder %s init %dx%d", cam_info.thread_name, buf_info.width, buf_info.height);
 
       if (buf_info.width > 0 && buf_info.height > 0) {
-        // main encoder
-        encoders.push_back(new Encoder(cam_info.filename, cam_info.type, buf_info.width, buf_info.height,
-                                      cam_info.fps, cam_info.bitrate,
-                                      cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264,
-                                      buf_info.width, buf_info.height, false));
-        // qcamera encoder
-        if (cam_info.has_qcamera) {
-          encoders.push_back(new Encoder(qcam_info.filename, cam_info.type, buf_info.width, buf_info.height,
-                                        qcam_info.fps, qcam_info.bitrate,
-                                        qcam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264,
-                                        qcam_info.frame_width, qcam_info.frame_height, false));
+        for (const auto &encoder_info: cam_info.encoder_infos){
+          encoders.push_back(new Encoder(encoder_info.filename, cam_info.type, buf_info.width, buf_info.height,
+                                        encoder_info.fps, encoder_info.bitrate,
+                                        encoder_info.encode_type,
+                                        encoder_info.frame_width, encoder_info.frame_height,
+                                        encoder_info.publish_name));
         }
       } else {
         LOGE("not initting empty encoder");
@@ -85,7 +80,7 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
       // detect loop around and drop the frames
       if (buf->get_frame_id() != extra.frame_id) {
         if (!lagging) {
-          LOGE("encoder %s lag  buffer id: %d  extra id: %d", cam_info.filename, buf->get_frame_id(), extra.frame_id);
+          LOGE("encoder %s lag  buffer id: %d  extra id: %d", cam_info.thread_name, buf->get_frame_id(), extra.frame_id);
           lagging = true;
         }
         continue;

system/loggerd/loggerd.cc

@@ -58,18 +58,13 @@ struct RemoteEncoder {
   bool seen_first_packet = false;
 };
 
-int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct RemoteEncoder &re) {
-  const LogCameraInfo &cam_info = (name == "driverEncodeData") ? cameras_logged[1] :
-    ((name == "wideRoadEncodeData") ? cameras_logged[2] :
-    ((name == "qRoadEncodeData") ? qcam_info : cameras_logged[0]));
+int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct RemoteEncoder &re, EncoderInfo encoder_info) {
   int bytes_count = 0;
 
   // extract the message
   capnp::FlatArrayMessageReader cmsg(kj::ArrayPtr<capnp::word>((capnp::word *)msg->getData(), msg->getSize() / sizeof(capnp::word)));
   auto event = cmsg.getRoot<cereal::Event>();
-  auto edata = (name == "driverEncodeData") ? event.getDriverEncodeData() :
-    ((name == "wideRoadEncodeData") ? event.getWideRoadEncodeData() :
-    ((name == "qRoadEncodeData") ? event.getQRoadEncodeData() : event.getRoadEncodeData()));
+  auto edata = (event.*(encoder_info.get_encode_data_func))();
   auto idx = edata.getIdx();
   auto flags = idx.getFlags();
 
@@ -95,7 +90,7 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
       // we are in this segment now, process any queued messages before this one
       if (!re.q.empty()) {
         for (auto &qmsg: re.q) {
-          bytes_count += handle_encoder_msg(s, qmsg, name, re);
+          bytes_count += handle_encoder_msg(s, qmsg, name, re, encoder_info);
         }
         re.q.clear();
       }
@@ -111,10 +106,10 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
           re.dropped_frames = 0;
         }
         // if we aren't actually recording, don't create the writer
-        if (cam_info.record) {
+        if (encoder_info.record) {
           re.writer.reset(new VideoWriter(s->segment_path,
-            cam_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C,
-            cam_info.frame_width, cam_info.frame_height, cam_info.fps, idx.getType()));
+            encoder_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C,
+            encoder_info.frame_width, encoder_info.frame_height, encoder_info.fps, idx.getType()));
           // write the header
           auto header = edata.getHeader();
           re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false);
@@ -142,10 +137,7 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
     MessageBuilder bmsg;
     auto evt = bmsg.initEvent(event.getValid());
     evt.setLogMonoTime(event.getLogMonoTime());
-    if (name == "driverEncodeData") { evt.setDriverEncodeIdx(idx); }
-    if (name == "wideRoadEncodeData") { evt.setWideRoadEncodeIdx(idx); }
-    if (name == "qRoadEncodeData") { evt.setQRoadEncodeIdx(idx); }
-    if (name == "roadEncodeData") { evt.setRoadEncodeIdx(idx); }
+    (evt.*(encoder_info.set_encode_idx_func))(idx);
     auto new_msg = bmsg.toBytes();
     logger_log(&s->logger, (uint8_t *)new_msg.begin(), new_msg.size(), true);   // always in qlog?
     bytes_count += new_msg.size();
@@ -211,11 +203,12 @@ void loggerd_thread() {
   logger_rotate(&s);
   Params().put("CurrentRoute", s.logger.route_name);
 
-  // init encoders
-  s.last_camera_seen_tms = millis_since_boot();
+  std::map<std::string, EncoderInfo> encoder_infos_dict;
   for (const auto &cam : cameras_logged) {
-    s.max_waiting++;
-    if (cam.has_qcamera) { s.max_waiting++; }
+    for (const auto &encoder_info: cam.encoder_infos) {
+      encoder_infos_dict[encoder_info.publish_name] = encoder_info;
+      s.max_waiting++;
+    }
   }
 
   uint64_t msg_count = 0, bytes_count = 0;
@@ -234,7 +227,7 @@ void loggerd_thread() {
 
         if (qs.encoder) {
           s.last_camera_seen_tms = millis_since_boot();
-          bytes_count += handle_encoder_msg(&s, msg, qs.name, remote_encoders[sock]);
+          bytes_count += handle_encoder_msg(&s, msg, qs.name, remote_encoders[sock], encoder_infos_dict[qs.name]);
         } else {
           logger_log(&s.logger, (uint8_t *)msg->getData(), msg->getSize(), in_qlog);
           bytes_count += msg->getSize();

system/loggerd/loggerd.h

@@ -35,69 +35,87 @@
 
 constexpr int MAIN_FPS = 20;
 const int MAIN_BITRATE = 10000000;
-const int DCAM_BITRATE = MAIN_BITRATE;
 
 #define NO_CAMERA_PATIENCE 500 // fall back to time-based rotation if all cameras are dead
 
 const bool LOGGERD_TEST = getenv("LOGGERD_TEST");
 const int SEGMENT_LENGTH = LOGGERD_TEST ? atoi(getenv("LOGGERD_SEGMENT_LENGTH")) : 60;
 
-struct LogCameraInfo {
-  CameraType type;
+class EncoderInfo {
+public:
+  const char *publish_name;
   const char *filename;
-  VisionStreamType stream_type;
-  int frame_width, frame_height;
-  int fps;
-  int bitrate;
-  bool is_h265;
-  bool has_qcamera;
-  bool record;
+  bool record = true;
+  int frame_width = 1928;
+  int frame_height = 1208;
+  int fps = MAIN_FPS;
+  int bitrate = MAIN_BITRATE;
+  cereal::EncodeIndex::Type encode_type = cereal::EncodeIndex::Type::FULL_H_E_V_C;
+  ::cereal::EncodeData::Reader (cereal::Event::Reader::*get_encode_data_func)() const;
+  void (cereal::Event::Builder::*set_encode_idx_func)(::cereal::EncodeIndex::Reader);
 };
 
-const LogCameraInfo cameras_logged[] = {
-  {
-    .type = RoadCam,
-    .stream_type = VISION_STREAM_ROAD,
-    .filename = "fcamera.hevc",
-    .fps = MAIN_FPS,
-    .bitrate = MAIN_BITRATE,
-    .is_h265 = true,
-    .has_qcamera = true,
-    .record = true,
-    .frame_width = 1928,
-    .frame_height = 1208,
-  },
-  {
-    .type = DriverCam,
-    .stream_type = VISION_STREAM_DRIVER,
-    .filename = "dcamera.hevc",
-    .fps = MAIN_FPS,
-    .bitrate = DCAM_BITRATE,
-    .is_h265 = true,
-    .has_qcamera = false,
-    .record = Params().getBool("RecordFront"),
-    .frame_width = 1928,
-    .frame_height = 1208,
-  },
-  {
-    .type = WideRoadCam,
-    .stream_type = VISION_STREAM_WIDE_ROAD,
-    .filename = "ecamera.hevc",
-    .fps = MAIN_FPS,
-    .bitrate = MAIN_BITRATE,
-    .is_h265 = true,
-    .has_qcamera = false,
-    .record = true,
-    .frame_width = 1928,
-    .frame_height = 1208,
-  },
+class LogCameraInfo {
+public:
+  const char *thread_name;
+  int fps = MAIN_FPS;
+  CameraType type;
+  VisionStreamType stream_type;
+  std::vector<EncoderInfo> encoder_infos;
 };
-const LogCameraInfo qcam_info = {
+
+const EncoderInfo main_road_encoder_info = {
+  .publish_name = "roadEncodeData",
+  .filename = "fcamera.hevc",
+  .get_encode_data_func = &cereal::Event::Reader::getRoadEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setRoadEncodeIdx,
+};
+const EncoderInfo main_wide_road_encoder_info = {
+  .publish_name = "wideRoadEncodeData",
+  .filename = "ecamera.hevc",
+  .get_encode_data_func = &cereal::Event::Reader::getWideRoadEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setWideRoadEncodeIdx,
+};
+const EncoderInfo main_driver_encoder_info = {
+   .publish_name = "driverEncodeData",
+  .filename = "dcamera.hevc",
+  .record = Params().getBool("RecordFront"),
+  .get_encode_data_func = &cereal::Event::Reader::getDriverEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setDriverEncodeIdx,
+};
+
+const EncoderInfo qcam_encoder_info = {
+  .publish_name = "qRoadEncodeData",
   .filename = "qcamera.ts",
-  .fps = MAIN_FPS,
   .bitrate = 256000,
-  .is_h265 = false,
-  .record = true,
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
   .frame_width = 526,
   .frame_height = 330,
+  .get_encode_data_func = &cereal::Event::Reader::getQRoadEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setQRoadEncodeIdx,
 };
+
+
+const LogCameraInfo road_camera_info{
+    .thread_name = "road_cam_encoder",
+    .type = RoadCam,
+    .stream_type = VISION_STREAM_ROAD,
+    .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
+    };
+
+const LogCameraInfo wide_road_camera_info{
+    .thread_name = "wide_road_cam_encoder",
+    .type = WideRoadCam,
+    .stream_type = VISION_STREAM_WIDE_ROAD,
+   .encoder_infos = {main_wide_road_encoder_info}
+    };
+  
+const LogCameraInfo driver_camera_info{
+    .thread_name = "driver_cam_encoder",
+    .type = DriverCam,
+    .stream_type = VISION_STREAM_DRIVER,
+    .encoder_infos = {main_driver_encoder_info}
+    };
+
+const LogCameraInfo cameras_logged[] = {road_camera_info, wide_road_camera_info, driver_camera_info};
+

system/logmessaged.py

@@ -12,7 +12,7 @@ def main() -> NoReturn:
   log_handler.setFormatter(SwagLogFileFormatter(None))
   log_level = 20  # logging.INFO
 
-  ctx = zmq.Context().instance()
+  ctx = zmq.Context.instance()
   sock = ctx.socket(zmq.PULL)
   sock.bind("ipc:///tmp/logmessage")
 
@@ -20,23 +20,37 @@ def main() -> NoReturn:
   log_message_sock = messaging.pub_sock('logMessage')
   error_log_message_sock = messaging.pub_sock('errorLogMessage')
 
-  while True:
-    dat = b''.join(sock.recv_multipart())
-    level = dat[0]
-    record = dat[1:].decode("utf-8")
-    if level >= log_level:
-      log_handler.emit(record)
+  try:
+    while True:
+      dat = b''.join(sock.recv_multipart())
+      level = dat[0]
+      record = dat[1:].decode("utf-8")
+      if level >= log_level:
+        log_handler.emit(record)
 
-    # then we publish them
-    msg = messaging.new_message()
-    msg.logMessage = record
-    log_message_sock.send(msg.to_bytes())
+      if len(record) > 2*1024*1024:
+        print("WARNING: log too big to publish", len(record))
+        print(print(record[:100]))
+        continue
 
-    if level >= 40:  # logging.ERROR
+      # then we publish them
       msg = messaging.new_message()
-      msg.errorLogMessage = record
-      error_log_message_sock.send(msg.to_bytes())
+      msg.logMessage = record
+      log_message_sock.send(msg.to_bytes())
 
+      if level >= 40:  # logging.ERROR
+        msg = messaging.new_message()
+        msg.errorLogMessage = record
+        error_log_message_sock.send(msg.to_bytes())
+  finally:
+    sock.close()
+    ctx.term()
+
+    # can hit this if interrupted during a rollover
+    try:
+      log_handler.close()
+    except ValueError:
+      pass
 
 if __name__ == "__main__":
   main()

system/sensord/rawgps/rawgpsd.py

@@ -5,7 +5,9 @@ import signal
 import itertools
 import math
 import time
+import pycurl
 import subprocess
+from datetime import datetime
 from typing import NoReturn
 from struct import unpack_from, calcsize, pack
 
@@ -107,6 +109,54 @@ def gps_enabled() -> bool:
   except subprocess.CalledProcessError as exc:
     raise Exception("failed to execute QGPS mmcli command") from exc
 
+def download_and_inject_assistance():
+  assist_data_file = '/tmp/xtra3grc.bin'
+  assistance_url = 'http://xtrapath3.izatcloud.net/xtra3grc.bin'
+
+  try:
+    # download assistance
+    try:
+      c = pycurl.Curl()
+      c.setopt(pycurl.URL, assistance_url)
+      c.setopt(pycurl.NOBODY, 1)
+      c.setopt(pycurl.CONNECTTIMEOUT, 2)
+      c.perform()
+      bytes_n = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
+      c.close()
+      if bytes_n > 1e5:
+        cloudlog.error("Qcom assistance data larger than expected")
+        return
+
+      with open(assist_data_file, 'wb') as fp:
+        c = pycurl.Curl()
+        c.setopt(pycurl.URL, assistance_url)
+        c.setopt(pycurl.CONNECTTIMEOUT, 5)
+
+        c.setopt(pycurl.WRITEDATA, fp)
+        c.perform()
+        c.close()
+    except pycurl.error:
+      cloudlog.exception("Failed to download assistance file")
+      return
+
+    # inject into module
+    try:
+      cmd = f"mmcli -m any --timeout 30 --location-inject-assistance-data={assist_data_file}"
+      subprocess.check_output(cmd, stderr=subprocess.PIPE, shell=True)
+      cloudlog.info("successfully loaded assistance data")
+    except subprocess.CalledProcessError as e:
+      cloudlog.event(
+        "rawgps.assistance_loading_failed",
+        error=True,
+        cmd=e.cmd,
+        output=e.output,
+        returncode=e.returncode
+      )
+  finally:
+    if os.path.exists(assist_data_file):
+      os.remove(assist_data_file)
+
+
 def setup_quectel(diag: ModemDiag):
   # enable OEMDRE in the NV
   # TODO: it has to reboot for this to take effect
@@ -120,13 +170,20 @@ def setup_quectel(diag: ModemDiag):
 
   if gps_enabled():
     at_cmd("AT+QGPSEND")
+  #at_cmd("AT+QGPSDEL=0")
 
   # disable DPO power savings for more accuracy
   at_cmd("AT+QGPSCFG=\"dpoenable\",0")
   # don't automatically turn on GNSS on powerup
   at_cmd("AT+QGPSCFG=\"autogps\",0")
 
-  at_cmd("AT+QGPSSUPLURL=\"supl.google.com:7275\"")
+  # Do internet assistance
+  at_cmd("AT+QGPSXTRA=1")
+  download_and_inject_assistance()
+  #at_cmd("AT+QGPSXTRADATA?")
+  time_str = datetime.utcnow().strftime("%Y/%m/%d,%H:%M:%S")
+  at_cmd(f"AT+QGPSXTRATIME=0,\"{time_str}\",1,1,1000")
+  
   at_cmd("AT+QGPSCFG=\"outport\",\"usbnmea\"")
   at_cmd("AT+QGPS=1")
 

system/ubloxd/ublox_msg.cc

@@ -112,8 +112,7 @@ std::pair<std::string, kj::Array<capnp::word>> UbloxMsgParser::gen_msg() {
   case 0x0a0b:
     return {"ubloxGnss", gen_mon_hw2(static_cast<ubx_t::mon_hw2_t*>(body))};
   case 0x0135:
-    // TODO return {"ubloxGnss", gen_nav_sat(static_cast<ubx_t::nav_sat_t*>(body))};
-    return {"ubloxGnss", kj::Array<capnp::word>()};
+    return {"ubloxGnss", gen_nav_sat(static_cast<ubx_t::nav_sat_t*>(body))};
   default:
     LOGE("Unknown message type %x", ubx_message.msg_type());
     return {"ubloxGnss", kj::Array<capnp::word>()};