dragonpilot beta3

date: 2023-07-26T22:20:36
commit: c6d842c412052be1985b63d683c63be9dcb2b0eb

system/loggerd/config.py

@@ -5,7 +5,7 @@ from system.hardware import PC
 if os.environ.get('LOG_ROOT', False):
   ROOT = os.environ['LOG_ROOT']
 elif PC:
-  ROOT = os.path.join(str(Path.home()), ".comma", "media", "0", "realdata")
+  ROOT = str(Path.home() / ".comma" / "media" / "0" / "realdata")
 else:
   ROOT = '/data/media/0/realdata/'
 
@@ -16,7 +16,7 @@ SEGMENT_LENGTH = 60
 STATS_DIR_FILE_LIMIT = 10000
 STATS_SOCKET = "ipc:///tmp/stats"
 if PC:
-  STATS_DIR = os.path.join(str(Path.home()), ".comma", "stats")
+  STATS_DIR = str(Path.home() / ".comma" / "stats")
 else:
   STATS_DIR = "/data/stats/"
 STATS_FLUSH_TIME_S = 60

system/loggerd/deleter.py

@@ -2,15 +2,48 @@
 import os
 import shutil
 import threading
+from typing import List
+
 from system.swaglog import cloudlog
 from system.loggerd.config import ROOT, get_available_bytes, get_available_percent
 from system.loggerd.uploader import listdir_by_creation
+from system.loggerd.xattr_cache import getxattr
 
 MIN_BYTES = 5 * 1024 * 1024 * 1024
 MIN_PERCENT = 10
 
 DELETE_LAST = ['boot', 'crash']
 
+PRESERVE_ATTR_NAME = 'user.preserve'
+PRESERVE_ATTR_VALUE = b'1'
+PRESERVE_COUNT = 5
+
+
+def has_preserve_xattr(d: str) -> bool:
+  return getxattr(os.path.join(ROOT, d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+
+def get_preserved_segments(dirs_by_creation: List[str]) -> List[str]:
+  preserved = []
+  for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
+    if n == PRESERVE_COUNT:
+      break
+    date_str, _, seg_str = d.rpartition("--")
+
+    # ignore non-segment directories
+    if not date_str:
+      continue
+    try:
+      seg_num = int(seg_str)
+    except ValueError:
+      continue
+
+    # preserve segment and its prior
+    preserved.append(d)
+    preserved.append(f"{date_str}--{seg_num - 1}")
+
+  return preserved
+
 
 def deleter_thread(exit_event):
   while not exit_event.is_set():
@@ -18,9 +51,13 @@ def deleter_thread(exit_event):
     out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
 
     if out_of_percent or out_of_bytes:
+      dirs = listdir_by_creation(ROOT)
+
+      # skip deleting most recent N preserved segments (and their prior segment)
+      preserved_dirs = get_preserved_segments(dirs)
+
       # remove the earliest directory we can
-      dirs = sorted(listdir_by_creation(ROOT), key=lambda x: x in DELETE_LAST)
-      for delete_dir in dirs:
+      for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
         delete_path = os.path.join(ROOT, delete_dir)
 
         if any(name.endswith(".lock") for name in os.listdir(delete_path)):

system/loggerd/encoder/encoder.h

@@ -8,40 +8,27 @@
 #include "cereal/visionipc/visionipc.h"
 #include "common/queue.h"
 #include "system/camerad/cameras/camera_common.h"
+#include "system/loggerd/loggerd.h"
 
 #define V4L2_BUF_FLAG_KEYFRAME 8
 
 class VideoEncoder {
 public:
-  VideoEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
-              int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height,
-              const char* publish_name)
-  : filename(filename), type(type), in_width(in_width), in_height(in_height), fps(fps),
-    bitrate(bitrate), codec(codec), out_width(out_width), out_height(out_height),
-    publish_name(publish_name) { }
-  virtual ~VideoEncoder();
+  VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  virtual ~VideoEncoder() {};
   virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
   virtual void encoder_open(const char* path) = 0;
   virtual void encoder_close() = 0;
 
-  void publisher_init();
   static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
 
 
 protected:
-  const char* filename;
-  const char* publish_name;
   int in_width, in_height;
-  int out_width, out_height, fps;
-  int bitrate;
-  cereal::EncodeIndex::Type codec;
-  CameraType type;
+  const EncoderInfo encoder_info;
 
 private:
   // total frames encoded
   int cnt = 0;
-
-  // publishing
   std::unique_ptr<PubMaster> pm;
-  const char *service_name;
 };

system/loggerd/encoder/ffmpeg_encoder.h

@@ -15,14 +15,9 @@ extern "C" {
 #include "system/loggerd/loggerd.h"
 
 class FfmpegEncoder : public VideoEncoder {
- public:
-  FfmpegEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
-
-                int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height,
-                const char* publish_name) :
-                VideoEncoder(filename, type, in_width, in_height, fps, bitrate, cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS, out_width, out_height, publish_name) { encoder_init(); }
+public:
+  FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
   ~FfmpegEncoder();
-  void encoder_init();
   int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
   void encoder_open(const char* path);
   void encoder_close();

system/loggerd/encoder/v4l_encoder.h

@@ -8,11 +8,8 @@
 
 class V4LEncoder : public VideoEncoder {
 public:
-  V4LEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
-             int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, const char* publish_name) :
-             VideoEncoder(filename, type, in_width, in_height, fps, bitrate, codec, out_width, out_height, publish_name) { encoder_init(); }
+  V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
   ~V4LEncoder();
-  void encoder_init();
   int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
   void encoder_open(const char* path);
   void encoder_close();

system/loggerd/loggerd.h

@@ -1,46 +1,32 @@
 #pragma once
 
-#include <unistd.h>
-
-#include <atomic>
-#include <cassert>
-#include <cerrno>
-#include <condition_variable>
-#include <mutex>
-#include <string>
-#include <thread>
-#include <unordered_map>
-#include <utility>
-
 #include "cereal/messaging/messaging.h"
 #include "cereal/services.h"
-#include "cereal/visionipc/visionipc.h"
 #include "cereal/visionipc/visionipc_client.h"
 #include "system/camerad/cameras/camera_common.h"
+#include "system/hardware/hw.h"
 #include "common/params.h"
 #include "common/swaglog.h"
-#include "common/timing.h"
 #include "common/util.h"
-#include "system/hardware/hw.h"
 
-#include "system/loggerd/encoder/encoder.h"
 #include "system/loggerd/logger.h"
-#ifdef QCOM2
-#include "system/loggerd/encoder/v4l_encoder.h"
-#define Encoder V4LEncoder
-#else
-#include "system/loggerd/encoder/ffmpeg_encoder.h"
-#define Encoder FfmpegEncoder
-#endif
 
 constexpr int MAIN_FPS = 20;
 const int MAIN_BITRATE = 10000000;
 
-#define NO_CAMERA_PATIENCE 500 // fall back to time-based rotation if all cameras are dead
+#define NO_CAMERA_PATIENCE 500  // fall back to time-based rotation if all cameras are dead
+
+#define INIT_ENCODE_FUNCTIONS(encode_type)                                \
+  .get_encode_data_func = &cereal::Event::Reader::get##encode_type##Data, \
+  .set_encode_idx_func = &cereal::Event::Builder::set##encode_type##Idx,  \
+  .init_encode_data_func = &cereal::Event::Builder::init##encode_type##Data
 
 const bool LOGGERD_TEST = getenv("LOGGERD_TEST");
 const int SEGMENT_LENGTH = LOGGERD_TEST ? atoi(getenv("LOGGERD_SEGMENT_LENGTH")) : 60;
 
+constexpr char PRESERVE_ATTR_NAME[] = "user.preserve";
+constexpr char PRESERVE_ATTR_VALUE = '1';
+
 class EncoderInfo {
 public:
   const char *publish_name;
@@ -50,9 +36,11 @@ public:
   int frame_height = 1208;
   int fps = MAIN_FPS;
   int bitrate = MAIN_BITRATE;
-  cereal::EncodeIndex::Type encode_type = cereal::EncodeIndex::Type::FULL_H_E_V_C;
+  cereal::EncodeIndex::Type encode_type = Hardware::PC() ? cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS
+                                                         : cereal::EncodeIndex::Type::FULL_H_E_V_C;
   ::cereal::EncodeData::Reader (cereal::Event::Reader::*get_encode_data_func)() const;
   void (cereal::Event::Builder::*set_encode_idx_func)(::cereal::EncodeIndex::Reader);
+  cereal::EncodeData::Builder (cereal::Event::Builder::*init_encode_data_func)();
 };
 
 class LogCameraInfo {
@@ -67,21 +55,18 @@ public:
 const EncoderInfo main_road_encoder_info = {
   .publish_name = "roadEncodeData",
   .filename = "fcamera.hevc",
-  .get_encode_data_func = &cereal::Event::Reader::getRoadEncodeData,
-  .set_encode_idx_func = &cereal::Event::Builder::setRoadEncodeIdx,
+  INIT_ENCODE_FUNCTIONS(RoadEncode),
 };
 const EncoderInfo main_wide_road_encoder_info = {
   .publish_name = "wideRoadEncodeData",
   .filename = "ecamera.hevc",
-  .get_encode_data_func = &cereal::Event::Reader::getWideRoadEncodeData,
-  .set_encode_idx_func = &cereal::Event::Builder::setWideRoadEncodeIdx,
+  INIT_ENCODE_FUNCTIONS(WideRoadEncode),
 };
 const EncoderInfo main_driver_encoder_info = {
-   .publish_name = "driverEncodeData",
+  .publish_name = "driverEncodeData",
   .filename = "dcamera.hevc",
   .record = Params().getBool("RecordFront"),
-  .get_encode_data_func = &cereal::Event::Reader::getDriverEncodeData,
-  .set_encode_idx_func = &cereal::Event::Builder::setDriverEncodeIdx,
+  INIT_ENCODE_FUNCTIONS(DriverEncode),
 };
 
 const EncoderInfo qcam_encoder_info = {
@@ -91,31 +76,29 @@ const EncoderInfo qcam_encoder_info = {
   .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
   .frame_width = 526,
   .frame_height = 330,
-  .get_encode_data_func = &cereal::Event::Reader::getQRoadEncodeData,
-  .set_encode_idx_func = &cereal::Event::Builder::setQRoadEncodeIdx,
+  INIT_ENCODE_FUNCTIONS(QRoadEncode),
 };
 
 
 const LogCameraInfo road_camera_info{
-    .thread_name = "road_cam_encoder",
-    .type = RoadCam,
-    .stream_type = VISION_STREAM_ROAD,
-    .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
-    };
+  .thread_name = "road_cam_encoder",
+  .type = RoadCam,
+  .stream_type = VISION_STREAM_ROAD,
+  .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
+};
 
 const LogCameraInfo wide_road_camera_info{
-    .thread_name = "wide_road_cam_encoder",
-    .type = WideRoadCam,
-    .stream_type = VISION_STREAM_WIDE_ROAD,
-   .encoder_infos = {main_wide_road_encoder_info}
-    };
-  
+  .thread_name = "wide_road_cam_encoder",
+  .type = WideRoadCam,
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {main_wide_road_encoder_info}
+};
+
 const LogCameraInfo driver_camera_info{
-    .thread_name = "driver_cam_encoder",
-    .type = DriverCam,
-    .stream_type = VISION_STREAM_DRIVER,
-    .encoder_infos = {main_driver_encoder_info}
-    };
+  .thread_name = "driver_cam_encoder",
+  .type = DriverCam,
+  .stream_type = VISION_STREAM_DRIVER,
+  .encoder_infos = {main_driver_encoder_info}
+};
 
 const LogCameraInfo cameras_logged[] = {road_camera_info, wide_road_camera_info, driver_camera_info};
-

system/loggerd/uploader.py

@@ -88,9 +88,6 @@ class Uploader:
     self.immediate_folders = ["crash/", "boot/"]
     self.immediate_priority = {"qlog": 0, "qlog.bz2": 0, "qcamera.ts": 1}
 
-    # dp
-    self.update_onced = False
-
   def get_upload_sort(self, name: str) -> int:
     if name in self.immediate_priority:
       return self.immediate_priority[name]
@@ -216,10 +213,6 @@ class Uploader:
           self.last_speed = (content_length / 1e6) / self.last_time
           cloudlog.event("upload_success", key=key, fn=fn, sz=sz, content_length=content_length, network_type=network_type, metered=metered, speed=self.last_speed)
         success = True
-        if not self.update_onced and stat.status_code != 412:
-          Params().put_bool('dp_upload_ignored', False)
-          self.update_onced = True
-        cloudlog.event("upload_success" if stat.status_code != 412 else "upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
       else:
         success = False
         cloudlog.event("upload_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)

system/loggerd/xattr_cache.py

@@ -1,6 +1,6 @@
 import os
 import errno
-from typing import Dict, Tuple, Optional
+from typing import Dict, Optional, Tuple
 
 _cached_attributes: Dict[Tuple, Optional[bytes]] = {}