dragonpilot v2023.07.05

version: dragonpilot v0.9.4 release
date: 2023-07-05T18:59:41
dp-dev(priv) master commit: 7b0489feab40283a422d2201ef95a9cb8c06f6cd

system/loggerd/.gitignore

@@ -0,0 +1,4 @@
+loggerd
+encoderd
+bootlog
+tests/test_logger

system/loggerd/config.py

@@ -0,0 +1,41 @@
+import os
+from pathlib import Path
+from system.hardware import PC
+
+if os.environ.get('LOG_ROOT', False):
+  ROOT = os.environ['LOG_ROOT']
+elif PC:
+  ROOT = os.path.join(str(Path.home()), ".comma", "media", "0", "realdata")
+else:
+  ROOT = '/data/media/0/realdata/'
+
+
+CAMERA_FPS = 20
+SEGMENT_LENGTH = 60
+
+STATS_DIR_FILE_LIMIT = 10000
+STATS_SOCKET = "ipc:///tmp/stats"
+if PC:
+  STATS_DIR = os.path.join(str(Path.home()), ".comma", "stats")
+else:
+  STATS_DIR = "/data/stats/"
+STATS_FLUSH_TIME_S = 60
+
+def get_available_percent(default=None):
+  try:
+    statvfs = os.statvfs(ROOT)
+    available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
+  except OSError:
+    available_percent = default
+
+  return available_percent
+
+
+def get_available_bytes(default=None):
+  try:
+    statvfs = os.statvfs(ROOT)
+    available_bytes = statvfs.f_bavail * statvfs.f_frsize
+  except OSError:
+    available_bytes = default
+
+  return available_bytes

system/loggerd/deleter.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import threading
+from system.swaglog import cloudlog
+from system.loggerd.config import ROOT, get_available_bytes, get_available_percent
+from system.loggerd.uploader import listdir_by_creation
+
+MIN_BYTES = 5 * 1024 * 1024 * 1024
+MIN_PERCENT = 10
+
+DELETE_LAST = ['boot', 'crash']
+
+
+def deleter_thread(exit_event):
+  while not exit_event.is_set():
+    out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
+    out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
+
+    if out_of_percent or out_of_bytes:
+      # remove the earliest directory we can
+      dirs = sorted(listdir_by_creation(ROOT), key=lambda x: x in DELETE_LAST)
+      for delete_dir in dirs:
+        delete_path = os.path.join(ROOT, delete_dir)
+
+        if any(name.endswith(".lock") for name in os.listdir(delete_path)):
+          continue
+
+        try:
+          cloudlog.info(f"deleting {delete_path}")
+          if os.path.isfile(delete_path):
+            os.remove(delete_path)
+          else:
+            shutil.rmtree(delete_path)
+          break
+        except OSError:
+          cloudlog.exception(f"issue deleting {delete_path}")
+      exit_event.wait(.1)
+    else:
+      exit_event.wait(30)
+
+
+def main():
+  deleter_thread(threading.Event())
+
+
+if __name__ == "__main__":
+  main()

system/loggerd/encoder/encoder.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <thread>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/visionipc/visionipc.h"
+#include "common/queue.h"
+#include "system/camerad/cameras/camera_common.h"
+
+#define V4L2_BUF_FLAG_KEYFRAME 8
+
+class VideoEncoder {
+public:
+  VideoEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
+              int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height,
+              const char* publish_name)
+  : filename(filename), type(type), in_width(in_width), in_height(in_height), fps(fps),
+    bitrate(bitrate), codec(codec), out_width(out_width), out_height(out_height),
+    publish_name(publish_name) { }
+  virtual ~VideoEncoder();
+  virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
+  virtual void encoder_open(const char* path) = 0;
+  virtual void encoder_close() = 0;
+
+  void publisher_init();
+  static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
+
+
+protected:
+  const char* filename;
+  const char* publish_name;
+  int in_width, in_height;
+  int out_width, out_height, fps;
+  int bitrate;
+  cereal::EncodeIndex::Type codec;
+  CameraType type;
+
+private:
+  // total frames encoded
+  int cnt = 0;
+
+  // publishing
+  std::unique_ptr<PubMaster> pm;
+  const char *service_name;
+};

system/loggerd/encoder/ffmpeg_encoder.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+}
+
+#include "system/loggerd/encoder/encoder.h"
+#include "system/loggerd/loggerd.h"
+
+class FfmpegEncoder : public VideoEncoder {
+ public:
+  FfmpegEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
+
+                int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height,
+                const char* publish_name) :
+                VideoEncoder(filename, type, in_width, in_height, fps, bitrate, cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS, out_width, out_height, publish_name) { encoder_init(); }
+  ~FfmpegEncoder();
+  void encoder_init();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open(const char* path);
+  void encoder_close();
+
+private:
+  int segment_num = -1;
+  int counter = 0;
+  bool is_open = false;
+
+  AVCodecContext *codec_ctx;
+  AVFrame *frame = NULL;
+  std::vector<uint8_t> convert_buf;
+  std::vector<uint8_t> downscale_buf;
+};

system/loggerd/encoder/v4l_encoder.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "common/queue.h"
+#include "system/loggerd/encoder/encoder.h"
+
+#define BUF_IN_COUNT 7
+#define BUF_OUT_COUNT 6
+
+class V4LEncoder : public VideoEncoder {
+public:
+  V4LEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
+             int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, const char* publish_name) :
+             VideoEncoder(filename, type, in_width, in_height, fps, bitrate, codec, out_width, out_height, publish_name) { encoder_init(); }
+  ~V4LEncoder();
+  void encoder_init();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open(const char* path);
+  void encoder_close();
+private:
+  int fd;
+
+  bool is_open = false;
+  int segment_num = -1;
+  int counter = 0;
+
+  SafeQueue<VisionIpcBufExtra> extras;
+
+  static void dequeue_handler(V4LEncoder *e);
+  std::thread dequeue_handler_thread;
+
+  VisionBuf buf_out[BUF_OUT_COUNT];
+  SafeQueue<unsigned int> free_buf_in;
+};

system/loggerd/logger.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include <cassert>
+#include <pthread.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <memory>
+
+#include <capnp/serialize.h>
+#include <kj/array.h>
+
+#include "cereal/messaging/messaging.h"
+#include "common/util.h"
+#include "common/swaglog.h"
+#include "system/hardware/hw.h"
+
+const std::string LOG_ROOT = Path::log_root();
+
+#define LOGGER_MAX_HANDLES 16
+
+class RawFile {
+ public:
+  RawFile(const char* path) {
+    file = util::safe_fopen(path, "wb");
+    assert(file != nullptr);
+  }
+  ~RawFile() {
+    util::safe_fflush(file);
+    int err = fclose(file);
+    assert(err == 0);
+  }
+  inline void write(void* data, size_t size) {
+    int written = util::safe_fwrite(data, 1, size, file);
+    assert(written == size);
+  }
+  inline void write(kj::ArrayPtr<capnp::byte> array) { write(array.begin(), array.size()); }
+
+ private:
+  FILE* file = nullptr;
+};
+
+typedef cereal::Sentinel::SentinelType SentinelType;
+
+typedef struct LoggerHandle {
+  pthread_mutex_t lock;
+  SentinelType end_sentinel_type;
+  int exit_signal;
+  int refcnt;
+  char segment_path[4096];
+  char log_path[4096];
+  char qlog_path[4096];
+  char lock_path[4096];
+  std::unique_ptr<RawFile> log, q_log;
+} LoggerHandle;
+
+typedef struct LoggerState {
+  pthread_mutex_t lock;
+  int part;
+  kj::Array<capnp::word> init_data;
+  std::string route_name;
+  char log_name[64];
+  bool has_qlog;
+
+  LoggerHandle handles[LOGGER_MAX_HANDLES];
+  LoggerHandle* cur_handle;
+} LoggerState;
+
+kj::Array<capnp::word> logger_build_init_data();
+std::string logger_get_route_name();
+void logger_init(LoggerState *s, bool has_qlog);
+int logger_next(LoggerState *s, const char* root_path,
+                            char* out_segment_path, size_t out_segment_path_len,
+                            int* out_part);
+LoggerHandle* logger_get_handle(LoggerState *s);
+void logger_close(LoggerState *s, ExitHandler *exit_handler=nullptr);
+void logger_log(LoggerState *s, uint8_t* data, size_t data_size, bool in_qlog);
+
+void lh_log(LoggerHandle* h, uint8_t* data, size_t data_size, bool in_qlog);
+void lh_close(LoggerHandle* h);

system/loggerd/loggerd.h

@@ -0,0 +1,121 @@
+#pragma once
+
+#include <unistd.h>
+
+#include <atomic>
+#include <cassert>
+#include <cerrno>
+#include <condition_variable>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/services.h"
+#include "cereal/visionipc/visionipc.h"
+#include "cereal/visionipc/visionipc_client.h"
+#include "system/camerad/cameras/camera_common.h"
+#include "common/params.h"
+#include "common/swaglog.h"
+#include "common/timing.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+
+#include "system/loggerd/encoder/encoder.h"
+#include "system/loggerd/logger.h"
+#ifdef QCOM2
+#include "system/loggerd/encoder/v4l_encoder.h"
+#define Encoder V4LEncoder
+#else
+#include "system/loggerd/encoder/ffmpeg_encoder.h"
+#define Encoder FfmpegEncoder
+#endif
+
+constexpr int MAIN_FPS = 20;
+const int MAIN_BITRATE = 10000000;
+
+#define NO_CAMERA_PATIENCE 500 // fall back to time-based rotation if all cameras are dead
+
+const bool LOGGERD_TEST = getenv("LOGGERD_TEST");
+const int SEGMENT_LENGTH = LOGGERD_TEST ? atoi(getenv("LOGGERD_SEGMENT_LENGTH")) : 60;
+
+class EncoderInfo {
+public:
+  const char *publish_name;
+  const char *filename;
+  bool record = true;
+  int frame_width = 1928;
+  int frame_height = 1208;
+  int fps = MAIN_FPS;
+  int bitrate = MAIN_BITRATE;
+  cereal::EncodeIndex::Type encode_type = cereal::EncodeIndex::Type::FULL_H_E_V_C;
+  ::cereal::EncodeData::Reader (cereal::Event::Reader::*get_encode_data_func)() const;
+  void (cereal::Event::Builder::*set_encode_idx_func)(::cereal::EncodeIndex::Reader);
+};
+
+class LogCameraInfo {
+public:
+  const char *thread_name;
+  int fps = MAIN_FPS;
+  CameraType type;
+  VisionStreamType stream_type;
+  std::vector<EncoderInfo> encoder_infos;
+};
+
+const EncoderInfo main_road_encoder_info = {
+  .publish_name = "roadEncodeData",
+  .filename = "fcamera.hevc",
+  .get_encode_data_func = &cereal::Event::Reader::getRoadEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setRoadEncodeIdx,
+};
+const EncoderInfo main_wide_road_encoder_info = {
+  .publish_name = "wideRoadEncodeData",
+  .filename = "ecamera.hevc",
+  .get_encode_data_func = &cereal::Event::Reader::getWideRoadEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setWideRoadEncodeIdx,
+};
+const EncoderInfo main_driver_encoder_info = {
+   .publish_name = "driverEncodeData",
+  .filename = "dcamera.hevc",
+  .record = Params().getBool("RecordFront"),
+  .get_encode_data_func = &cereal::Event::Reader::getDriverEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setDriverEncodeIdx,
+};
+
+const EncoderInfo qcam_encoder_info = {
+  .publish_name = "qRoadEncodeData",
+  .filename = "qcamera.ts",
+  .bitrate = 256000,
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .frame_width = 526,
+  .frame_height = 330,
+  .get_encode_data_func = &cereal::Event::Reader::getQRoadEncodeData,
+  .set_encode_idx_func = &cereal::Event::Builder::setQRoadEncodeIdx,
+};
+
+
+const LogCameraInfo road_camera_info{
+    .thread_name = "road_cam_encoder",
+    .type = RoadCam,
+    .stream_type = VISION_STREAM_ROAD,
+    .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
+    };
+
+const LogCameraInfo wide_road_camera_info{
+    .thread_name = "wide_road_cam_encoder",
+    .type = WideRoadCam,
+    .stream_type = VISION_STREAM_WIDE_ROAD,
+   .encoder_infos = {main_wide_road_encoder_info}
+    };
+  
+const LogCameraInfo driver_camera_info{
+    .thread_name = "driver_cam_encoder",
+    .type = DriverCam,
+    .stream_type = VISION_STREAM_DRIVER,
+    .encoder_infos = {main_driver_encoder_info}
+    };
+
+const LogCameraInfo cameras_logged[] = {road_camera_info, wide_road_camera_info, driver_camera_info};
+

system/loggerd/uploader.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+import bz2
+import io
+import json
+import os
+import random
+import requests
+import threading
+import time
+import traceback
+from pathlib import Path
+from typing import BinaryIO, Iterator, List, Optional, Tuple, Union
+
+from cereal import log
+import cereal.messaging as messaging
+from common.api import Api
+from common.params import Params
+from common.realtime import set_core_affinity
+from system.hardware import TICI
+from system.loggerd.xattr_cache import getxattr, setxattr
+from system.loggerd.config import ROOT
+from system.swaglog import cloudlog
+
+NetworkType = log.DeviceState.NetworkType
+UPLOAD_ATTR_NAME = 'user.upload'
+UPLOAD_ATTR_VALUE = b'1'
+
+UPLOAD_QLOG_QCAM_MAX_SIZE = 100 * 1e6  # MB
+
+allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
+force_wifi = os.getenv("FORCEWIFI") is not None
+fake_upload = os.getenv("FAKEUPLOAD") is not None
+
+
+class FakeRequest:
+  def __init__(self):
+    self.headers = {"Content-Length": "0"}
+
+
+class FakeResponse:
+  def __init__(self):
+    self.status_code = 200
+    self.request = FakeRequest()
+
+
+UploadResponse = Union[requests.Response, FakeResponse]
+
+def get_directory_sort(d: str) -> List[str]:
+  return list(map(lambda s: s.rjust(10, '0'), d.rsplit('--', 1)))
+
+def listdir_by_creation(d: str) -> List[str]:
+  try:
+    paths = os.listdir(d)
+    paths = sorted(paths, key=get_directory_sort)
+    return paths
+  except OSError:
+    cloudlog.exception("listdir_by_creation failed")
+    return list()
+
+def clear_locks(root: str) -> None:
+  for logname in os.listdir(root):
+    path = os.path.join(root, logname)
+    try:
+      for fname in os.listdir(path):
+        if fname.endswith(".lock"):
+          os.unlink(os.path.join(path, fname))
+    except OSError:
+      cloudlog.exception("clear_locks failed")
+
+
+class Uploader:
+  def __init__(self, dongle_id: str, root: str):
+    self.dongle_id = dongle_id
+    self.api = Api(dongle_id)
+    self.root = root
+
+    self.last_resp: Optional[UploadResponse] = None
+    self.last_exc: Optional[Tuple[Exception, str]] = None
+
+    self.immediate_size = 0
+    self.immediate_count = 0
+
+    # stats for last successfully uploaded file
+    self.last_time = 0.0
+    self.last_speed = 0.0
+    self.last_filename = ""
+
+    self.immediate_folders = ["crash/", "boot/"]
+    self.immediate_priority = {"qlog": 0, "qlog.bz2": 0, "qcamera.ts": 1}
+
+    # dp
+    self.update_onced = False
+
+  def get_upload_sort(self, name: str) -> int:
+    if name in self.immediate_priority:
+      return self.immediate_priority[name]
+    return 1000
+
+  def list_upload_files(self) -> Iterator[Tuple[str, str, str]]:
+    if not os.path.isdir(self.root):
+      return
+
+    self.immediate_size = 0
+    self.immediate_count = 0
+
+    for logname in listdir_by_creation(self.root):
+      path = os.path.join(self.root, logname)
+      try:
+        names = os.listdir(path)
+      except OSError:
+        continue
+
+      if any(name.endswith(".lock") for name in names):
+        continue
+
+      for name in sorted(names, key=self.get_upload_sort):
+        key = os.path.join(logname, name)
+        fn = os.path.join(path, name)
+        # skip files already uploaded
+        try:
+          is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
+        except OSError:
+          cloudlog.event("uploader_getxattr_failed", exc=self.last_exc, key=key, fn=fn)
+          is_uploaded = True  # deleter could have deleted
+        if is_uploaded:
+          continue
+
+        try:
+          if name in self.immediate_priority:
+            self.immediate_count += 1
+            self.immediate_size += os.path.getsize(fn)
+        except OSError:
+          pass
+
+        yield name, key, fn
+
+  def next_file_to_upload(self) -> Optional[Tuple[str, str, str]]:
+    upload_files = list(self.list_upload_files())
+
+    for name, key, fn in upload_files:
+      if any(f in fn for f in self.immediate_folders):
+        return name, key, fn
+
+    for name, key, fn in upload_files:
+      if name in self.immediate_priority:
+        return name, key, fn
+
+    return None
+
+  def do_upload(self, key: str, fn: str) -> None:
+    try:
+      url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token())
+      if url_resp.status_code == 412:
+        self.last_resp = url_resp
+        return
+
+      url_resp_json = json.loads(url_resp.text)
+      url = url_resp_json['url']
+      headers = url_resp_json['headers']
+      cloudlog.debug("upload_url v1.4 %s %s", url, str(headers))
+
+      if fake_upload:
+        cloudlog.debug(f"*** WARNING, THIS IS A FAKE UPLOAD TO {url} ***")
+        self.last_resp = FakeResponse()
+      else:
+        with open(fn, "rb") as f:
+          data: BinaryIO
+          if key.endswith('.bz2') and not fn.endswith('.bz2'):
+            compressed = bz2.compress(f.read())
+            data = io.BytesIO(compressed)
+          else:
+            data = f
+
+          self.last_resp = requests.put(url, data=data, headers=headers, timeout=10)
+    except Exception as e:
+      self.last_exc = (e, traceback.format_exc())
+      raise
+
+  def normal_upload(self, key: str, fn: str) -> Optional[UploadResponse]:
+    self.last_resp = None
+    self.last_exc = None
+
+    try:
+      self.do_upload(key, fn)
+    except Exception:
+      pass
+
+    return self.last_resp
+
+  def upload(self, name: str, key: str, fn: str, network_type: int, metered: bool) -> bool:
+    try:
+      sz = os.path.getsize(fn)
+    except OSError:
+      cloudlog.exception("upload: getsize failed")
+      return False
+
+    cloudlog.event("upload_start", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if sz == 0:
+      # tag files of 0 size as uploaded
+      success = True
+    elif name in self.immediate_priority and sz > UPLOAD_QLOG_QCAM_MAX_SIZE:
+      cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz)
+      success = True
+    else:
+      start_time = time.monotonic()
+      stat = self.normal_upload(key, fn)
+      if stat is not None and stat.status_code in (200, 201, 401, 403, 412):
+        self.last_filename = fn
+        self.last_time = time.monotonic() - start_time
+        if stat.status_code == 412:
+          self.last_speed = 0
+          cloudlog.event("upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+        else:
+          content_length = int(stat.request.headers.get("Content-Length", 0))
+          self.last_speed = (content_length / 1e6) / self.last_time
+          cloudlog.event("upload_success", key=key, fn=fn, sz=sz, content_length=content_length, network_type=network_type, metered=metered, speed=self.last_speed)
+        success = True
+        if not self.update_onced and stat.status_code != 412:
+          Params().put_bool('dp_upload_ignored', False)
+          self.update_onced = True
+        cloudlog.event("upload_success" if stat.status_code != 412 else "upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+      else:
+        success = False
+        cloudlog.event("upload_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if success:
+      # tag file as uploaded
+      try:
+        setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
+      except OSError:
+        cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz)
+
+    return success
+
+  def get_msg(self):
+    msg = messaging.new_message("uploaderState")
+    us = msg.uploaderState
+    us.immediateQueueSize = int(self.immediate_size / 1e6)
+    us.immediateQueueCount = self.immediate_count
+    us.lastTime = self.last_time
+    us.lastSpeed = self.last_speed
+    us.lastFilename = self.last_filename
+    return msg
+
+
+def uploader_fn(exit_event: threading.Event) -> None:
+  try:
+    set_core_affinity([0, 1, 2, 3])
+  except Exception:
+    cloudlog.exception("failed to set core affinity")
+
+  clear_locks(ROOT)
+
+  params = Params()
+  dongle_id = params.get("DongleId", encoding='utf8')
+
+  if dongle_id is None:
+    cloudlog.info("uploader missing dongle_id")
+    raise Exception("uploader can't start without dongle id")
+
+  if TICI and not Path("/data/media").is_mount():
+    cloudlog.warning("NVME not mounted")
+
+  sm = messaging.SubMaster(['deviceState'])
+  pm = messaging.PubMaster(['uploaderState'])
+  uploader = Uploader(dongle_id, ROOT)
+
+  backoff = 0.1
+  while not exit_event.is_set():
+    sm.update(0)
+    offroad = params.get_bool("IsOffroad")
+    network_type = sm['deviceState'].networkType if not force_wifi else NetworkType.wifi
+    if network_type == NetworkType.none:
+      if allow_sleep:
+        time.sleep(60 if offroad else 5)
+      continue
+
+    d = uploader.next_file_to_upload()
+    if d is None:  # Nothing to upload
+      if allow_sleep:
+        time.sleep(60 if offroad else 5)
+      continue
+
+    name, key, fn = d
+
+    # qlogs and bootlogs need to be compressed before uploading
+    if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.bz2')):
+      key += ".bz2"
+
+    success = uploader.upload(name, key, fn, sm['deviceState'].networkType.raw, sm['deviceState'].networkMetered)
+    if success:
+      backoff = 0.1
+    elif allow_sleep:
+      cloudlog.info("upload backoff %r", backoff)
+      time.sleep(backoff + random.uniform(0, backoff))
+      backoff = min(backoff*2, 120)
+
+    pm.send("uploaderState", uploader.get_msg())
+
+
+def main() -> None:
+  uploader_fn(threading.Event())
+
+
+if __name__ == "__main__":
+  main()

system/loggerd/video_writer.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <string>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
+
+#include "cereal/messaging/messaging.h"
+
+class VideoWriter {
+public:
+  VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec);
+  void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe);
+  ~VideoWriter();
+private:
+  std::string vid_path, lock_path;
+
+  FILE *of = nullptr;
+
+  AVCodecContext *codec_ctx;
+  AVFormatContext *ofmt_ctx;
+  AVStream *out_stream;
+  bool remuxing, raw;
+};

system/loggerd/xattr_cache.py

@@ -0,0 +1,23 @@
+import os
+import errno
+from typing import Dict, Tuple, Optional
+
+_cached_attributes: Dict[Tuple, Optional[bytes]] = {}
+
+def getxattr(path: str, attr_name: str) -> Optional[bytes]:
+  key = (path, attr_name)
+  if key not in _cached_attributes:
+    try:
+      response = os.getxattr(path, attr_name)
+    except OSError as e:
+      # ENODATA means attribute hasn't been set
+      if e.errno == errno.ENODATA:
+        response = None
+      else:
+        raise
+    _cached_attributes[key] = response
+  return _cached_attributes[key]
+
+def setxattr(path: str, attr_name: str, attr_value: bytes) -> None:
+  _cached_attributes.pop((path, attr_name), None)
+  return os.setxattr(path, attr_name, attr_value)