dragonpilot v0.9.8

system/loggerd/.gitignore

@@ -0,0 +1,4 @@
+loggerd
+encoderd
+bootlog
+tests/test_logger

system/loggerd/SConscript

@@ -0,0 +1,26 @@
+Import('env', 'arch', 'messaging', 'common', 'visionipc')
+
+libs = [common, messaging, visionipc,
+        'avformat', 'avcodec', 'avutil',
+        'yuv', 'OpenCL', 'pthread', 'zstd']
+
+src = ['logger.cc', 'zstd_writer.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/v4l_encoder.cc', 'encoder/jpeg_encoder.cc']
+if arch != "larch64":
+  src += ['encoder/ffmpeg_encoder.cc']
+
+if arch == "Darwin":
+  # fix OpenCL
+  del libs[libs.index('OpenCL')]
+  env['FRAMEWORKS'] = ['OpenCL']
+  # exclude v4l
+  del src[src.index('encoder/v4l_encoder.cc')]
+
+logger_lib = env.Library('logger', src)
+libs.insert(0, logger_lib)
+
+env.Program('loggerd', ['loggerd.cc'], LIBS=libs)
+env.Program('encoderd', ['encoderd.cc'], LIBS=libs + ["jpeg"])
+env.Program('bootlog.cc', LIBS=libs)
+
+if GetOption('extras'):
+  env.Program('tests/test_logger', ['tests/test_runner.cc', 'tests/test_logger.cc', 'tests/test_zstd_writer.cc'], LIBS=libs + ['curl', 'crypto'])

system/loggerd/config.py

@@ -0,0 +1,29 @@
+import os
+from openpilot.system.hardware.hw import Paths
+
+
+CAMERA_FPS = 20
+SEGMENT_LENGTH = 60
+
+STATS_DIR_FILE_LIMIT = 10000
+STATS_SOCKET = "ipc:///tmp/stats"
+STATS_FLUSH_TIME_S = 60
+
+def get_available_percent(default: float) -> float:
+  try:
+    statvfs = os.statvfs(Paths.log_root())
+    available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
+  except OSError:
+    available_percent = default
+
+  return available_percent
+
+
+def get_available_bytes(default: int) -> int:
+  try:
+    statvfs = os.statvfs(Paths.log_root())
+    available_bytes = statvfs.f_bavail * statvfs.f_frsize
+  except OSError:
+    available_bytes = default
+
+  return available_bytes

system/loggerd/deleter.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import threading
+from openpilot.system.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
+from openpilot.system.loggerd.uploader import listdir_by_creation
+from openpilot.system.loggerd.xattr_cache import getxattr
+
+MIN_BYTES = 5 * 1024 * 1024 * 1024
+MIN_PERCENT = 10
+
+DELETE_LAST = ['boot', 'crash']
+
+PRESERVE_ATTR_NAME = 'user.preserve'
+PRESERVE_ATTR_VALUE = b'1'
+PRESERVE_COUNT = 5
+
+
+def has_preserve_xattr(d: str) -> bool:
+  return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+
+def get_preserved_segments(dirs_by_creation: list[str]) -> set[str]:
+  # skip deleting most recent N preserved segments (and their prior segment)
+  preserved = set()
+  for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
+    if n == PRESERVE_COUNT:
+      break
+    date_str, _, seg_str = d.rpartition("--")
+
+    # ignore non-segment directories
+    if not date_str:
+      continue
+    try:
+      seg_num = int(seg_str)
+    except ValueError:
+      continue
+
+    # preserve segment and two prior
+    for _seg_num in range(max(0, seg_num - 2), seg_num + 1):
+      preserved.add(f"{date_str}--{_seg_num}")
+
+  return preserved
+
+
+def deleter_thread(exit_event: threading.Event):
+  while not exit_event.is_set():
+    out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
+    out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
+
+    if out_of_percent or out_of_bytes:
+      dirs = listdir_by_creation(Paths.log_root())
+      preserved_dirs = get_preserved_segments(dirs)
+
+      # remove the earliest directory we can
+      for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
+        delete_path = os.path.join(Paths.log_root(), delete_dir)
+
+        if any(name.endswith(".lock") for name in os.listdir(delete_path)):
+          continue
+
+        try:
+          cloudlog.info(f"deleting {delete_path}")
+          shutil.rmtree(delete_path)
+          break
+        except OSError:
+          cloudlog.exception(f"issue deleting {delete_path}")
+      exit_event.wait(.1)
+    else:
+      exit_event.wait(30)
+
+
+def main():
+  deleter_thread(threading.Event())
+
+
+if __name__ == "__main__":
+  main()

system/loggerd/encoder/encoder.h

@@ -0,0 +1,42 @@
+#pragma once
+
+// has to be in this order
+#ifdef __linux__
+#include "third_party/linux/include/v4l2-controls.h"
+#include <linux/videodev2.h>
+#else
+#define V4L2_BUF_FLAG_KEYFRAME 8
+#endif
+
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionipc.h"
+#include "common/queue.h"
+#include "system/loggerd/loggerd.h"
+
+class VideoEncoder {
+public:
+  VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  virtual ~VideoEncoder() {}
+  virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
+  virtual void encoder_open() = 0;
+  virtual void encoder_close() = 0;
+
+  void publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
+
+protected:
+  int in_width, in_height;
+  int out_width, out_height;
+  const EncoderInfo encoder_info;
+
+private:
+  // total frames encoded
+  int cnt = 0;
+  std::unique_ptr<PubMaster> pm;
+  std::vector<capnp::byte> msg_cache;
+};

system/loggerd/encoder/ffmpeg_encoder.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+}
+
+#include "system/loggerd/encoder/encoder.h"
+#include "system/loggerd/loggerd.h"
+
+class FfmpegEncoder : public VideoEncoder {
+public:
+  FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  ~FfmpegEncoder();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open();
+  void encoder_close();
+
+private:
+  int segment_num = -1;
+  int counter = 0;
+  bool is_open = false;
+
+  AVCodecContext *codec_ctx;
+  AVFrame *frame = NULL;
+  std::vector<uint8_t> convert_buf;
+  std::vector<uint8_t> downscale_buf;
+};

system/loggerd/encoder/jpeg_encoder.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstddef>
+#include <cstdint>
+#include <jpeglib.h>
+#include <vector>
+#include <memory>
+#include "cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionbuf.h"
+
+class JpegEncoder {
+public:
+  JpegEncoder(const std::string &pusblish_name, int width, int height);
+  ~JpegEncoder();
+  void pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra);
+
+private:
+  void generateThumbnail(const uint8_t *y, const uint8_t *uv, int width, int height, int stride);
+  void compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane);
+
+  int thumbnail_width;
+  int thumbnail_height;
+  std::string publish_name;
+  std::vector<uint8_t> yuv_buffer;
+  std::unique_ptr<PubMaster> pm;
+
+  // JPEG output buffer
+  unsigned char* out_buffer = nullptr;
+  unsigned long out_size = 0;
+};

system/loggerd/encoder/v4l_encoder.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "common/queue.h"
+#include "system/loggerd/encoder/encoder.h"
+
+#define BUF_IN_COUNT 7
+#define BUF_OUT_COUNT 6
+
+class V4LEncoder : public VideoEncoder {
+public:
+  V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  ~V4LEncoder();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open();
+  void encoder_close();
+
+private:
+  int fd;
+
+  bool is_open = false;
+  int segment_num = -1;
+  int counter = 0;
+
+  SafeQueue<VisionIpcBufExtra> extras;
+
+  static void dequeue_handler(V4LEncoder *e);
+  std::thread dequeue_handler_thread;
+
+  VisionBuf buf_out[BUF_OUT_COUNT];
+  SafeQueue<unsigned int> free_buf_in;
+};

system/loggerd/logger.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <cassert>
+#include <memory>
+#include <string>
+
+#include "cereal/messaging/messaging.h"
+#include "common/util.h"
+#include "system/hardware/hw.h"
+#include "system/loggerd/zstd_writer.h"
+
+constexpr int LOG_COMPRESSION_LEVEL = 10;
+
+typedef cereal::Sentinel::SentinelType SentinelType;
+
+class LoggerState {
+public:
+  LoggerState(const std::string& log_root = Path::log_root());
+  ~LoggerState();
+  bool next();
+  void write(uint8_t* data, size_t size, bool in_qlog);
+  inline int segment() const { return part; }
+  inline const std::string& segmentPath() const { return segment_path; }
+  inline const std::string& routeName() const { return route_name; }
+  inline void write(kj::ArrayPtr<kj::byte> bytes, bool in_qlog) { write(bytes.begin(), bytes.size(), in_qlog); }
+  inline void setExitSignal(int signal) { exit_signal = signal; }
+
+protected:
+  int part = -1, exit_signal = 0;
+  std::string route_path, route_name, segment_path, lock_file;
+  kj::Array<capnp::word> init_data;
+  std::unique_ptr<ZstdFileWriter> rlog, qlog;
+};
+
+kj::Array<capnp::word> logger_build_init_data();
+std::string logger_get_identifier(std::string key);
+std::string zstd_decompress(const std::string &in);

system/loggerd/loggerd.h

@@ -0,0 +1,149 @@
+#pragma once
+
+#include <vector>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/services.h"
+#include "msgq/visionipc/visionipc_client.h"
+#include "system/hardware/hw.h"
+#include "common/params.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+
+#include "system/loggerd/logger.h"
+
+constexpr int MAIN_FPS = 20;
+const int MAIN_BITRATE = 1e7;
+const int LIVESTREAM_BITRATE = 1e6;
+const int QCAM_BITRATE = 256000;
+
+#define NO_CAMERA_PATIENCE 500  // fall back to time-based rotation if all cameras are dead
+
+#define INIT_ENCODE_FUNCTIONS(encode_type)                                \
+  .get_encode_data_func = &cereal::Event::Reader::get##encode_type##Data, \
+  .set_encode_idx_func = &cereal::Event::Builder::set##encode_type##Idx,  \
+  .init_encode_data_func = &cereal::Event::Builder::init##encode_type##Data
+
+const bool LOGGERD_TEST = getenv("LOGGERD_TEST");
+const int SEGMENT_LENGTH = LOGGERD_TEST ? atoi(getenv("LOGGERD_SEGMENT_LENGTH")) : 60;
+
+constexpr char PRESERVE_ATTR_NAME[] = "user.preserve";
+constexpr char PRESERVE_ATTR_VALUE = '1';
+class EncoderInfo {
+public:
+  const char *publish_name;
+  const char *thumbnail_name = NULL;
+  const char *filename = NULL;
+  bool record = true;
+  int frame_width = -1;
+  int frame_height = -1;
+  int fps = MAIN_FPS;
+  int bitrate = MAIN_BITRATE;
+  cereal::EncodeIndex::Type encode_type = Hardware::PC() ? cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS
+                                                         : cereal::EncodeIndex::Type::FULL_H_E_V_C;
+  ::cereal::EncodeData::Reader (cereal::Event::Reader::*get_encode_data_func)() const;
+  void (cereal::Event::Builder::*set_encode_idx_func)(::cereal::EncodeIndex::Reader);
+  cereal::EncodeData::Builder (cereal::Event::Builder::*init_encode_data_func)();
+};
+
+class LogCameraInfo {
+public:
+  const char *thread_name;
+  int fps = MAIN_FPS;
+  VisionStreamType stream_type;
+  std::vector<EncoderInfo> encoder_infos;
+};
+
+const EncoderInfo main_road_encoder_info = {
+  .publish_name = "roadEncodeData",
+  .thumbnail_name = "thumbnail",
+  .filename = "fcamera.hevc",
+  INIT_ENCODE_FUNCTIONS(RoadEncode),
+};
+
+const EncoderInfo main_wide_road_encoder_info = {
+  .publish_name = "wideRoadEncodeData",
+  .filename = "ecamera.hevc",
+  INIT_ENCODE_FUNCTIONS(WideRoadEncode),
+};
+
+const EncoderInfo main_driver_encoder_info = {
+  .publish_name = "driverEncodeData",
+  .filename = "dcamera.hevc",
+  .record = !getenv("DISABLE_DRIVER") && Params().getBool("RecordFront"),
+  INIT_ENCODE_FUNCTIONS(DriverEncode),
+};
+
+const EncoderInfo stream_road_encoder_info = {
+  .publish_name = "livestreamRoadEncodeData",
+  //.thumbnail_name = "thumbnail",
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .record = false,
+  .bitrate = LIVESTREAM_BITRATE,
+  INIT_ENCODE_FUNCTIONS(LivestreamRoadEncode),
+};
+
+const EncoderInfo stream_wide_road_encoder_info = {
+  .publish_name = "livestreamWideRoadEncodeData",
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .record = false,
+  .bitrate = LIVESTREAM_BITRATE,
+  INIT_ENCODE_FUNCTIONS(LivestreamWideRoadEncode),
+};
+
+const EncoderInfo stream_driver_encoder_info = {
+  .publish_name = "livestreamDriverEncodeData",
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .record = false,
+  .bitrate = LIVESTREAM_BITRATE,
+  INIT_ENCODE_FUNCTIONS(LivestreamDriverEncode),
+};
+
+const EncoderInfo qcam_encoder_info = {
+  .publish_name = "qRoadEncodeData",
+  .filename = "qcamera.ts",
+  .bitrate = QCAM_BITRATE,
+  .encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+  .frame_width = 526,
+  .frame_height = 330,
+  INIT_ENCODE_FUNCTIONS(QRoadEncode),
+};
+
+const LogCameraInfo road_camera_info{
+  .thread_name = "road_cam_encoder",
+  .stream_type = VISION_STREAM_ROAD,
+  .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
+};
+
+const LogCameraInfo wide_road_camera_info{
+  .thread_name = "wide_road_cam_encoder",
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {main_wide_road_encoder_info}
+};
+
+const LogCameraInfo driver_camera_info{
+  .thread_name = "driver_cam_encoder",
+  .stream_type = VISION_STREAM_DRIVER,
+  .encoder_infos = {main_driver_encoder_info}
+};
+
+const LogCameraInfo stream_road_camera_info{
+  .thread_name = "road_cam_encoder",
+  .stream_type = VISION_STREAM_ROAD,
+  .encoder_infos = {stream_road_encoder_info}
+};
+
+const LogCameraInfo stream_wide_road_camera_info{
+  .thread_name = "wide_road_cam_encoder",
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {stream_wide_road_encoder_info}
+};
+
+const LogCameraInfo stream_driver_camera_info{
+  .thread_name = "driver_cam_encoder",
+  .stream_type = VISION_STREAM_DRIVER,
+  .encoder_infos = {stream_driver_encoder_info}
+};
+
+const LogCameraInfo cameras_logged[] = {road_camera_info, wide_road_camera_info, driver_camera_info};
+const LogCameraInfo stream_cameras_logged[] = {stream_road_camera_info, stream_wide_road_camera_info, stream_driver_camera_info};

system/loggerd/uploader.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+import json
+import os
+import random
+import requests
+import threading
+import time
+import traceback
+import datetime
+from collections.abc import Iterator
+
+from cereal import log
+import cereal.messaging as messaging
+from openpilot.common.api import Api
+from openpilot.common.file_helpers import get_upload_stream
+from openpilot.common.params import Params
+from openpilot.common.realtime import set_core_affinity
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
+from openpilot.common.swaglog import cloudlog
+
+NetworkType = log.DeviceState.NetworkType
+UPLOAD_ATTR_NAME = 'user.upload'
+UPLOAD_ATTR_VALUE = b'1'
+
+MAX_UPLOAD_SIZES = {
+  "qlog": 25*1e6,  # can't be too restrictive here since we use qlogs to find
+                   # bugs, including ones that can cause massive log sizes
+  "qcam": 5*1e6,
+}
+
+allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
+force_wifi = os.getenv("FORCEWIFI") is not None
+fake_upload = os.getenv("FAKEUPLOAD") is not None
+
+
+class FakeRequest:
+  def __init__(self):
+    self.headers = {"Content-Length": "0"}
+
+
+class FakeResponse:
+  def __init__(self):
+    self.status_code = 200
+    self.request = FakeRequest()
+
+
+def get_directory_sort(d: str) -> list[str]:
+  # ensure old format is sorted sooner
+  o = ["0", ] if d.startswith("2024-") else ["1", ]
+  return o + [s.rjust(10, '0') for s in d.rsplit('--', 1)]
+
+def listdir_by_creation(d: str) -> list[str]:
+  if not os.path.isdir(d):
+    return []
+
+  try:
+    paths = [f for f in os.listdir(d) if os.path.isdir(os.path.join(d, f))]
+    paths = sorted(paths, key=get_directory_sort)
+    return paths
+  except OSError:
+    cloudlog.exception("listdir_by_creation failed")
+    return []
+
+def clear_locks(root: str) -> None:
+  for logdir in os.listdir(root):
+    path = os.path.join(root, logdir)
+    try:
+      for fname in os.listdir(path):
+        if fname.endswith(".lock"):
+          os.unlink(os.path.join(path, fname))
+    except OSError:
+      cloudlog.exception("clear_locks failed")
+
+
+class Uploader:
+  def __init__(self, dongle_id: str, root: str):
+    self.dongle_id = dongle_id
+    self.api = Api(dongle_id)
+    self.root = root
+
+    self.params = Params()
+
+    # stats for last successfully uploaded file
+    self.last_filename = ""
+
+    self.immediate_folders = ["crash/", "boot/"]
+    self.immediate_priority = {"qlog": 0, "qlog.zst": 0, "qcamera.ts": 1}
+
+  def list_upload_files(self, metered: bool) -> Iterator[tuple[str, str, str]]:
+    r = self.params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
+    requested_routes = [] if r is None else r.split(",")
+
+    for logdir in listdir_by_creation(self.root):
+      path = os.path.join(self.root, logdir)
+      try:
+        names = os.listdir(path)
+      except OSError:
+        continue
+
+      if any(name.endswith(".lock") for name in names):
+        continue
+
+      for name in sorted(names, key=lambda n: self.immediate_priority.get(n, 1000)):
+        key = os.path.join(logdir, name)
+        fn = os.path.join(path, name)
+        # skip files already uploaded
+        try:
+          ctime = os.path.getctime(fn)
+          is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
+        except OSError:
+          cloudlog.event("uploader_getxattr_failed", key=key, fn=fn)
+          # deleter could have deleted, so skip
+          continue
+        if is_uploaded:
+          continue
+
+        # limit uploading on metered connections
+        if metered:
+          dt = datetime.timedelta(hours=12)
+          if logdir in self.immediate_folders and (datetime.datetime.now() - datetime.datetime.fromtimestamp(ctime)) < dt:
+            continue
+
+          if name == "qcamera.ts" and not any(logdir.startswith(r.split('|')[-1]) for r in requested_routes):
+            continue
+
+        yield name, key, fn
+
+  def next_file_to_upload(self, metered: bool) -> tuple[str, str, str] | None:
+    upload_files = list(self.list_upload_files(metered))
+
+    for name, key, fn in upload_files:
+      if any(f in fn for f in self.immediate_folders):
+        return name, key, fn
+
+    for name, key, fn in upload_files:
+      if name in self.immediate_priority:
+        return name, key, fn
+
+    return None
+
+  def do_upload(self, key: str, fn: str):
+    url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token())
+    if url_resp.status_code == 412:
+      return url_resp
+
+    url_resp_json = json.loads(url_resp.text)
+    url = url_resp_json['url']
+    headers = url_resp_json['headers']
+    cloudlog.debug("upload_url v1.4 %s %s", url, str(headers))
+
+    if fake_upload:
+      return FakeResponse()
+
+    stream = None
+    try:
+      compress = key.endswith('.zst') and not fn.endswith('.zst')
+      stream, _ = get_upload_stream(fn, compress)
+      response = requests.put(url, data=stream, headers=headers, timeout=10)
+      return response
+    finally:
+      if stream:
+        stream.close()
+
+  def upload(self, name: str, key: str, fn: str, network_type: int, metered: bool) -> bool:
+    try:
+      sz = os.path.getsize(fn)
+    except OSError:
+      cloudlog.exception("upload: getsize failed")
+      return False
+
+    cloudlog.event("upload_start", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if sz == 0:
+      # tag files of 0 size as uploaded
+      success = True
+    elif name in MAX_UPLOAD_SIZES and sz > MAX_UPLOAD_SIZES[name]:
+      cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz)
+      success = True
+    else:
+      start_time = time.monotonic()
+
+      stat = None
+      last_exc = None
+      try:
+        stat = self.do_upload(key, fn)
+      except Exception as e:
+        last_exc = (e, traceback.format_exc())
+
+      if stat is not None and stat.status_code in (200, 201, 401, 403, 412):
+        self.last_filename = fn
+        dt = time.monotonic() - start_time
+        if stat.status_code == 412:
+          cloudlog.event("upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+        else:
+          content_length = int(stat.request.headers.get("Content-Length", 0))
+          speed = (content_length / 1e6) / dt
+          cloudlog.event("upload_success", key=key, fn=fn, sz=sz, content_length=content_length,
+                         network_type=network_type, metered=metered, speed=speed)
+        success = True
+      else:
+        success = False
+        cloudlog.event("upload_failed", stat=stat, exc=last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if success:
+      # tag file as uploaded
+      try:
+        setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
+      except OSError:
+        cloudlog.event("uploader_setxattr_failed", exc=last_exc, key=key, fn=fn, sz=sz)
+
+    return success
+
+
+  def step(self, network_type: int, metered: bool) -> bool | None:
+    d = self.next_file_to_upload(metered)
+    if d is None:
+      return None
+
+    name, key, fn = d
+
+    # qlogs and bootlogs need to be compressed before uploading
+    if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.zst')):
+      key += ".zst"
+
+    return self.upload(name, key, fn, network_type, metered)
+
+
+def main(exit_event: threading.Event = None) -> None:
+  if exit_event is None:
+    exit_event = threading.Event()
+
+  try:
+    set_core_affinity([0, 1, 2, 3])
+  except Exception:
+    cloudlog.exception("failed to set core affinity")
+
+  clear_locks(Paths.log_root())
+
+  params = Params()
+  dongle_id = params.get("DongleId", encoding='utf8')
+
+  if dongle_id is None:
+    cloudlog.info("uploader missing dongle_id")
+    raise Exception("uploader can't start without dongle id")
+
+  sm = messaging.SubMaster(['deviceState'])
+  uploader = Uploader(dongle_id, Paths.log_root())
+
+  backoff = 0.1
+  while not exit_event.is_set():
+    sm.update(0)
+    offroad = params.get_bool("IsOffroad")
+    network_type = sm['deviceState'].networkType if not force_wifi else NetworkType.wifi
+    if network_type == NetworkType.none:
+      if allow_sleep:
+        time.sleep(60 if offroad else 5)
+      continue
+
+    success = uploader.step(sm['deviceState'].networkType.raw, sm['deviceState'].networkMetered)
+    if success is None:
+      backoff = 60 if offroad else 5
+    elif success:
+      backoff = 0.1
+    else:
+      cloudlog.info("upload backoff %r", backoff)
+      backoff = min(backoff*2, 120)
+    if allow_sleep:
+      time.sleep(backoff + random.uniform(0, backoff))
+
+
+if __name__ == "__main__":
+  main()

system/loggerd/video_writer.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <string>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
+
+#include "cereal/messaging/messaging.h"
+
+class VideoWriter {
+public:
+  VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec);
+  void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe);
+  ~VideoWriter();
+private:
+  std::string vid_path, lock_path;
+  FILE *of = nullptr;
+
+  AVCodecContext *codec_ctx;
+  AVFormatContext *ofmt_ctx;
+  AVStream *out_stream;
+  bool remuxing;
+};

system/loggerd/xattr_cache.py

@@ -0,0 +1,22 @@
+import os
+import errno
+
+_cached_attributes: dict[tuple, bytes | None] = {}
+
+def getxattr(path: str, attr_name: str) -> bytes | None:
+  key = (path, attr_name)
+  if key not in _cached_attributes:
+    try:
+      response = os.getxattr(path, attr_name)
+    except OSError as e:
+      # ENODATA means attribute hasn't been set
+      if e.errno == errno.ENODATA:
+        response = None
+      else:
+        raise
+    _cached_attributes[key] = response
+  return _cached_attributes[key]
+
+def setxattr(path: str, attr_name: str, attr_value: bytes) -> None:
+  _cached_attributes.pop((path, attr_name), None)
+  return os.setxattr(path, attr_name, attr_value)

system/loggerd/zstd_writer.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <zstd.h>
+
+#include <string>
+#include <vector>
+#include <capnp/common.h>
+
+class ZstdFileWriter {
+public:
+  ZstdFileWriter(const std::string &filename, int compression_level);
+  ~ZstdFileWriter();
+  void write(void* data, size_t size);
+  inline void write(kj::ArrayPtr<capnp::byte> array) { write(array.begin(), array.size()); }
+
+private:
+  void flushCache(bool last_chunk);
+
+  size_t input_cache_capacity_ = 0;
+  std::vector<char> input_cache_;
+  std::vector<char> output_buffer_;
+  ZSTD_CStream *cstream_;
+  FILE* file_ = nullptr;
+};