Remove Raw Upload toggle (#23143)

* Remove Raw Upload toggle

* fix tests
old-commit-hash: 2e0c73fc0cbac64e442085390dd22e6b408d0478

selfdrive/common/params.cc

@@ -153,7 +153,6 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"TrainingVersion", PERSISTENT},
     {"UpdateAvailable", CLEAR_ON_MANAGER_START},
     {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
-    {"UploadRaw", PERSISTENT},
     {"Version", PERSISTENT},
     {"VisionRadarToggle", PERSISTENT},
     {"ApiCache_Device", PERSISTENT},

selfdrive/loggerd/tests/loggerd_tests_common.py

@@ -56,7 +56,6 @@ class MockParams():
   def __init__(self):
     self.params = {
       "DongleId": b"0000000000000000",
-      "UploadRaw": b"1",
       "IsOffroad": b"1",
     }
 

selfdrive/loggerd/tests/test_uploader.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+import os
 import time
 import threading
 import unittest
@@ -50,66 +52,72 @@ class TestUploader(UploaderTestCase):
     self.end_event.set()
     self.up_thread.join()
 
-  def gen_files(self, lock=False):
+  def gen_files(self, lock=False, boot=True):
     f_paths = list()
-    for t in ["bootlog.bz2", "qlog.bz2", "rlog.bz2", "dcamera.hevc", "fcamera.hevc"]:
+    for t in ["qlog.bz2", "rlog.bz2", "dcamera.hevc", "fcamera.hevc"]:
       f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock))
+
+    if boot:
+      f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}.bz2", 1, lock=lock))
     return f_paths
 
-  def gen_order(self, seg1, seg2):
-    keys = [f"{self.seg_format.format(i)}/qlog.bz2" for i in seg1]
+  def gen_order(self, seg1, seg2, boot=True):
+    keys = []
+    if boot:
+      keys += [f"boot/{self.seg_format.format(i)}.bz2" for i in seg1]
+      keys += [f"boot/{self.seg_format2.format(i)}.bz2" for i in seg2]
+    keys += [f"{self.seg_format.format(i)}/qlog.bz2" for i in seg1]
     keys += [f"{self.seg_format2.format(i)}/qlog.bz2" for i in seg2]
-    for i in seg1:
-      keys += [f"{self.seg_format.format(i)}/{f}" for f in ['rlog.bz2', 'fcamera.hevc', 'dcamera.hevc']]
-    for i in seg2:
-      keys += [f"{self.seg_format2.format(i)}/{f}" for f in ['rlog.bz2', 'fcamera.hevc', 'dcamera.hevc']]
-    keys += [f"{self.seg_format.format(i)}/bootlog.bz2" for i in seg1]
-    keys += [f"{self.seg_format2.format(i)}/bootlog.bz2" for i in seg2]
     return keys
 
   def test_upload(self):
-    f_paths = self.gen_files(lock=False)
+    self.gen_files(lock=False)
 
     self.start_thread()
     # allow enough time that files could upload twice if there is a bug in the logic
     time.sleep(5)
     self.join_thread()
 
-    self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored")
-    self.assertFalse(len(log_handler.upload_order) < len(f_paths), "Some files failed to upload")
-    self.assertFalse(len(log_handler.upload_order) > len(f_paths), "Some files were uploaded twice")
-    for f_path in f_paths:
-      self.assertTrue(getxattr(f_path, uploader.UPLOAD_ATTR_NAME), "All files not uploaded")
     exp_order = self.gen_order([self.seg_num], [])
+
+    self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored")
+    self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
+    self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
+    for f_path in exp_order:
+      self.assertTrue(getxattr(os.path.join(self.root, f_path), uploader.UPLOAD_ATTR_NAME), "All files not uploaded")
+
     self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
 
   def test_upload_ignored(self):
     self.set_ignore()
-    f_paths = self.gen_files(lock=False)
+    self.gen_files(lock=False)
 
     self.start_thread()
     # allow enough time that files could upload twice if there is a bug in the logic
     time.sleep(5)
     self.join_thread()
 
-    self.assertTrue(len(log_handler.upload_order) == 0, "Some files were not ignored")
-    self.assertFalse(len(log_handler.upload_ignored) < len(f_paths), "Some files failed to ignore")
-    self.assertFalse(len(log_handler.upload_ignored) > len(f_paths), "Some files were ignored twice")
-    for f_path in f_paths:
-      self.assertTrue(getxattr(f_path, uploader.UPLOAD_ATTR_NAME), "All files not ignored")
     exp_order = self.gen_order([self.seg_num], [])
+
+    self.assertTrue(len(log_handler.upload_order) == 0, "Some files were not ignored")
+    self.assertFalse(len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore")
+    self.assertFalse(len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice")
+    for f_path in exp_order:
+      self.assertTrue(getxattr(os.path.join(self.root, f_path), uploader.UPLOAD_ATTR_NAME), "All files not ignored")
+
     self.assertTrue(log_handler.upload_ignored == exp_order, "Files ignored in wrong order")
 
   def test_upload_files_in_create_order(self):
-    f_paths = list()
     seg1_nums = [0, 1, 2, 10, 20]
     for i in seg1_nums:
       self.seg_dir = self.seg_format.format(i)
-      f_paths += self.gen_files()
+      self.gen_files(boot=False)
     seg2_nums = [5, 50, 51]
     for i in seg2_nums:
       self.seg_dir = self.seg_format2.format(i)
-      f_paths += self.gen_files()
+      self.gen_files(boot=False)
+
+    exp_order = self.gen_order(seg1_nums, seg2_nums, boot=False)
 
     self.start_thread()
     # allow enough time that files could upload twice if there is a bug in the logic
@@ -117,15 +125,15 @@ class TestUploader(UploaderTestCase):
     self.join_thread()
 
     self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored")
-    self.assertFalse(len(log_handler.upload_order) < len(f_paths), "Some files failed to upload")
-    self.assertFalse(len(log_handler.upload_order) > len(f_paths), "Some files were uploaded twice")
-    for f_path in f_paths:
-      self.assertTrue(getxattr(f_path, uploader.UPLOAD_ATTR_NAME), "All files not uploaded")
-    exp_order = self.gen_order(seg1_nums, seg2_nums)
+    self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
+    self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
+    for f_path in exp_order:
+      self.assertTrue(getxattr(os.path.join(self.root, f_path), uploader.UPLOAD_ATTR_NAME), "All files not uploaded")
+
     self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
 
   def test_no_upload_with_lock_file(self):
-    f_paths = self.gen_files(lock=True)
+    f_paths = self.gen_files(lock=True, boot=False)
 
     self.start_thread()
     # allow enough time that files should have been uploaded if they would be uploaded
@@ -137,4 +145,4 @@ class TestUploader(UploaderTestCase):
 
 
 if __name__ == "__main__":
-  unittest.main()
+  unittest.main(failfast=True)

selfdrive/loggerd/uploader.py

@@ -60,8 +60,6 @@ class Uploader():
     self.last_resp = None
     self.last_exc = None
 
-    self.raw_size = 0
-    self.raw_count = 0
     self.immediate_size = 0
     self.immediate_count = 0
 
@@ -72,21 +70,16 @@ class Uploader():
 
     self.immediate_folders = ["crash/", "boot/"]
     self.immediate_priority = {"qlog.bz2": 0, "qcamera.ts": 1}
-    self.high_priority = {"rlog.bz2": 0, "fcamera.hevc": 1, "dcamera.hevc": 2, "ecamera.hevc": 3}
 
   def get_upload_sort(self, name):
     if name in self.immediate_priority:
       return self.immediate_priority[name]
-    if name in self.high_priority:
-      return self.high_priority[name] + 100
     return 1000
 
   def list_upload_files(self):
     if not os.path.isdir(self.root):
       return
 
-    self.raw_size = 0
-    self.raw_count = 0
     self.immediate_size = 0
     self.immediate_count = 0
 
@@ -116,38 +109,27 @@ class Uploader():
           if name in self.immediate_priority:
             self.immediate_count += 1
             self.immediate_size += os.path.getsize(fn)
-          else:
-            self.raw_count += 1
-            self.raw_size += os.path.getsize(fn)
         except OSError:
           pass
 
         yield (name, key, fn)
 
-  def next_file_to_upload(self, with_raw):
+  def next_file_to_upload(self):
     upload_files = list(self.list_upload_files())
 
-    # try to upload qlog files first
     for name, key, fn in upload_files:
-      if name in self.immediate_priority or any(f in fn for f in self.immediate_folders):
+      if any(f in fn for f in self.immediate_folders):
         return (key, fn)
 
-    if with_raw:
-      # then upload the full log files, rear and front camera files
-      for name, key, fn in upload_files:
-        if name in self.high_priority:
-          return (key, fn)
-
-      # then upload other files
-      for name, key, fn in upload_files:
-        if not name.endswith('.lock') and not name.endswith(".tmp"):
-          return (key, fn)
+    for name, key, fn in upload_files:
+      if name in self.immediate_priority:
+        return (key, fn)
 
     return None
 
   def do_upload(self, key, fn):
     try:
-      url_resp = self.api.get("v1.4/"+self.dongle_id+"/upload_url/", timeout=10, path=key, access_token=self.api.get_token())
+      url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token())
       if url_resp.status_code == 412:
         self.last_resp = url_resp
         return
@@ -226,8 +208,6 @@ class Uploader():
   def get_msg(self):
     msg = messaging.new_message("uploaderState")
     us = msg.uploaderState
-    us.rawQueueSize = int(self.raw_size / 1e6)
-    us.rawQueueCount = self.raw_count
     us.immediateQueueSize = int(self.immediate_size / 1e6)
     us.immediateQueueCount = self.immediate_count
     us.lastTime = self.last_time
@@ -260,10 +240,7 @@ def uploader_fn(exit_event):
         time.sleep(60 if offroad else 5)
       continue
 
-    good_internet = network_type in [NetworkType.wifi, NetworkType.ethernet]
-    allow_raw_upload = params.get_bool("UploadRaw")
-
-    d = uploader.next_file_to_upload(with_raw=allow_raw_upload and good_internet and offroad)
+    d = uploader.next_file_to_upload()
     if d is None:  # Nothing to upload
       if allow_sleep:
         time.sleep(60 if offroad else 5)

selfdrive/ui/qt/offroad/settings.cc

@@ -59,12 +59,6 @@ TogglesPanel::TogglesPanel(SettingsWindow *parent) : ListWidget(parent) {
       "Use features, such as community supported hardware, from the open source community that are not maintained or supported by comma.ai and have not been confirmed to meet the standard safety model. Be extra cautious when using these features",
       "../assets/offroad/icon_shell.png",
     },
-    {
-      "UploadRaw",
-      "Upload Raw Logs",
-      "Upload full logs and full resolution video by default while on Wi-Fi. If not enabled, individual logs can be marked for upload at useradmin.comma.ai.",
-      "../assets/offroad/icon_network.png",
-    },
     {
       "RecordFront",
       "Record and Upload Driver Camera",