Merge branch 'upstream/master' into sync-20260517-new-new

common/file_chunker.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+import sys
 import math
 import os
 from pathlib import Path
@@ -10,10 +12,13 @@ def get_chunk_name(name, idx, num_chunks):
 def get_manifest_path(name):
   return f"{name}.chunkmanifest"
 
-def get_chunk_paths(path, file_size):
-  num_chunks = math.ceil(file_size / CHUNK_SIZE)
+def _chunk_paths(path, num_chunks):
   return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
 
+def get_chunk_targets(path, file_size):
+  num_chunks = math.ceil(file_size / CHUNK_SIZE)
+  return _chunk_paths(path, num_chunks)
+
 def chunk_file(path, targets):
   manifest_path, *chunk_paths = targets
   with open(path, 'rb') as f:
@@ -26,6 +31,13 @@ def chunk_file(path, targets):
   Path(manifest_path).write_text(str(len(chunk_paths)))
   os.remove(path)
 
+def get_existing_chunks(path):
+  if os.path.isfile(path):
+    return [path]
+  if os.path.isfile(manifest := get_manifest_path(path)):
+    num_chunks = int(Path(manifest).read_text().strip())
+    return _chunk_paths(path, num_chunks)
+  raise FileNotFoundError(path)
 
 def read_file_chunked(path):
   manifest_path = get_manifest_path(path)
@@ -35,3 +47,9 @@ def read_file_chunked(path):
   if os.path.isfile(path):
     return Path(path).read_bytes()
   raise FileNotFoundError(path)
+
+
+if __name__ == "__main__":
+  path = sys.argv[1]
+  chunk_paths = get_chunk_targets(path, os.path.getsize(path))
+  chunk_file(path, chunk_paths)

common/parameterized.py

@@ -1,8 +1,13 @@
+import re
 import sys
 import pytest
 import inspect
 
 
+def _to_safe_name(s):
+  return re.sub(r"[^a-zA-Z0-9_]+", "_", str(s)).strip("_")
+
+
 class parameterized:
   @staticmethod
   def expand(cases):
@@ -34,7 +39,9 @@ def parameterized_class(attrs, input_list=None):
   def decorator(cls):
     globs = sys._getframe(1).f_globals
     for i, params in enumerate(params_list):
-      name = f"{cls.__name__}_{i}"
+      # append sanitized string param values so pytest -k can filter by them
+      suffix = "_".join(filter(None, (_to_safe_name(v) for v in params.values() if isinstance(v, str))))
+      name = f"{cls.__name__}_{i}" + (f"_{suffix}" if suffix else "")
       new_cls = type(name, (cls,), dict(params))
       new_cls.__module__ = cls.__module__
       new_cls.__test__ = True  # override inherited False so pytest collects this subclass

common/params.py

@@ -14,6 +14,6 @@ if __name__ == "__main__":
   if len(sys.argv) == 3:
     val = sys.argv[2]
     print(f"SET: {key} = {val}")
-    params.put(key, val)
+    params.put(key, val, block=True)
   elif len(sys.argv) == 2:
     print(f"GET: {key} = {params.get(key)}")

common/params_keys.h

@@ -80,6 +80,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
     {"LiveParameters", {PERSISTENT, JSON}},
     {"LiveParametersV2", {PERSISTENT, BYTES}},
+    {"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
     {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
     {"LocationFilterInitialState", {PERSISTENT, BYTES}},
     {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
@@ -103,8 +104,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
     {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
     {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
-    {"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
     {"PrimeType", {PERSISTENT, INT}},
     {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
     {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -132,6 +131,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
     {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
     {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
     {"Version", {PERSISTENT, STRING}},
 
     // --- sunnypilot params --- //

common/params_pyx.pyx

@@ -142,33 +142,28 @@ cdef class Params:
     cdef ParamKeyType t = self.p.getKeyType(k)
     return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
 
-  def put(self, key, dat):
+  def put(self, key, dat, bool block = False):
     """
-    Warning: This function blocks until the param is written to disk!
+    Warning: block=True blocks until the param is written to disk!
     In very rare cases this can take over a second, and your code will hang.
-    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
-    in general try to avoid writing params as much as possible.
+    Use block=False in time sensitive code, but in general try to avoid
+    writing params as much as possible.
     """
     cdef string k = self.check_key(key)
     cdef string dat_bytes = self._put_cast(key, dat)
     with nogil:
-      self.p.put(k, dat_bytes)
+      if block:
+        self.p.put(k, dat_bytes)
+      else:
+        self.p.putNonBlocking(k, dat_bytes)
 
-  def put_bool(self, key, bool val):
+  def put_bool(self, key, bool val, bool block = False):
     cdef string k = self.check_key(key)
     with nogil:
-      self.p.putBool(k, val)
-
-  def put_nonblocking(self, key, dat):
-    cdef string k = self.check_key(key)
-    cdef string dat_bytes = self._put_cast(key, dat)
-    with nogil:
-      self.p.putNonBlocking(k, dat_bytes)
-
-  def put_bool_nonblocking(self, key, bool val):
-    cdef string k = self.check_key(key)
-    with nogil:
-      self.p.putBoolNonBlocking(k, val)
+      if block:
+        self.p.putBool(k, val)
+      else:
+        self.p.putBoolNonBlocking(k, val)
 
   def remove(self, key):
     cdef string k = self.check_key(key)

common/realtime.py

@@ -28,6 +28,11 @@ class Priority:
   CTRL_HIGH = 53
 
 
+def drop_realtime() -> None:
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
+
+
 def set_core_affinity(cores: list[int]) -> None:
   if sys.platform == 'linux' and not PC:
     os.sched_setaffinity(0, cores)

common/swaglog.cc

@@ -11,7 +11,7 @@
 
 #include <zmq.h>
 #include <stdarg.h>
-#include "third_party/json11/json11.hpp"
+#include "json11/json11.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"
 

common/tests/test_params.py

@@ -12,17 +12,17 @@ class TestParams:
     self.params = Params()
 
   def test_params_put_and_get(self):
-    self.params.put("DongleId", "cb38263377b873ee")
+    self.params.put("DongleId", "cb38263377b873ee", block=True)
     assert self.params.get("DongleId") == "cb38263377b873ee"
 
   def test_params_non_ascii(self):
     st = b"\xe1\x90\xff"
-    self.params.put("CarParams", st)
+    self.params.put("CarParams", st, block=True)
     assert self.params.get("CarParams") == st
 
   def test_params_get_cleared_manager_start(self):
-    self.params.put("CarParams", b"test")
-    self.params.put("DongleId", "cb38263377b873ee")
+    self.params.put("CarParams", b"test", block=True)
+    self.params.put("DongleId", "cb38263377b873ee", block=True)
     assert self.params.get("CarParams") == b"test"
 
     undefined_param = self.params.get_param_path(uuid.uuid4().hex)
@@ -36,15 +36,15 @@ class TestParams:
     assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
-    self.params.put("DongleId", "bob")
-    self.params.put("AthenadPid", 123)
+    self.params.put("DongleId", "bob", block=True)
+    self.params.put("AthenadPid", 123, block=True)
     assert self.params.get("DongleId") == "bob"
     assert self.params.get("AthenadPid") == 123
 
   def test_params_get_block(self):
     def _delayed_writer():
       time.sleep(0.1)
-      self.params.put("CarParams", b"test")
+      self.params.put("CarParams", b"test", block=True)
     threading.Thread(target=_delayed_writer).start()
     assert self.params.get("CarParams") is None
     assert self.params.get("CarParams", block=True) == b"test"
@@ -57,10 +57,10 @@ class TestParams:
       self.params.get_bool("swag")
 
     with pytest.raises(UnknownKeyName):
-      self.params.put("swag", "abc")
+      self.params.put("swag", "abc", block=True)
 
     with pytest.raises(UnknownKeyName):
-      self.params.put_bool("swag", True)
+      self.params.put_bool("swag", True, block=True)
 
   def test_remove_not_there(self):
     assert self.params.get("CarParams") is None
@@ -71,23 +71,23 @@ class TestParams:
     self.params.remove("IsMetric")
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put_bool("IsMetric", True)
+    self.params.put_bool("IsMetric", True, block=True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put_bool("IsMetric", False)
+    self.params.put_bool("IsMetric", False, block=True)
     assert not self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", True)
+    self.params.put("IsMetric", True, block=True)
     assert self.params.get_bool("IsMetric")
 
-    self.params.put("IsMetric", False)
+    self.params.put("IsMetric", False, block=True)
     assert not self.params.get_bool("IsMetric")
 
   def test_put_non_blocking_with_get_block(self):
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_nonblocking("CarParams", b"test")
+      Params().put("CarParams", b"test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
@@ -96,7 +96,7 @@ class TestParams:
     q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      Params().put_bool_nonblocking("CarParams", True)
+      Params().put_bool("CarParams", True)
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"1"
@@ -123,19 +123,19 @@ class TestParams:
 
   def test_params_get_type(self):
     # json
-    self.params.put("ApiCache_FirehoseStats", {"a": 0})
+    self.params.put("ApiCache_FirehoseStats", {"a": 0}, block=True)
     assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
 
     # int
-    self.params.put("BootCount", 1441)
+    self.params.put("BootCount", 1441, block=True)
     assert self.params.get("BootCount") == 1441
 
     # bool
-    self.params.put("AdbEnabled", True)
+    self.params.put("AdbEnabled", True, block=True)
     assert self.params.get("AdbEnabled")
     assert isinstance(self.params.get("AdbEnabled"), bool)
 
     # time
     now = datetime.datetime.now(datetime.UTC)
-    self.params.put("InstallDate", now)
+    self.params.put("InstallDate", now, block=True)
     assert self.params.get("InstallDate") == now

common/tests/test_swaglog.cc

@@ -7,7 +7,7 @@
 #include "common/util.h"
 #include "common/version.h"
 #include "system/hardware/hw.h"
-#include "third_party/json11/json11.hpp"
+#include "json11/json11.hpp"
 
 #include "sunnypilot/common/version.h"
 

common/utils.py

@@ -48,7 +48,7 @@ def sudo_write(val: str, path: str) -> None:
 
 def sudo_read(path: str) -> str:
   try:
-    return subprocess.check_output(f"sudo cat {path}", shell=True, encoding='utf8').strip()
+    return subprocess.check_output(["sudo", "cat", "--", path], encoding='utf8').strip()
   except Exception:
     return ""
 

common/version.h

@@ -1 +1 @@
-#define COMMA_VERSION "0.11.1"
+#define COMMA_VERSION "0.11.2"