StarPilot

system/manager/build.py

@@ -29,13 +29,22 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
   if AGNOS:
     HARDWARE.set_power_save(False)
     os.sched_setaffinity(0, range(8))  # ensure we can use the isolcpus cores
+    # Extremely conservative on-device compile to avoid lowmemorykiller SIGKILLs.
+    attempts = [1]
+  else:
+    attempts = [nproc, max(1, nproc // 2), 1]
 
-  # building with all cores can result in using too
-  # much memory, so retry with less parallelism
+  # Preserve order while de-duplicating.
+  attempts = list(dict.fromkeys(max(1, int(n)) for n in attempts))
+
+  # building with all cores can result in using too much memory,
+  # so retry with less parallelism.
   compile_output: list[bytes] = []
-  for n in (nproc, nproc/2, 1):
-    compile_output.clear()
-    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+  last_returncode = 0
+
+  def run_scons(n: int, cache_args: list[str]) -> int:
+    nonlocal compile_output, spinner, env
+    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", *cache_args, *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
     assert scons.stderr is not None
 
     # Read progress from stderr and update spinner
@@ -56,14 +65,27 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
       except Exception:
         pass
 
-    if scons.returncode == 0:
+    if scons.returncode != 0 and scons.stderr is not None:
+      compile_output += scons.stderr.read().split(b'\n')
+    return scons.returncode
+
+  for n in attempts:
+    compile_output.clear()
+    last_returncode = run_scons(n, ["--cache-populate"])
+    if last_returncode == 0:
       break
 
-  if scons.returncode != 0:
-    # Read remaining output
-    if scons.stderr is not None:
-      compile_output += scons.stderr.read().split(b'\n')
+  if last_returncode != 0:
+    # OOM-ish builds often fail with "Error -9" when clang gets SIGKILL.
+    # Do one final conservative retry before surfacing a hard failure.
+    output_blob = b"\n".join(compile_output)
+    if AGNOS and b"Error -9" in output_blob:
+      cloudlog.warning("scons likely OOM-killed (Error -9), retrying with -j1 --cache-disable")
+      print("Build retry: detected Error -9, retrying with -j1 --cache-disable")
+      compile_output.clear()
+      last_returncode = run_scons(1, ["--cache-disable"])
 
+  if last_returncode != 0:
     # Build failed log errors
     error_s = b"\n".join(compile_output).decode('utf8', 'replace')
     add_file_handler(cloudlog)

system/manager/manager.py

@@ -1,16 +1,18 @@
 #!/usr/bin/env python3
 import datetime
+import json
 import os
+from pathlib import Path
 import signal
 import sys
 import time
 import traceback
 
-from cereal import log
+from cereal import car, log
 import cereal.messaging as messaging
 import openpilot.system.sentry as sentry
 from openpilot.common.utils import atomic_write
-from openpilot.common.params import Params, ParamKeyFlag
+from openpilot.common.params import Params, ParamKeyFlag, ParamKeyType
 from openpilot.common.text_window import TextWindow
 from openpilot.system.hardware import HARDWARE
 from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog
@@ -25,6 +27,229 @@ from openpilot.frogpilot.common.frogpilot_functions import frogpilot_boot_functi
 from openpilot.frogpilot.common.frogpilot_variables import get_frogpilot_toggles
 
 
+LEGACY_BOLT_FP_MIGRATION_FLAG = Path("/data") / "legacy_bolt_fp_migration_v1"
+STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG = Path("/data") / "starpilot_defaults_parity_v1"
+STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG = Path("/data") / "starpilot_param_canonicalization_v1"
+LEGACY_CARMODEL_MIGRATIONS = {
+  "CHEVROLET_BOLT_CC_2019_2021": "CHEVROLET_BOLT_CC_2018_2021",
+}
+
+
+def _to_text(value):
+  if value is None:
+    return None
+  if isinstance(value, bytes):
+    return value.decode("utf-8", errors="ignore")
+  return str(value)
+
+
+def migrate_legacy_bolt_fingerprint(params: Params) -> None:
+  old_fp, new_fp = next(iter(LEGACY_CARMODEL_MIGRATIONS.items()))
+  carparams_keys = ("CarParams", "CarParamsCache", "CarParamsPersistent", "CarParamsPrevRoute")
+  keys_to_clear = (
+    "CarParams",
+    "CarParamsCache",
+    "CarParamsPersistent",
+    "CarParamsPrevRoute",
+    "FrogPilotCarParams",
+    "FrogPilotCarParamsPersistent",
+  )
+
+  car_model = _to_text(params.get("CarModel"))
+  legacy_detected = car_model == old_fp
+  if not legacy_detected:
+    old_fp_bytes = old_fp.encode()
+    for key in carparams_keys:
+      raw = params.get(key)
+      if raw is None:
+        continue
+
+      raw_bytes = raw if isinstance(raw, bytes) else str(raw).encode()
+      # Fast path for payloads that still embed the legacy fingerprint string.
+      if old_fp_bytes in raw_bytes:
+        legacy_detected = True
+        break
+
+      # Fallback decode for payloads that don't expose the raw string directly.
+      try:
+        with car.CarParams.from_bytes(raw_bytes) as cp:
+          if cp.carFingerprint == old_fp:
+            legacy_detected = True
+            break
+      except Exception:
+        continue
+
+  if not legacy_detected:
+    return
+
+  cleared_keys: list[str] = []
+  for key in keys_to_clear:
+    if params.get(key) is None:
+      continue
+    params.remove(key)
+    cleared_keys.append(key)
+
+  if car_model == old_fp:
+    params.put("CarModel", new_fp)
+  car_model_name = _to_text(params.get("CarModelName")) or ""
+  if "2019-21" in car_model_name:
+    params.put("CarModelName", car_model_name.replace("2019-21", "2018-21"))
+
+  cloudlog.warning(
+    f"Detected legacy Bolt fingerprint {old_fp}; cleared={cleared_keys}, remapped CarModel to {new_fp}"
+  )
+
+  try:
+    LEGACY_BOLT_FP_MIGRATION_FLAG.parent.mkdir(parents=True, exist_ok=True)
+    LEGACY_BOLT_FP_MIGRATION_FLAG.write_text(f"{datetime.datetime.now(datetime.UTC).isoformat()}\n")
+  except Exception:
+    cloudlog.exception(f"Failed to write migration flag: {LEGACY_BOLT_FP_MIGRATION_FLAG}")
+
+
+def migrate_starpilot_default_parity(params: Params, params_cache: Params) -> None:
+  if STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG.exists():
+    return
+
+  desired_bool_values = {
+    "AdvancedLateralTune": True,
+    "ForceAutoTuneOff": True,
+    "HumanAcceleration": False,
+    "HumanFollowing": False,
+    "NNFF": False,
+    "NNFFLite": False,
+  }
+
+  for key, value in desired_bool_values.items():
+    params.put_bool(key, value)
+    params_cache.put_bool(key, value)
+
+  params.put_float("CEModelStopTime", 7.0)
+  params_cache.put_float("CEModelStopTime", 7.0)
+
+  cloudlog.warning("Applied one-time StarPilot default parity migration for lateral/longitudinal toggles")
+
+  try:
+    STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG.parent.mkdir(parents=True, exist_ok=True)
+    STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG.write_text(f"{datetime.datetime.now(datetime.UTC).isoformat()}\n")
+  except Exception:
+    cloudlog.exception(f"Failed to write migration flag: {STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG}")
+
+
+def _read_raw_param_bytes(params: Params, key: str | bytes):
+  try:
+    path = params.get_param_path(key)
+  except Exception:
+    return None
+
+  if not path or not os.path.isfile(path):
+    return None
+
+  try:
+    with open(path, "rb") as f:
+      return f.read()
+  except Exception:
+    return None
+
+
+def _parse_legacy_time(raw_text: str):
+  text = raw_text.strip()
+  if not text:
+    return None
+
+  try:
+    return datetime.datetime.fromisoformat(text)
+  except ValueError:
+    pass
+
+  for fmt in ("%B %d, %Y - %I:%M%p", "%B %d, %Y - %I:%M %p"):
+    try:
+      return datetime.datetime.strptime(text, fmt)
+    except ValueError:
+      continue
+
+  return None
+
+
+def migrate_param_type_canonicalization(params: Params) -> None:
+  if STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG.exists():
+    return
+
+  normalized_keys: list[str] = []
+
+  for raw_key in params.all_keys():
+    key = raw_key.decode() if isinstance(raw_key, bytes) else str(raw_key)
+    raw_value = _read_raw_param_bytes(params, raw_key)
+    if not raw_value:
+      continue
+
+    try:
+      text_value = raw_value.decode("utf-8", errors="strict").strip()
+    except UnicodeDecodeError:
+      continue
+
+    if not text_value:
+      continue
+
+    try:
+      expected_type = params.get_type(raw_key)
+    except Exception:
+      continue
+
+    try:
+      if expected_type == ParamKeyType.INT:
+        parsed = float(text_value)
+        # Canonicalize decimal/exponent forms into integer storage.
+        canonical = str(int(parsed))
+        if canonical != text_value:
+          params.put_int(raw_key, int(parsed))
+          normalized_keys.append(key)
+
+      elif expected_type == ParamKeyType.FLOAT:
+        parsed = float(text_value)
+        canonical = str(parsed)
+        if canonical != text_value:
+          params.put_float(raw_key, parsed)
+          normalized_keys.append(key)
+
+      elif expected_type == ParamKeyType.BOOL:
+        lowered = text_value.lower()
+        if lowered in ("1", "true", "yes", "on"):
+          if text_value != "1":
+            params.put_bool(raw_key, True)
+            normalized_keys.append(key)
+        elif lowered in ("0", "false", "no", "off"):
+          if text_value != "0":
+            params.put_bool(raw_key, False)
+            normalized_keys.append(key)
+
+      elif expected_type == ParamKeyType.TIME:
+        dt = _parse_legacy_time(text_value)
+        if dt is not None:
+          if dt.tzinfo is not None:
+            dt = dt.astimezone(datetime.UTC).replace(tzinfo=None)
+          if text_value != dt.isoformat():
+            params.put(raw_key, dt)
+            normalized_keys.append(key)
+
+      elif expected_type == ParamKeyType.JSON:
+        parsed = json.loads(text_value)
+        canonical = json.dumps(parsed, separators=(",", ":"))
+        if canonical != text_value:
+          params.put(raw_key, parsed)
+          normalized_keys.append(key)
+    except Exception:
+      continue
+
+  if normalized_keys:
+    cloudlog.warning(f"Canonicalized legacy param values for {len(normalized_keys)} keys")
+
+  try:
+    STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG.parent.mkdir(parents=True, exist_ok=True)
+    STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG.write_text(f"{datetime.datetime.now(datetime.UTC).isoformat()}\n")
+  except Exception:
+    cloudlog.exception(f"Failed to write migration flag: {STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG}")
+
+
 def manager_init() -> None:
   save_bootlog()
 
@@ -42,7 +267,14 @@ def manager_init() -> None:
     params.put_bool("RecordFront", True)
 
   # FrogPilot variables
-  params_cache = Params("/cache/params", return_defaults=True)
+  cache_params_path = "/cache/params"
+  if HARDWARE.get_device_type() == "pc":
+    cache_params_path = os.path.join(Paths.comma_home(), "cache", "params")
+  params_cache = Params(cache_params_path, return_defaults=True)
+
+  # Canonicalize legacy string encodings (e.g. INT params stored as "26.000000")
+  # before bulk reads below to avoid repeated cast warnings and UI-side churn.
+  migrate_param_type_canonicalization(params)
 
   # set unset params to their default value
   for k in params.all_keys():
@@ -75,6 +307,10 @@ def manager_init() -> None:
   params.put_bool("IsReleaseBranch", build_metadata.release_channel)
   params.put("HardwareSerial", serial)
 
+  # Branch migration: rename legacy Bolt fingerprint persisted in CarParams.
+  migrate_legacy_bolt_fingerprint(params)
+  migrate_starpilot_default_parity(params, params_cache)
+
   # set dongle id
   reg_res = register(show_spinner=True)
   if reg_res:

system/manager/process.py

@@ -22,8 +22,11 @@ from openpilot.common.watchdog import WATCHDOG_FN
 ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None
 
 
-def launcher(proc: str, name: str) -> None:
+def launcher(proc: str, name: str, nice: int | None = None) -> None:
   try:
+    if nice is not None:
+      os.nice(nice)
+
     # import the process
     mod = importlib.import_module(proc)
 
@@ -48,7 +51,10 @@ def launcher(proc: str, name: str) -> None:
     raise
 
 
-def nativelauncher(pargs: list[str], cwd: str, name: str) -> None:
+def nativelauncher(pargs: list[str], cwd: str, name: str, nice: int | None = None) -> None:
+  if nice is not None:
+    os.nice(nice)
+
   os.environ['MANAGER_DAEMON'] = name
 
   # exec the process
@@ -168,7 +174,7 @@ class ManagerProcess(ABC):
 
 
 class NativeProcess(ManagerProcess):
-  def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
+  def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, watchdog_max_dt=None, nice=None):
     self.name = name
     self.cwd = cwd
     self.cmdline = cmdline
@@ -176,6 +182,7 @@ class NativeProcess(ManagerProcess):
     self.enabled = enabled
     self.sigkill = sigkill
     self.watchdog_max_dt = watchdog_max_dt
+    self.nice = nice
     self.launcher = nativelauncher
 
   def prepare(self) -> None:
@@ -191,20 +198,21 @@ class NativeProcess(ManagerProcess):
 
     cwd = os.path.join(BASEDIR, self.cwd)
     cloudlog.info(f"starting process {self.name}")
-    self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name))
+    self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name, self.nice))
     self.proc.start()
     self.watchdog_seen = False
     self.shutting_down = False
 
 
 class PythonProcess(ManagerProcess):
-  def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
+  def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None, nice=None):
     self.name = name
     self.module = module
     self.should_run = should_run
     self.enabled = enabled
     self.sigkill = sigkill
     self.watchdog_max_dt = watchdog_max_dt
+    self.nice = nice
     self.launcher = launcher
 
   def prepare(self) -> None:
@@ -225,7 +233,7 @@ class PythonProcess(ManagerProcess):
     name = self.name if "modeld" not in self.name else "MainProcess"
 
     cloudlog.info(f"starting python {self.module}")
-    self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name))
+    self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name, self.nice))
     self.proc.start()
     self.watchdog_seen = False
     self.shutting_down = False
@@ -284,6 +292,11 @@ def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None
 
   running = []
   for p in procs:
+    # Reap crashed processes so they can be cleanly restarted below.
+    if p.proc is not None and p.proc.exitcode is not None and not p.shutting_down:
+      cloudlog.error(f"Process {p.name} crashed with exitcode {p.proc.exitcode}, restarting")
+      p.stop(retry=False)
+
     if p.enabled and p.name not in not_run and p.should_run(started, params, CP, frogpilot_toggles):
       running.append(p)
     else:

system/manager/process_config.py

@@ -68,7 +68,7 @@ def allow_logging(started: bool, params: Params, CP: car.CarParams, frogpilot_to
   return not frogpilot_toggles.no_logging
 
 def allow_uploads(started: bool, params: Params, CP: car.CarParams, frogpilot_toggles: SimpleNamespace) -> bool:
-  return not frogpilot_toggles.no_uploads or frogpilot_toggles.no_onroad_uploads
+  return params.get_bool("AlwaysAllowUploads") or not frogpilot_toggles.no_uploads or frogpilot_toggles.no_onroad_uploads
 
 def run_speed_limit_filler(started: bool, params: Params, CP: car.CarParams, frogpilot_toggles: SimpleNamespace) -> bool:
   return frogpilot_toggles.speed_limit_filler
@@ -88,8 +88,8 @@ procs = [
   PythonProcess("micd", "system.micd", iscar),
   PythonProcess("timed", "system.timed", always_run, enabled=not PC),
 
-  PythonProcess("modeld", "selfdrive.modeld.modeld", only_onroad),
-  PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(WEBCAM or not PC)),
+  PythonProcess("modeld", "frogpilot.tinygrad_modeld.tinygrad_modeld", only_onroad),
+  PythonProcess("dmonitoringmodeld", "frogpilot.tinygrad_modeld.dmonitoringmodeld", driverview, enabled=(WEBCAM or not PC)),
 
   PythonProcess("sensord", "system.sensord.sensord", only_onroad, enabled=not PC),
   PythonProcess("soundd", "selfdrive.ui.soundd", driverview),
@@ -127,14 +127,18 @@ procs = [
 ]
 
 # FrogPilot variables
-if HARDWARE.get_device_type() == "mici":
-  procs.append(PythonProcess("ui", "selfdrive.ui.ui", always_run))
-elif TICI:
-  procs.append(NativeProcess("ui", "selfdrive/ui", ["./ui"], always_run, watchdog_max_dt=5)),
+device_type = HARDWARE.get_device_type()
+if device_type in ("tici", "tizi"):
+  procs.append(NativeProcess("ui", "selfdrive/ui", ["./ui"], always_run, watchdog_max_dt=5))
+else:
+  # C4 (mici) runs the Python raylib UI path; keep watchdog parity with C3/C3X.
+  procs.append(PythonProcess("ui", "selfdrive.ui.ui", always_run, watchdog_max_dt=5))
 procs += [
   PythonProcess("device_syncd", "frogpilot.system.device_syncd", always_run),
   PythonProcess("frogpilot_process", "frogpilot.frogpilot_process", always_run),
   NativeProcess("mapd", "frogpilot/navigation", ["./mapd"], always_run),
+  PythonProcess("the_pond", "frogpilot.system.the_pond.the_pond", always_run, nice=19),
+  PythonProcess("galaxy", "frogpilot.system.galaxy.galaxy", always_run, nice=19),
   PythonProcess("speed_limit_filler", "frogpilot.system.speed_limit_filler", run_speed_limit_filler),
 ]