version: dragonpilot development version for EON/C2

date: 2024-04-10T19:21:57
commit: 8f2124beef4031cf1ad7a123b5fe1eb4985aab4f

CHANGELOGS-D2.md

@@ -0,0 +1,40 @@
+2024-04-10
+========================
+* Attempt to fix honda op long issue (take 2).
+
+2024-03-20
+========================
+* Bug fixes
+
+2024-03-19
+========================
+* Attempt to fix 70 mins LKAS/Harness error on Toyotas.
+
+2024-03-01
+========================
+* Display correct changelogs.
+
+
+2024-02-29
+========================
+* Added Lead Vehicle Warning
+* Added Disable Auto Updates toggle
+* Reverted panda back to last working version for honda with minimal changes from op master. (breaks red panda support for now)
+* Added EON Installer (https://raw.githubusercontent.com/dragonpilot-community/dragonpilot/d2/scripts/eon_installer.sh)
+* Conditionally include red panda firmware.
+* Enabled branch selector.
+* Reverted transform patch.
+
+2024-02-27
+========================
+* Fixed door lock/unlock for Toyotas.
+* otisserv and fileserv only for offroad.
+* Enabling otisserv for offroad status and snapshot.
+* Toyota: improve longitudinal control (https://github.com/commaai/openpilot/pull/30697)
+* Updated manager/ modules
+
+2024-02-26
+========================
+* applied transform patch. (https://github.com/commaai/openpilot/pull/31495)
+* adjust de2e param accordingly for transform patch.
+* increase lead sensitivity.

CHANGELOGS-R2.md

@@ -0,0 +1,11 @@
+2024-02-06
+========================
+* NEW:
+    * Rainbow colored Path
+* BUGFIXES:
+    * Toggle for Flight Panel
+* NEW VEHICLES:
+    * Hyundai Staria 2023
+    * Kia Niro Plug-in Hybrid 2022
+    * Toyota RAV4 2023-24
+    * Toyota RAV4 Hybrid 2023-24

common/swaglog.py

@@ -0,0 +1,138 @@
+'''
+rick:
+this file is identical to system/swaglog.py, put a copy here to reduce code changes when updating from op master branch.
+'''
+import logging
+import os
+import time
+import warnings
+from pathlib import Path
+from logging.handlers import BaseRotatingHandler
+
+import zmq
+
+from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
+from openpilot.system.hardware import PC
+
+if PC:
+  SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log")
+else:
+  SWAGLOG_DIR = "/data/log/"
+
+def get_file_handler():
+  Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
+  base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
+  handler = SwaglogRotatingFileHandler(base_filename)
+  return handler
+
+class SwaglogRotatingFileHandler(BaseRotatingHandler):
+  def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
+    super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
+    self.base_filename = base_filename
+    self.interval = interval # seconds
+    self.max_bytes = max_bytes
+    self.backup_count = backup_count
+    self.log_files = self.get_existing_logfiles()
+    log_indexes = [f.split(".")[-1] for f in self.log_files]
+    self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
+    self.last_rollover = None
+    self.doRollover()
+
+  def _open(self):
+    self.last_rollover = time.monotonic()
+    self.last_file_idx += 1
+    next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
+    stream = open(next_filename, self.mode, encoding=self.encoding)
+    self.log_files.insert(0, next_filename)
+    return stream
+
+  def get_existing_logfiles(self):
+    log_files = list()
+    base_dir = os.path.dirname(self.base_filename)
+    for fn in os.listdir(base_dir):
+      fp = os.path.join(base_dir, fn)
+      if fp.startswith(self.base_filename) and os.path.isfile(fp):
+        log_files.append(fp)
+    return sorted(log_files)
+
+  def shouldRollover(self, record):
+    size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
+    time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
+    return size_exceeded or time_exceeded
+
+  def doRollover(self):
+    if self.stream:
+      self.stream.close()
+    self.stream = self._open()
+
+    if self.backup_count > 0:
+      while len(self.log_files) > self.backup_count:
+        to_delete = self.log_files.pop()
+        if os.path.exists(to_delete): # just being safe, should always exist
+          os.remove(to_delete)
+
+class UnixDomainSocketHandler(logging.Handler):
+  def __init__(self, formatter):
+    logging.Handler.__init__(self)
+    self.setFormatter(formatter)
+    self.pid = None
+
+    self.zctx = None
+    self.sock = None
+
+  def __del__(self):
+    if self.sock is not None:
+      self.sock.close()
+    if self.zctx is not None:
+      self.zctx.term()
+
+  def connect(self):
+    self.zctx = zmq.Context()
+    self.sock = self.zctx.socket(zmq.PUSH)
+    self.sock.setsockopt(zmq.LINGER, 10)
+    self.sock.connect("ipc:///tmp/logmessage")
+    self.pid = os.getpid()
+
+  def emit(self, record):
+    if os.getpid() != self.pid:
+      # TODO suppresses warning about forking proc with zmq socket, fix root cause
+      warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
+      self.connect()
+
+    msg = self.format(record).rstrip('\n')
+    # print("SEND".format(repr(msg)))
+    try:
+      s = chr(record.levelno)+msg
+      self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
+    except zmq.error.Again:
+      # drop :/
+      pass
+
+
+def add_file_handler(log):
+  """
+  Function to add the file log handler to swaglog.
+  This can be used to store logs when logmessaged is not running.
+  """
+  handler = get_file_handler()
+  handler.setFormatter(SwagLogFileFormatter(log))
+  log.addHandler(handler)
+
+
+cloudlog = log = SwagLogger()
+log.setLevel(logging.DEBUG)
+
+
+outhandler = logging.StreamHandler()
+
+print_level = os.environ.get('LOGPRINT', 'warning')
+if print_level == 'debug':
+  outhandler.setLevel(logging.DEBUG)
+elif print_level == 'info':
+  outhandler.setLevel(logging.INFO)
+elif print_level == 'warning':
+  outhandler.setLevel(logging.WARNING)
+
+log.addHandler(outhandler)
+# logs are sent through IPC before writing to disk to prevent disk I/O blocking
+log.addHandler(UnixDomainSocketHandler(SwagFormatter(log)))

common/version.h

@@ -1 +1 @@
-#define COMMA_VERSION "2024.02.06"
+#define COMMA_VERSION "2024.04.10"

scripts/eon_installer.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/bash
+rm -fr /data/openpilot ;
+cd /data/ &&
+
+git clone https://github.com/dragonpilot-community/dragonpilot -b "$1" --single-branch --depth=1 openpilot &&
+
+touch /data/data/com.termux/files/continue.sh &&
+echo "#!/usr/bin/bash" >> /data/data/com.termux/files/continue.sh &&
+echo "cd /data/openpilot" >> /data/data/com.termux/files/continue.sh &&
+echo "exec ./launch_openpilot.sh" >> /data/data/com.termux/files/continue.sh &&
+
+chmod u+x /data/data/com.termux/files/continue.sh &&
+reboot

selfdrive/camerad/snapshot/snapshot.py

@@ -28,8 +28,11 @@ def jpeg_write(fn, dat):
   img.save(fn, "JPEG")
 
 
-def extract_image(buf, w, h, stride):
-  img = np.hstack([buf[i * stride:i * stride + 3 * w] for i in range(h)])
+def extract_image(buf):
+  w = buf.width
+  h = buf.height
+  stride = buf.stride
+  img = np.hstack([buf.data[i * stride:i * stride + 3 * w] for i in range(h)])
   b = img[::3].reshape(h, w)
   g = img[1::3].reshape(h, w)
   r = img[2::3].reshape(h, w)
@@ -63,10 +66,10 @@ def get_snapshots(frame="roadCameraState", front_frame="driverCameraState", focu
   rear, front = None, None
   if frame is not None:
     c = vipc_clients[frame]
-    rear = extract_image(c.recv(), c.width, c.height, c.stride)
+    rear = extract_image(c.recv())
   if front_frame is not None:
     c = vipc_clients[front_frame]
-    front = extract_image(c.recv(), c.width, c.height, c.stride)
+    front = extract_image(c.recv())
   return rear, front
 
 

selfdrive/car/toyota/carcontroller.py

@@ -12,6 +12,7 @@ from openpilot.common.params import Params
 
 SteerControlType = car.CarParams.SteerControlType
 VisualAlert = car.CarControl.HUDControl.VisualAlert
+LongCtrlState = car.CarControl.Actuators.LongControlState
 
 # LKA limits
 # EPS faults if you apply torque while the steering rate is above 100 deg/s for too long
@@ -26,6 +27,12 @@ MAX_USER_TORQUE = 500
 MAX_LTA_ANGLE = 94.9461  # deg
 MAX_LTA_DRIVER_TORQUE_ALLOWANCE = 150  # slightly above steering pressed allows some resistance when changing lanes
 
+# PCM compensatory force calculation threshold
+# a variation in accel command is more pronounced at higher speeds, let compensatory forces ramp to zero before
+# applying when speed is high
+COMPENSATORY_CALCULATION_THRESHOLD_V = [-0.3, -0.25, 0.]  # m/s^2
+COMPENSATORY_CALCULATION_THRESHOLD_BP = [0., 11., 23.]  # m/s
+
 # rick - toyota auto lock / unlock
 GearShifter = car.CarState.GearShifter
 UNLOCK_CMD = b'\x40\x05\x30\x11\x00\x40\x00\x00'
@@ -59,6 +66,7 @@ class CarController:
     self.last_standstill = False
     self.standstill_req = False
     self.steer_rate_counter = 0
+    self.prohibit_neg_calculation = True
 
     self.packer = CANPacker(dbc_name)
     self.gas = 0
@@ -89,6 +97,7 @@ class CarController:
     hud_control = CC.hudControl
     pcm_cancel_cmd = CC.cruiseControl.cancel
     lat_active = CC.latActive and abs(CS.out.steeringTorque) < MAX_USER_TORQUE
+    stopping = actuators.longControlState == LongCtrlState.stopping
 
     # *** control msgs ***
     can_sends = []
@@ -211,7 +220,28 @@ class CarController:
       interceptor_gas_cmd = clip(pedal_command, 0., MAX_INTERCEPTOR_GAS)
     else:
       interceptor_gas_cmd = 0.
-    pcm_accel_cmd = clip(actuators.accel, self.params.ACCEL_MIN, self.params.ACCEL_MAX)
+
+    # prohibit negative compensatory calculations when first activating long after accelerator depression or engagement
+    if not CC.longActive:
+      self.prohibit_neg_calculation = True
+    comp_thresh = interp(CS.out.vEgo, COMPENSATORY_CALCULATION_THRESHOLD_BP, COMPENSATORY_CALCULATION_THRESHOLD_V)
+    # don't reset until a reasonable compensatory value is reached
+    if CS.pcm_neutral_force > comp_thresh * self.CP.mass:
+      self.prohibit_neg_calculation = False
+    # NO_STOP_TIMER_CAR will creep if compensation is applied when stopping or stopped, don't compensate when stopped or stopping
+    should_compensate = True
+    if (self.CP.carFingerprint in NO_STOP_TIMER_CAR and actuators.accel < 1e-3 or stopping) or CS.out.vEgo < 1e-3:
+      should_compensate = False
+    # limit minimum to only positive until first positive is reached after engagement, don't calculate when long isn't active
+    if CC.longActive and should_compensate and not self.prohibit_neg_calculation:
+      accel_offset = CS.pcm_neutral_force / self.CP.mass
+    else:
+      accel_offset = 0.
+    # only calculate pcm_accel_cmd when long is active to prevent disengagement from accelerator depression
+    if CC.longActive:
+      pcm_accel_cmd = clip(actuators.accel + accel_offset, self.params.ACCEL_MIN, self.params.ACCEL_MAX)
+    else:
+      pcm_accel_cmd = 0.
 
     # TODO: probably can delete this. CS.pcm_acc_status uses a different signal
     # than CS.cruiseState.enabled. confirm they're not meaningfully different
@@ -236,15 +266,18 @@ class CarController:
     # we can spam can to cancel the system even if we are using lat only control
     if (self.frame % 3 == 0 and self.CP.openpilotLongitudinalControl) or pcm_cancel_cmd:
       lead = hud_control.leadVisible or CS.out.vEgo < 12.  # at low speed we always assume the lead is present so ACC can be engaged
+      # when stopping, send -2.5 raw acceleration immediately to prevent vehicle from creeping, else send actuators.accel
+      accel_raw = -2.5 if stopping else actuators.accel
 
       # Lexus IS uses a different cancellation message
       if pcm_cancel_cmd and self.CP.carFingerprint in UNSUPPORTED_DSU_CAR:
         can_sends.append(toyotacan.create_acc_cancel_command(self.packer))
       elif self.CP.openpilotLongitudinalControl:
-        can_sends.append(toyotacan.create_accel_command(self.packer, pcm_accel_cmd, pcm_cancel_cmd, self.standstill_req, lead, CS.acc_type, fcw_alert))
+        can_sends.append(toyotacan.create_accel_command(self.packer, pcm_accel_cmd, accel_raw, pcm_cancel_cmd,
+                                                        self.standstill_req, lead, CS.acc_type, fcw_alert))
         self.accel = pcm_accel_cmd
       else:
-        can_sends.append(toyotacan.create_accel_command(self.packer, 0, pcm_cancel_cmd, False, lead, CS.acc_type, False))
+        can_sends.append(toyotacan.create_accel_command(self.packer, 0, 0, pcm_cancel_cmd, False, lead, CS.acc_type, False))
 
     if self.frame % 2 == 0 and self.CP.enableGasInterceptor and self.CP.openpilotLongitudinalControl:
       # send exactly zero if gas cmd is zero. Interceptor will send the max between read value and gas cmd.
@@ -259,7 +292,7 @@ class CarController:
       # - there is something to stop displaying
       send_ui = False
       if ((fcw_alert or steer_alert) and not self.alert_active) or \
-         (not (fcw_alert or steer_alert) and self.alert_active):
+        (not (fcw_alert or steer_alert) and self.alert_active):
         send_ui = True
         self.alert_active = not self.alert_active
       elif pcm_cancel_cmd:

selfdrive/car/toyota/carstate.py

@@ -9,7 +9,7 @@ from opendbc.can.can_define import CANDefine
 from opendbc.can.parser import CANParser
 from openpilot.selfdrive.car.interfaces import CarStateBase
 from openpilot.selfdrive.car.toyota.values import ToyotaFlags, CAR, DBC, STEER_THRESHOLD, NO_STOP_TIMER_CAR, \
-                                                  TSS2_CAR, RADAR_ACC_CAR, EPS_SCALE, UNSUPPORTED_DSU_CAR
+  TSS2_CAR, RADAR_ACC_CAR, EPS_SCALE, UNSUPPORTED_DSU_CAR
 
 from openpilot.common.params import Params
 
@@ -97,7 +97,7 @@ class CarState(CarStateBase):
     ret.vEgo, ret.aEgo = self.update_speed_kf(ret.vEgoRaw)
     ret.vEgoCluster = ret.vEgo * 1.015  # minimum of all the cars
 
-    ret.standstill = ret.vEgoRaw == 0
+    ret.standstill = abs(ret.vEgoRaw) < 1e-3
 
     ret.steeringAngleDeg = cp.vl["STEER_ANGLE_SENSOR"]["STEER_ANGLE"] + cp.vl["STEER_ANGLE_SENSOR"]["STEER_FRACTION"]
     ret.steeringRateDeg = cp.vl["STEER_ANGLE_SENSOR"]["STEER_RATE"]
@@ -149,6 +149,9 @@ class CarState(CarStateBase):
     ret.leftBlinker = cp.vl["BLINKERS_STATE"]["TURN_SIGNALS"] == 1
     ret.rightBlinker = cp.vl["BLINKERS_STATE"]["TURN_SIGNALS"] == 2
 
+    if self.CP.carFingerprint != CAR.MIRAI:
+      ret.engineRpm = cp.vl["ENGINE_RPM"]["RPM"]
+
     ret.steeringTorque = cp.vl["STEER_TORQUE_SENSOR"]["STEER_TORQUE_DRIVER"]
     ret.steeringTorqueEps = cp.vl["STEER_TORQUE_SENSOR"]["STEER_TORQUE_EPS"] * self.eps_torque_scale
     # we could use the override bit from dbc, but it's triggered at too high torque values
@@ -191,7 +194,7 @@ class CarState(CarStateBase):
     # TODO: it is possible to avoid the lockout and gain stop and go if you
     # send your own ACC_CONTROL msg on startup with ACC_TYPE set to 1
     if (self.CP.carFingerprint not in TSS2_CAR and self.CP.carFingerprint not in UNSUPPORTED_DSU_CAR) or \
-       (self.CP.carFingerprint in TSS2_CAR and self.acc_type == 1):
+      (self.CP.carFingerprint in TSS2_CAR and self.acc_type == 1):
       self.low_speed_lockout = cp.vl["PCM_CRUISE_2"]["LOW_SPEED_LOCKOUT"] == 2
 
     self.pcm_acc_status = cp.vl["PCM_CRUISE"]["CRUISE_STATE"]
@@ -200,6 +203,7 @@ class CarState(CarStateBase):
       ret.cruiseState.standstill = self.pcm_acc_status == 7
     ret.cruiseState.enabled = bool(cp.vl["PCM_CRUISE"]["CRUISE_ACTIVE"])
     ret.cruiseState.nonAdaptive = cp.vl["PCM_CRUISE"]["CRUISE_STATE"] in (1, 2, 3, 4, 5, 6)
+    self.pcm_neutral_force = cp.vl["PCM_CRUISE"]["NEUTRAL_FORCE"]
 
     ret.genericToggle = bool(cp.vl["LIGHT_STALK"]["AUTO_HIGH_BEAM"])
     ret.espDisabled = cp.vl["ESP_CONTROL"]["TC_DISABLED"] != 0
@@ -278,6 +282,9 @@ class CarState(CarStateBase):
       ("STEER_TORQUE_SENSOR", 50),
     ]
 
+    if CP.carFingerprint != CAR.MIRAI:
+      messages.append(("ENGINE_RPM", 42))
+
     if CP.carFingerprint in UNSUPPORTED_DSU_CAR:
       messages.append(("DSU_CRUISE", 5))
       messages.append(("PCM_CRUISE_ALT", 1))

selfdrive/car/toyota/fingerprints.py

@@ -38,6 +38,7 @@ FW_VERSIONS = {
       b'F152607180\x00\x00\x00\x00\x00\x00',
       b'F152641040\x00\x00\x00\x00\x00\x00',
       b'F152641050\x00\x00\x00\x00\x00\x00',
+      b'F152641060\x00\x00\x00\x00\x00\x00',
       b'F152641061\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.dsu, 0x791, None): [
@@ -55,10 +56,12 @@ FW_VERSIONS = {
       b'\x01896630725100\x00\x00\x00\x00',
       b'\x01896630725200\x00\x00\x00\x00',
       b'\x01896630725300\x00\x00\x00\x00',
+      b'\x01896630725400\x00\x00\x00\x00',
       b'\x01896630735100\x00\x00\x00\x00',
       b'\x01896630738000\x00\x00\x00\x00',
       b'\x02896630724000\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
       b'\x02896630728000\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
+      b'\x02896630734000\x00\x00\x00\x00897CF3305001\x00\x00\x00\x00',
       b'\x02896630737000\x00\x00\x00\x00897CF3305001\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
@@ -570,6 +573,7 @@ FW_VERSIONS = {
       b'\x018821F6201400\x00\x00\x00\x00',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
+      b'\x028646F12010C0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F12010D0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F1201100\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F1201200\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
@@ -840,6 +844,7 @@ FW_VERSIONS = {
       b'8965B47023\x00\x00\x00\x00\x00\x00',
       b'8965B47050\x00\x00\x00\x00\x00\x00',
       b'8965B47060\x00\x00\x00\x00\x00\x00',
+      b'8965B47070\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
       b'F152647290\x00\x00\x00\x00\x00\x00',
@@ -1021,6 +1026,7 @@ FW_VERSIONS = {
       b'\x02896634A13000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02896634A13001\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
       b'\x02896634A13101\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
+      b'\x02896634A13201\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
       b'\x02896634A14001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
       b'\x02896634A14001\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
       b'\x02896634A14101\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
@@ -1361,16 +1367,19 @@ FW_VERSIONS = {
       b'\x018966378G3000\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
+      b'\x0237881000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x0237887000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02378A0000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02378F4000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
       b'\x01F152678221\x00\x00\x00\x00\x00\x00',
+      b'F152678200\x00\x00\x00\x00\x00\x00',
       b'F152678210\x00\x00\x00\x00\x00\x00',
       b'F152678211\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
+      b'8965B78110\x00\x00\x00\x00\x00\x00',
       b'8965B78120\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
@@ -1417,6 +1426,7 @@ FW_VERSIONS = {
   },
   CAR.LEXUS_RX: {
     (Ecu.engine, 0x700, None): [
+      b'\x01896630E36100\x00\x00\x00\x00',
       b'\x01896630E36200\x00\x00\x00\x00',
       b'\x01896630E36300\x00\x00\x00\x00',
       b'\x01896630E37100\x00\x00\x00\x00',

selfdrive/car/toyota/interface.py

@@ -3,13 +3,11 @@ from openpilot.common.conversions import Conversions as CV
 from panda import Panda
 from panda.python import uds
 from openpilot.selfdrive.car.toyota.values import Ecu, CAR, DBC, ToyotaFlags, CarControllerParams, TSS2_CAR, RADAR_ACC_CAR, NO_DSU_CAR, \
-                                        MIN_ACC_SPEED, EPS_SCALE, UNSUPPORTED_DSU_CAR, NO_STOP_TIMER_CAR, ANGLE_CONTROL_CAR
+  MIN_ACC_SPEED, EPS_SCALE, UNSUPPORTED_DSU_CAR, NO_STOP_TIMER_CAR, ANGLE_CONTROL_CAR
 from openpilot.selfdrive.car import get_safety_config
 from openpilot.selfdrive.car.disable_ecu import disable_ecu
 from openpilot.selfdrive.car.interfaces import CarInterfaceBase
 
-from openpilot.common.params import Params
-
 EventName = car.CarEvent.EventName
 SteerControlType = car.CarParams.SteerControlType
 
@@ -46,8 +44,6 @@ class CarInterface(CarInterfaceBase):
 
     stop_and_go = candidate in TSS2_CAR
 
-    ret.dashcamOnly = False if ret.dashcamOnly and Params().get_bool("dp_car_dashcam_mode_removal") else ret.dashcamOnly
-
     if candidate == CAR.PRIUS:
       stop_and_go = True
       ret.wheelbase = 2.70
@@ -222,21 +218,17 @@ class CarInterface(CarInterfaceBase):
     # In TSS2 cars, the camera does long control
     found_ecus = [fw.ecu for fw in car_fw]
     ret.enableDsu = len(found_ecus) > 0 and Ecu.dsu not in found_ecus and candidate not in (NO_DSU_CAR | UNSUPPORTED_DSU_CAR) \
-                                        and not (ret.flags & ToyotaFlags.SMART_DSU)
-    ret.enableGasInterceptor = 0x201 in fingerprint[0]
-
-    if ret.enableGasInterceptor:
-      ret.safetyConfigs[0].safetyParam |= Panda.FLAG_TOYOTA_GAS_INTERCEPTOR
+                    and not (ret.flags & ToyotaFlags.SMART_DSU)
 
     # if the smartDSU is detected, openpilot can send ACC_CONTROL and the smartDSU will block it from the DSU or radar.
     # since we don't yet parse radar on TSS2/TSS-P radar-based ACC cars, gate longitudinal behind experimental toggle
     use_sdsu = bool(ret.flags & ToyotaFlags.SMART_DSU)
     if candidate in (RADAR_ACC_CAR | NO_DSU_CAR):
-      ret.experimentalLongitudinalAvailable = use_sdsu
+      ret.experimentalLongitudinalAvailable = use_sdsu or candidate in RADAR_ACC_CAR
 
       if not use_sdsu:
         # Disabling radar is only supported on TSS2 radar-ACC cars
-        if experimental_long and candidate in RADAR_ACC_CAR and Params().get_bool("dp_toyota_tss2_radar_disabled"):  # TODO: disabling radar isn't supported yet
+        if experimental_long and candidate in RADAR_ACC_CAR:
           ret.flags |= ToyotaFlags.DISABLE_RADAR.value
       else:
         use_sdsu = use_sdsu and experimental_long
@@ -251,31 +243,33 @@ class CarInterface(CarInterfaceBase):
     #  - TSS-P DSU-less cars w/ CAN filter installed (no radar parser yet)
     ret.openpilotLongitudinalControl = use_sdsu or ret.enableDsu or candidate in (TSS2_CAR - RADAR_ACC_CAR) or bool(ret.flags & ToyotaFlags.DISABLE_RADAR.value)
     ret.autoResumeSng = ret.openpilotLongitudinalControl and candidate in NO_STOP_TIMER_CAR
+    ret.enableGasInterceptor = 0x201 in fingerprint[0] and ret.openpilotLongitudinalControl
 
     if not ret.openpilotLongitudinalControl:
       ret.safetyConfigs[0].safetyParam |= Panda.FLAG_TOYOTA_STOCK_LONGITUDINAL
 
+    if ret.enableGasInterceptor:
+      ret.safetyConfigs[0].safetyParam |= Panda.FLAG_TOYOTA_GAS_INTERCEPTOR
+
     # min speed to enable ACC. if car can do stop and go, then set enabling speed
     # to a negative value, so it won't matter.
     ret.minEnableSpeed = -1. if (stop_and_go or ret.enableGasInterceptor) else MIN_ACC_SPEED
 
+    # on stock Toyota this is -2.5
+    ret.stopAccel = -2.5
+
     tune = ret.longitudinalTuning
-    tune.deadzoneBP = [0., 9.]
-    tune.deadzoneV = [.0, .15]
-    if candidate in TSS2_CAR or ret.enableGasInterceptor:
-      tune.kpBP = [0., 5., 20.]
-      tune.kpV = [1.3, 1.0, 0.7]
-      tune.kiBP = [0., 5., 12., 20., 27.]
-      tune.kiV = [.35, .23, .20, .17, .1]
-      if candidate in TSS2_CAR:
-        ret.vEgoStopping = 0.25
-        ret.vEgoStarting = 0.25
-        ret.stoppingDecelRate = 0.3  # reach stopping target smoothly
-    else:
-      tune.kpBP = [0., 5., 35.]
-      tune.kiBP = [0., 35.]
-      tune.kpV = [3.6, 2.4, 1.5]
-      tune.kiV = [0.54, 0.36]
+    tune.deadzoneBP = [0., 16., 20., 30.]
+    tune.deadzoneV = [.04, .05, .08, .15]
+    ret.stoppingDecelRate = 0.17  # This is okay for TSS-P
+    if candidate in TSS2_CAR:
+      ret.vEgoStopping = 0.25
+      ret.vEgoStarting = 0.25
+      ret.stoppingDecelRate = 0.009  # reach stopping target smoothly
+    tune.kpBP = [0., 5.]
+    tune.kpV = [0.8, 1.]
+    tune.kiBP = [0., 5.]
+    tune.kiV = [0.3, 1.]
 
     return ret
 

selfdrive/car/toyota/toyotacan.py

@@ -33,10 +33,10 @@ def create_lta_steer_command(packer, steer_control_type, steer_angle, steer_req,
   return packer.make_can_msg("STEERING_LTA", 0, values)
 
 
-def create_accel_command(packer, accel, pcm_cancel, standstill_req, lead, acc_type, fcw_alert):
+def create_accel_command(packer, accel, accel_raw, pcm_cancel, standstill_req, lead, acc_type, fcw_alert):
   # TODO: find the exact canceling bit that does not create a chime
   values = {
-    "ACCEL_CMD": accel,
+    "ACCEL_CMD": accel,  # compensated accel command
     "ACC_TYPE": acc_type,
     "DISTANCE": 0,
     "MINI_CAR": lead,
@@ -45,6 +45,7 @@ def create_accel_command(packer, accel, pcm_cancel, standstill_req, lead, acc_ty
     "CANCEL_REQ": pcm_cancel,
     "ALLOW_LONG_PRESS": 1,
     "ACC_CUT_IN": fcw_alert,  # only shown when ACC enabled
+    "ACCEL_CMD_ALT":  accel_raw,  # raw accel command, pcm uses this to calculate a compensatory force
   }
   return packer.make_can_msg("ACC_CONTROL", 0, values)
 

selfdrive/controls/controlsd.py

@@ -2,6 +2,7 @@
 import os
 import math
 import time
+import threading
 from typing import SupportsFloat
 
 from cereal import car, log
@@ -65,6 +66,9 @@ DP_VAG_TIMEBOMB_BYPASS_WARNING = 34000
 DP_VAG_TIMEBOMB_BYPASS_START = 345000
 DP_VAG_TIMEBOMB_BYPASS_END = 348000
 
+DP_LONG_MISSING_LEAD_COUNT = 2. / DT_CTRL
+DP_LONG_MISSING_LEAD_SPEED = 19.44  # 70 kph
+
 class Controls:
   def __init__(self, sm=None, pm=None, can_sock=None, CI=None):
     config_realtime_process(4 if TICI else 3, Priority.CTRL_HIGH)
@@ -72,7 +76,7 @@ class Controls:
     self.dp_gps_ok_once = False
 
     # Ensure the current branch is cached, otherwise the first iteration of controlsd lags
-    self.branch = get_short_branch("")
+    self.branch = get_short_branch()
 
     # Setup sockets
     self.pm = pm
@@ -103,6 +107,9 @@ class Controls:
     self._dp_vag_timebomb_bypass = self.params.get_bool("dp_vag_timebomb_bypass")
     self._dp_lat_lane_change_assist_disabled = int(self.params.get("dp_lat_lane_change_assist_speed", encoding="utf-8")) == 0
     self._dp_lat_lane_change_assist_disabled_active = False
+    self._dp_long_missing_lead_warning = self.params.get_bool("dp_long_missing_lead_warning")
+    self._dp_long_missing_lead_count = 0
+    self._dp_long_missing_lead_prev = False
     self.sm = sm
     if self.sm is None:
       ignore = ['testJoystick']
@@ -271,6 +278,26 @@ class Controls:
     if self.read_only:
       return
 
+    # lead missing alert
+    # when driving on highway and the lead car suddenly gone missing, hazard ahead?
+    if self._dp_long_missing_lead_warning and CS.vEgo >= DP_LONG_MISSING_LEAD_SPEED:
+      _dp_long_missing_lead = not self.sm['longitudinalPlan'].hasLead
+
+      # lead vehicle missing started
+      if not self._dp_long_missing_lead_prev and _dp_long_missing_lead:
+        self._dp_long_missing_lead_count = DP_LONG_MISSING_LEAD_COUNT
+
+      # only send event when counter reach 0
+      if self._dp_long_missing_lead_count > 0:
+        if CS.steeringPressed or CS.brakePressed or not _dp_long_missing_lead:
+          self._dp_long_missing_lead_count = 0
+        else:
+          self._dp_long_missing_lead_count -= 1
+          if self._dp_long_missing_lead_count == 0:
+            self.events.add(EventName.promptDriverDistracted)
+
+      self._dp_long_missing_lead_prev = _dp_long_missing_lead
+
     # ALKA combination
     if self._dp_alka and CS.brakePressed:
       # rick - allow ALKA to be enabled/disabled when brake + main pressed twice in 0.5 secs
@@ -963,18 +990,10 @@ class Controls:
 
   def step(self):
     start_time = time.monotonic()
-    self.prof.checkpoint("Ratekeeper", ignore=True)
-
-    self.is_metric = self.params.get_bool("IsMetric")
-    self.experimental_mode = self.params.get_bool("ExperimentalMode") and self.CP.openpilotLongitudinalControl
-    # rick - we should disable experimental mode on radarless car w/ 0.8.13 model
-    if self.CP.radarUnavailable and self.dp_0813:
-      self.experimental_mode = False
 
     # Sample data from sockets and get a carState
     CS = self.data_sample()
     cloudlog.timestamp("Data sampled")
-    self.prof.checkpoint("Sample")
 
     self.update_events(CS)
     cloudlog.timestamp("Events updated")
@@ -982,24 +1001,37 @@ class Controls:
     if not self.read_only and self.initialized:
       # Update control state
       self.state_transition(CS)
-      self.prof.checkpoint("State transition")
 
     # Compute actuators (runs PID loops and lateral MPC)
     CC, lac_log = self.state_control(CS)
 
-    self.prof.checkpoint("State Control")
-
     # Publish data
     self.publish_logs(CS, start_time, CC, lac_log)
-    self.prof.checkpoint("Sent")
 
     self.CS_prev = CS
 
+  def params_thread(self, evt):
+    while not evt.is_set():
+      self.is_metric = self.params.get_bool("IsMetric")
+      self.experimental_mode = self.params.get_bool("ExperimentalMode") and self.CP.openpilotLongitudinalControl
+      # rick - we should disable experimental mode on radarless car w/ 0.8.13 model
+      if self.CP.radarUnavailable and self.dp_0813:
+        self.experimental_mode = False
+      if self.CP.notCar:
+        self.joystick_mode = self.params.get_bool("JoystickDebugMode")
+      time.sleep(0.1)
+
   def controlsd_thread(self):
-    while True:
-      self.step()
-      self.rk.monitor_time()
-      self.prof.display()
+    e = threading.Event()
+    t = threading.Thread(target=self.params_thread, args=(e, ))
+    try:
+      t.start()
+      while True:
+        self.step()
+        self.rk.monitor_time()
+    except SystemExit:
+      e.set()
+      t.join()
 
 
 def main(sm=None, pm=None, logcan=None):

selfdrive/controls/lib/drive_helpers.py

@@ -13,7 +13,7 @@ V_CRUISE_MIN = 8
 V_CRUISE_MAX = 145
 V_CRUISE_UNSET = 255
 V_CRUISE_INITIAL = 40
-V_CRUISE_INITIAL_EXPERIMENTAL_MODE = 40
+V_CRUISE_INITIAL_EXPERIMENTAL_MODE = 55
 IMPERIAL_INCREMENT = 1.6  # should be CV.MPH_TO_KPH, but this causes rounding errors
 
 MIN_SPEED = 1.0
@@ -22,7 +22,6 @@ CAR_ROTATION_RADIUS = 0.0
 
 # EU guidelines
 MAX_LATERAL_JERK = 5.0
-
 MAX_VEL_ERR = 5.0
 
 ButtonEvent = car.CarState.ButtonEvent
@@ -200,6 +199,16 @@ def get_lag_adjusted_curvature(CP, v_ego, psis, curvatures, curvature_rates):
   return safe_desired_curvature, safe_desired_curvature_rate
 
 
+# rick - we dont need this as it is for the latest model (0.9.5+) + controlsd
+def clip_curvature(v_ego, prev_curvature, new_curvature):
+  v_ego = max(MIN_SPEED, v_ego)
+  max_curvature_rate = MAX_LATERAL_JERK / (v_ego**2) # inexact calculation, check https://github.com/commaai/openpilot/pull/24755
+  safe_desired_curvature = clip(new_curvature,
+                                prev_curvature - max_curvature_rate * DT_CTRL,
+                                prev_curvature + max_curvature_rate * DT_CTRL)
+
+  return safe_desired_curvature
+
 def get_friction(lateral_accel_error: float, lateral_accel_deadzone: float, friction_threshold: float,
                  torque_params: car.CarParams.LateralTorqueTuning, friction_compensation: bool) -> float:
   friction_interp = interp(

selfdrive/controls/lib/dynamic_endtoend_controller.py

@@ -21,35 +21,37 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
-# Version = 2023-12-13
+# Version = 2024-02-28
 from common.numpy_fast import interp
 
 # d-e2e, from modeldata.h
 TRAJECTORY_SIZE = 33
 
-LEAD_WINDOW_SIZE = 5
-LEAD_PROB = 0.6
+LEAD_WINDOW_SIZE = 3
+LEAD_PROB = 0.5
 
 SLOW_DOWN_WINDOW_SIZE = 5
-SLOW_DOWN_PROB = 0.6
+SLOW_DOWN_PROB = 0.5
 SLOW_DOWN_BP = [0., 10., 20., 30., 40., 50., 55.]
 SLOW_DOWN_DIST = [10, 30., 50., 70., 80., 90., 120.]
 
-SLOWNESS_WINDOW_SIZE = 20
-SLOWNESS_PROB = 0.6
+SLOWNESS_WINDOW_SIZE = 10
+SLOWNESS_PROB = 0.5
 SLOWNESS_CRUISE_OFFSET = 1.05
 
-DANGEROUS_TTC_WINDOW_SIZE = 5
+DANGEROUS_TTC_WINDOW_SIZE = 3
 DANGEROUS_TTC = 2.0
 
-HIGHWAY_CRUISE_KPH = 75
+HIGHWAY_CRUISE_KPH = 70
 
 STOP_AND_GO_FRAME = 60
 
 SET_MODE_TIMEOUT = 10
 
-MPC_FCW_WINDOW_SIZE = 5
-MPC_FCW_PROB = 0.6
+MPC_FCW_WINDOW_SIZE = 10
+MPC_FCW_PROB = 0.5
+
+V_ACC_MIN = 9.72
 
 class SNG_State:
   off = 0
@@ -129,18 +131,18 @@ class DynamicEndtoEndController:
 
     # fcw detection
     self._mpc_fcw_gmac.add_data(self._mpc_fcw_crash_cnt > 0)
-    self._has_mpc_fcw = self._mpc_fcw_gmac.get_moving_average() >= MPC_FCW_PROB
+    self._has_mpc_fcw = self._mpc_fcw_gmac.get_moving_average() > MPC_FCW_PROB
 
     # nav enable detection
     self._has_nav_enabled = md.navEnabled
 
     # lead detection
     self._lead_gmac.add_data(lead_one.status)
-    self._has_lead_filtered = self._lead_gmac.get_moving_average() >= LEAD_PROB
+    self._has_lead_filtered = self._lead_gmac.get_moving_average() > LEAD_PROB
 
     # slow down detection
     self._slow_down_gmac.add_data(len(md.orientation.x) == len(md.position.x) == TRAJECTORY_SIZE and md.position.x[TRAJECTORY_SIZE - 1] < interp(self._v_ego_kph, SLOW_DOWN_BP, SLOW_DOWN_DIST))
-    self._has_slow_down = self._slow_down_gmac.get_moving_average() >= SLOW_DOWN_PROB
+    self._has_slow_down = self._slow_down_gmac.get_moving_average() > SLOW_DOWN_PROB
 
     # blinker detection
     self._has_blinkers = car_state.leftBlinker or car_state.rightBlinker
@@ -162,7 +164,7 @@ class DynamicEndtoEndController:
     # slowness detection
     if not self._has_standstill:
       self._slowness_gmac.add_data(self._v_ego_kph <= (self._v_cruise_kph*SLOWNESS_CRUISE_OFFSET))
-      self._has_slowness = self._slowness_gmac.get_moving_average() >= SLOWNESS_PROB
+      self._has_slowness = self._slowness_gmac.get_moving_average() > SLOWNESS_PROB
 
     # dangerous TTC detection
     if not self._has_lead_filtered and self._has_lead_filtered_prev:
@@ -180,16 +182,16 @@ class DynamicEndtoEndController:
     self._has_lead_filtered_prev = self._has_lead_filtered
     self._frame += 1
 
-  def _blended_priority_mode(self):
+  def _radarless_mode(self):
     # when mpc fcw crash prob is high
     # use blended to slow down quickly
     if self._has_mpc_fcw:
       self._set_mode('blended')
       return
 
-    # when blinker is on and speed is driving below highway cruise speed: blended
+    # when blinker is on and speed is driving below V_ACC_MIN: blended
     # we dont want it to switch mode at higher speed, blended may trigger hard brake
-    if self._has_blinkers and self._v_ego_kph < HIGHWAY_CRUISE_KPH:
+    if self._has_blinkers and self._v_ego_kph < V_ACC_MIN:
       self._set_mode('blended')
       return
 
@@ -223,9 +225,9 @@ class DynamicEndtoEndController:
       self._set_mode('acc')
       return
 
-    self._set_mode('blended')
+    self._set_mode('acc')
 
-  def _acc_priority_mode(self):
+  def _radar_mode(self):
     # when mpc fcw crash prob is high
     # use blended to slow down quickly
     if self._has_mpc_fcw:
@@ -266,9 +268,9 @@ class DynamicEndtoEndController:
     if self._is_enabled:
       self._update(car_state, lead_one, md, controls_state)
       if radar_unavailable:
-        self._blended_priority_mode()
+        self._radarless_mode()
       else:
-        self._acc_priority_mode()
+        self._radar_mode()
 
     self._mode_prev = self._mode
     return self._mode

selfdrive/controls/lib/legacy_longitudinal_planner.py

@@ -76,7 +76,10 @@ class LongitudinalPlanner:
     self.dp_long_use_krkeegen_tune_active = False
 
   def read_param(self):
-    self.personality = int(self.params.get('LongitudinalPersonality'))
+    try:
+      self.personality = int(self.params.get('LongitudinalPersonality'))
+    except (ValueError, TypeError):
+      self.personality = log.LongitudinalPersonality.standard
     self.dp_long_use_df_tune = self.params.get_bool('dp_long_use_df_tune')
     self.dp_long_use_krkeegen_tune = self.params.get_bool('dp_long_use_krkeegen_tune')
 

selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py

@@ -79,14 +79,14 @@ def get_T_FOLLOW(personality=log.LongitudinalPersonality.standard):
 
 def get_dynamic_follow(v_ego, personality=log.LongitudinalPersonality.standard):
   if personality==log.LongitudinalPersonality.relaxed:
-    x_vel =  [0,    3.05,   3.61,   4.16,   7.14,   11.11]
-    y_dist = [1.75, 1.75, 1.77, 1.75, 1.8,  1.8]
+    x_vel =  [0.0,  3.0,  8.33,  13.90,  20,    25,    40]
+    y_dist = [1.2,  1.25, 1.40,  1.40,   1.50,  1.85,  2.0]
   elif personality==log.LongitudinalPersonality.standard:
-    x_vel =  [0,    3.05,   3.61,   4.16,   7.14,   11.11]
-    y_dist = [1.5,  1.5,  1.51,  1.5,  1.5,  1.45]
+    x_vel =  [0.0,  3.0,  8.33,  13.90,  20,    25,    40]
+    y_dist = [1.00,  1.00, 1.20,  1.20,   1.25,  1.45,  1.5]
   elif personality==log.LongitudinalPersonality.aggressive:
-    x_vel =  [0,    3.05,   3.61,   4.16,   7.14,   11.11]
-    y_dist = [1.12, 1.12, 1.13, 1.12, 1.22, 1.22]
+    x_vel =  [0.0,  4.00, 8.33,  13.89,  20,    25,    40]
+    y_dist = [0.8,  0.80, 0.90,  0.90,   0.9,  1.105, 1.12]
   else:
     raise NotImplementedError("Dynamic Follow personality not supported")
   return np.interp(v_ego, x_vel, y_dist)

selfdrive/controls/lib/longitudinal_planner.py

@@ -30,6 +30,7 @@ A_CRUISE_MAX_BP = [0., 10.0, 25., 40.]
 _A_TOTAL_MAX_V = [1.7, 3.2]
 _A_TOTAL_MAX_BP = [20., 40.]
 
+V_ACC_MIN = 9.72  # 35 kph
 
 def get_max_accel(v_ego):
   return interp(v_ego, A_CRUISE_MAX_BP, A_CRUISE_MAX_VALS)
@@ -174,7 +175,7 @@ class LongitudinalPlanner:
     self.mpc.set_accel_limits(accel_limits_turns[0], accel_limits_turns[1])
     self.mpc.set_cur_state(self.v_desired_filter.x, self.a_desired)
     x, v, a, j = self.parse_model(sm['modelV2'], self.v_model_error, v_ego, taco=True)
-    self.dp_long_use_krkeegen_tune_active = self.dp_long_use_krkeegen_tune and v_ego <= 7.5
+    self.dp_long_use_krkeegen_tune_active = self.dp_long_use_krkeegen_tune and v_ego <= V_ACC_MIN
     self.dp_long_use_df_tune_active = self.dp_long_use_df_tune and sm['radarState'].leadOne.status
     self.mpc.update(sm['radarState'], v_cruise_sol, x, v, a, j, personality=self.personality, use_df_tune=self.dp_long_use_df_tune_active, use_krkeegen_tune=self.dp_long_use_krkeegen_tune_active)
 

selfdrive/controls/radard.py

@@ -170,7 +170,7 @@ def get_RadarState_from_vision(lead_msg: capnp._DynamicStructReader, v_ego: floa
 def get_lead(v_ego: float, ready: bool, tracks: Dict[int, Track], lead_msg: capnp._DynamicStructReader,
              model_v_ego: float, low_speed_override: bool = True) -> Dict[str, Any]:
   # Determine leads, this is where the essential logic happens
-  if len(tracks) > 0 and ready and lead_msg.prob > .5:
+  if len(tracks) > 0 and ready and lead_msg.prob > .4:
     track = match_vision_to_track(v_ego, lead_msg, tracks)
   else:
     track = None
@@ -178,7 +178,7 @@ def get_lead(v_ego: float, ready: bool, tracks: Dict[int, Track], lead_msg: capn
   lead_dict = {'status': False}
   if track is not None:
     lead_dict = track.get_RadarState(lead_msg.prob)
-  elif (track is None) and ready and (lead_msg.prob > .5):
+  elif (track is None) and ready and (lead_msg.prob > .4):
     lead_dict = get_RadarState_from_vision(lead_msg, v_ego, model_v_ego)
 
   if low_speed_override:

selfdrive/locationd/models/generated/car.h

@@ -9,27 +9,27 @@ void car_update_27(double *in_x, double *in_P, double *in_z, double *in_R, doubl
 void car_update_29(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
 void car_update_28(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
 void car_update_31(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
-void car_err_fun(double *nom_x, double *delta_x, double *out_2472665262004959677);
-void car_inv_err_fun(double *nom_x, double *true_x, double *out_8982529719675541846);
-void car_H_mod_fun(double *state, double *out_1829537519410656848);
-void car_f_fun(double *state, double dt, double *out_1757186677020913895);
-void car_F_fun(double *state, double dt, double *out_5278060812158919790);
-void car_h_25(double *state, double *unused, double *out_5407080294330651747);
-void car_H_25(double *state, double *unused, double *out_3554883914703153887);
-void car_h_24(double *state, double *unused, double *out_1039462419148683192);
-void car_H_24(double *state, double *unused, double *out_3084426432659815163);
-void car_h_30(double *state, double *unused, double *out_1363754932019699189);
-void car_H_30(double *state, double *unused, double *out_3425544967559913817);
-void car_h_26(double *state, double *unused, double *out_420245079118036754);
-void car_H_26(double *state, double *unused, double *out_186619404170902337);
-void car_h_27(double *state, double *unused, double *out_8037990511718054224);
-void car_H_27(double *state, double *unused, double *out_1250781655759488906);
-void car_h_29(double *state, double *unused, double *out_4400540983464031874);
-void car_H_29(double *state, double *unused, double *out_462581071110062127);
-void car_h_28(double *state, double *unused, double *out_758318952547942916);
-void car_H_28(double *state, double *unused, double *out_1501049200455264124);
-void car_h_31(double *state, double *unused, double *out_6857711686959534914);
-void car_H_31(double *state, double *unused, double *out_3585529876580114315);
+void car_err_fun(double *nom_x, double *delta_x, double *out_2373438450883635122);
+void car_inv_err_fun(double *nom_x, double *true_x, double *out_4678845552861946597);
+void car_H_mod_fun(double *state, double *out_7488586493190929620);
+void car_f_fun(double *state, double dt, double *out_6363234589794725649);
+void car_F_fun(double *state, double dt, double *out_3089997159050685325);
+void car_h_25(double *state, double *unused, double *out_582173192503055026);
+void car_H_25(double *state, double *unused, double *out_4242371084562866524);
+void car_h_24(double *state, double *unused, double *out_9199538503225060709);
+void car_H_24(double *state, double *unused, double *out_7800296683365189660);
+void car_h_30(double *state, double *unused, double *out_2365552797852429935);
+void car_H_30(double *state, double *unused, double *out_7287682647655068337);
+void car_h_26(double *state, double *unused, double *out_5178104901122283318);
+void car_H_26(double *state, double *unused, double *out_4899225148673178428);
+void car_h_27(double *state, double *unused, double *out_7968644223126639134);
+void car_H_27(double *state, double *unused, double *out_8984298114254058368);
+void car_h_29(double *state, double *unused, double *out_1197808996776288198);
+void car_H_29(double *state, double *unused, double *out_6777451303340676153);
+void car_h_28(double *state, double *unused, double *out_6278196388390958174);
+void car_H_28(double *state, double *unused, double *out_6586893753299344889);
+void car_h_31(double *state, double *unused, double *out_19244965110308528);
+void car_H_31(double *state, double *unused, double *out_4273017046439826952);
 void car_predict(double *in_x, double *in_P, double *in_Q, double dt);
 void car_set_mass(double x);
 void car_set_rotational_inertia(double x);

selfdrive/locationd/models/generated/gnss.h

@@ -5,18 +5,18 @@ void gnss_update_6(double *in_x, double *in_P, double *in_z, double *in_R, doubl
 void gnss_update_20(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
 void gnss_update_7(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
 void gnss_update_21(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
-void gnss_err_fun(double *nom_x, double *delta_x, double *out_5782114737457059360);
-void gnss_inv_err_fun(double *nom_x, double *true_x, double *out_8977268484679926314);
-void gnss_H_mod_fun(double *state, double *out_338300992960698012);
-void gnss_f_fun(double *state, double dt, double *out_1371145784839709762);
-void gnss_F_fun(double *state, double dt, double *out_8435684940982077033);
-void gnss_h_6(double *state, double *sat_pos, double *out_5387927519311920081);
-void gnss_H_6(double *state, double *sat_pos, double *out_1097586483885744352);
-void gnss_h_20(double *state, double *sat_pos, double *out_281837704672554358);
-void gnss_H_20(double *state, double *sat_pos, double *out_1069919034497094674);
-void gnss_h_7(double *state, double *sat_pos_vel, double *out_4066013704851088785);
-void gnss_H_7(double *state, double *sat_pos_vel, double *out_204057149496812951);
-void gnss_h_21(double *state, double *sat_pos_vel, double *out_4066013704851088785);
-void gnss_H_21(double *state, double *sat_pos_vel, double *out_204057149496812951);
+void gnss_err_fun(double *nom_x, double *delta_x, double *out_6647689131199657613);
+void gnss_inv_err_fun(double *nom_x, double *true_x, double *out_8021804499807982886);
+void gnss_H_mod_fun(double *state, double *out_7990695309930200593);
+void gnss_f_fun(double *state, double dt, double *out_8004965045134918007);
+void gnss_F_fun(double *state, double dt, double *out_2717500358276134323);
+void gnss_h_6(double *state, double *sat_pos, double *out_182141457505723631);
+void gnss_H_6(double *state, double *sat_pos, double *out_2143497049330031194);
+void gnss_h_20(double *state, double *sat_pos, double *out_2957539294942517241);
+void gnss_H_20(double *state, double *sat_pos, double *out_5178148131249966948);
+void gnss_h_7(double *state, double *sat_pos_vel, double *out_2639259269154791275);
+void gnss_H_7(double *state, double *sat_pos_vel, double *out_6741483478705714184);
+void gnss_h_21(double *state, double *sat_pos_vel, double *out_2639259269154791275);
+void gnss_H_21(double *state, double *sat_pos_vel, double *out_6741483478705714184);
 void gnss_predict(double *in_x, double *in_P, double *in_Q, double dt);
 }

selfdrive/locationd/models/generated/live.h

@@ -10,29 +10,29 @@ void live_update_32(double *in_x, double *in_P, double *in_z, double *in_R, doub
 void live_update_13(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
 void live_update_14(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
 void live_update_33(double *in_x, double *in_P, double *in_z, double *in_R, double *in_ea);
-void live_H(double *in_vec, double *out_768290455835651354);
-void live_err_fun(double *nom_x, double *delta_x, double *out_5955955075753187712);
-void live_inv_err_fun(double *nom_x, double *true_x, double *out_5388686253131612340);
-void live_H_mod_fun(double *state, double *out_2552112100007039261);
-void live_f_fun(double *state, double dt, double *out_1532088538845507207);
-void live_F_fun(double *state, double dt, double *out_8958354368422034212);
-void live_h_4(double *state, double *unused, double *out_1213802693008545631);
-void live_H_4(double *state, double *unused, double *out_4709009694507262627);
-void live_h_9(double *state, double *unused, double *out_246909689773124009);
-void live_H_9(double *state, double *unused, double *out_551841958152485144);
-void live_h_10(double *state, double *unused, double *out_7769468188847797302);
-void live_H_10(double *state, double *unused, double *out_2350300367353127752);
-void live_h_12(double *state, double *unused, double *out_3244101654778954460);
-void live_H_12(double *state, double *unused, double *out_5330108719554856294);
-void live_h_35(double *state, double *unused, double *out_3662689924684647337);
-void live_H_35(double *state, double *unused, double *out_8075671751879870003);
-void live_h_32(double *state, double *unused, double *out_2198963332362222475);
-void live_H_32(double *state, double *unused, double *out_8155445796665977458);
-void live_h_13(double *state, double *unused, double *out_3486139371308475582);
-void live_H_13(double *state, double *unused, double *out_1290085859790998985);
-void live_h_14(double *state, double *unused, double *out_246909689773124009);
-void live_H_14(double *state, double *unused, double *out_551841958152485144);
-void live_h_33(double *state, double *unused, double *out_2379161178703843220);
-void live_H_33(double *state, double *unused, double *out_7220515317190824009);
+void live_H(double *in_vec, double *out_2514925971324519153);
+void live_err_fun(double *nom_x, double *delta_x, double *out_8069791558270424482);
+void live_inv_err_fun(double *nom_x, double *true_x, double *out_429301602353769159);
+void live_H_mod_fun(double *state, double *out_1386605300632116763);
+void live_f_fun(double *state, double dt, double *out_858533111488398624);
+void live_F_fun(double *state, double dt, double *out_1333631458729715779);
+void live_h_4(double *state, double *unused, double *out_4226654547665910661);
+void live_H_4(double *state, double *unused, double *out_803496816155160547);
+void live_h_9(double *state, double *unused, double *out_1637398521647854197);
+void live_H_9(double *state, double *unused, double *out_1044686462784751192);
+void live_h_10(double *state, double *unused, double *out_2911177701660318769);
+void live_H_10(double *state, double *unused, double *out_8738977984252755693);
+void live_h_12(double *state, double *unused, double *out_8509321518908937806);
+void live_H_12(double *state, double *unused, double *out_1424595841202754214);
+void live_h_35(double *state, double *unused, double *out_2717107203032309566);
+void live_H_35(double *state, double *unused, double *out_4170158873527767923);
+void live_h_32(double *state, double *unused, double *out_5509610638763499116);
+void live_H_32(double *state, double *unused, double *out_3857537178640322895);
+void live_h_13(double *state, double *unused, double *out_5565586117472497019);
+void live_H_13(double *state, double *unused, double *out_6962032018273124346);
+void live_h_14(double *state, double *unused, double *out_1637398521647854197);
+void live_H_14(double *state, double *unused, double *out_1044686462784751192);
+void live_h_33(double *state, double *unused, double *out_6039189548824972318);
+void live_H_33(double *state, double *unused, double *out_7320715878166625527);
 void live_predict(double *in_x, double *in_P, double *in_Q, double dt);
 }

selfdrive/manager/build.py

@@ -9,7 +9,7 @@ from openpilot.common.basedir import BASEDIR
 from openpilot.common.spinner import Spinner
 from openpilot.common.text_window import TextWindow
 from openpilot.system.hardware import AGNOS
-from openpilot.system.swaglog import cloudlog, add_file_handler
+from openpilot.common.swaglog import cloudlog, add_file_handler
 from openpilot.system.version import is_dirty
 
 MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
@@ -19,19 +19,21 @@ TOTAL_SCONS_NODES = 2560
 MAX_BUILD_PROGRESS = 100
 PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
 
-def build(spinner: Spinner, dirty: bool = False) -> None:
+def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
   env = os.environ.copy()
   env['SCONS_PROGRESS'] = "1"
   nproc = os.cpu_count()
   if nproc is None:
-    nproc = 2
+    nproc =3
+
+  extra_args = ["--minimal"] if minimal else []
 
   # building with all cores can result in using too
   # much memory, so retry with less parallelism
   compile_output: List[bytes] = []
   for n in (nproc, nproc/2, 1):
     compile_output.clear()
-    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
     assert scons.stderr is not None
 
     # Read progress from stderr and update spinner

selfdrive/manager/helpers.py

@@ -1,9 +1,16 @@
+import errno
+import fcntl
 import os
 import sys
-import fcntl
-import errno
+import pathlib
+import shutil
 import signal
+import subprocess
+import tempfile
+import threading
 
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
 
 def unblock_stdout() -> None:
   # get a non-blocking stdout
@@ -41,3 +48,20 @@ def unblock_stdout() -> None:
 def write_onroad_params(started, params):
   params.put_bool("IsOnroad", started)
   params.put_bool("IsOffroad", not started)
+
+
+def save_bootlog():
+  # copy current params
+  tmp = tempfile.mkdtemp()
+  params_dirname = pathlib.Path(Params().get_param_path()).name
+  params_dir = os.path.join(tmp, params_dirname)
+  shutil.copytree(Params().get_param_path(), params_dir, dirs_exist_ok=True)
+
+  def fn(tmpdir):
+    env = os.environ.copy()
+    env['PARAMS_COPY_PATH'] = tmpdir
+    subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), env=env)
+    shutil.rmtree(tmpdir)
+  t = threading.Thread(target=fn, args=(tmp, ))
+  t.daemon = True
+  t.start()

selfdrive/manager/manager.py

@@ -19,10 +19,11 @@ from openpilot.selfdrive.manager.helpers import unblock_stdout, write_onroad_par
 from openpilot.selfdrive.manager.process import ensure_running
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
-from openpilot.system.swaglog import cloudlog, add_file_handler
+from openpilot.common.swaglog import cloudlog, add_file_handler
 from openpilot.system.version import is_dirty, get_commit, get_version, get_origin, get_short_branch, \
-                           get_normalized_origin, terms_version, training_version, \
-                           is_tested_branch, is_release_branch
+  get_normalized_origin, terms_version, training_version, \
+  is_tested_branch, is_release_branch, get_commit_date
+
 import json
 from openpilot.selfdrive.car.fingerprints import all_known_cars, all_legacy_fingerprint_cars
 
@@ -38,6 +39,8 @@ def manager_init() -> None:
   params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
   params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
   params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
+  if is_release_branch():
+    params.clear_all(ParamKeyType.DEVELOPMENT_ONLY)
 
   default_params: List[Tuple[str, Union[str, bytes]]] = [
     ("CompletedTrainingVersion", "0"),
@@ -82,9 +85,9 @@ def manager_init() -> None:
     ("dp_long_accel_btn", "0"),
     ("dp_long_personality_btn", "0"),
     ("dp_lat_lane_change_assist_speed", "20"),
-    ("dp_toyota_tss2_radar_disabled", "0"),
     ("dp_device_display_flight_panel", "0"),
     ("dp_ui_rainbow", "0"),
+    ("dp_long_missing_lead_warning", "0"),
   ]
   if not PC:
     default_params.append(("LastUpdateTime", datetime.datetime.utcnow().isoformat().encode('utf8')))
@@ -99,13 +102,6 @@ def manager_init() -> None:
     if params.get(k) is None:
       params.put(k, v)
 
-  # is this dashcam?
-  if os.getenv("PASSIVE") is not None:
-    params.put_bool("Passive", bool(int(os.getenv("PASSIVE", "0"))))
-
-  if params.get("Passive") is None:
-    raise Exception("Passive must be set to continue")
-
   # Create folders needed for msgq
   try:
     os.mkdir("/dev/shm")
@@ -118,9 +114,10 @@ def manager_init() -> None:
   params.put("Version", get_version())
   params.put("TermsVersion", terms_version)
   params.put("TrainingVersion", training_version)
-  params.put("GitCommit", get_commit(default=""))
-  params.put("GitBranch", get_short_branch(default=""))
-  params.put("GitRemote", get_origin(default=""))
+  params.put("GitCommit", get_commit())
+  params.put("GitCommitDate", get_commit_date())
+  params.put("GitBranch", get_short_branch())
+  params.put("GitRemote", get_origin())
   params.put_bool("IsTestedBranch", is_tested_branch())
   params.put_bool("IsReleaseBranch", is_release_branch())
 
@@ -132,6 +129,9 @@ def manager_init() -> None:
     serial = params.get("HardwareSerial")
     raise Exception(f"Registration failed for device {serial}")
   os.environ['DONGLE_ID'] = dongle_id  # Needed for swaglog
+  os.environ['GIT_ORIGIN'] = get_normalized_origin() # Needed for swaglog
+  os.environ['GIT_BRANCH'] = get_short_branch() # Needed for swaglog
+  os.environ['GIT_COMMIT'] = get_commit() # Needed for swaglog
 
   if not is_dirty():
     os.environ['CLEAN'] = '1'
@@ -146,8 +146,7 @@ def manager_init() -> None:
                        dirty=is_dirty(),
                        device=HARDWARE.get_device_type())
 
-
-def manager_prepare() -> None:
+  # preimport all processes
   for p in managed_processes.values():
     p.prepare()
 
@@ -193,7 +192,7 @@ def manager_thread() -> None:
   if not params.get_bool("dp_otisserv"):
     ignore += ["otisserv"]
 
-  sm = messaging.SubMaster(['deviceState', 'carParams'], poll=['deviceState'])
+  sm = messaging.SubMaster(['deviceState', 'carParams'], poll='deviceState')
   pm = messaging.PubMaster(['managerState'])
 
   write_onroad_params(False, params)
@@ -202,7 +201,7 @@ def manager_thread() -> None:
   started_prev = False
 
   while True:
-    sm.update()
+    sm.update(1000)
 
     started = sm['deviceState'].started
 
@@ -219,7 +218,7 @@ def manager_thread() -> None:
 
     ensure_running(managed_processes.values(), started, params=params, CP=sm['carParams'], not_run=ignore)
 
-    running = ' '.join("%s%s\u001b[0m" % ("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
+    running = ' '.join("{}{}\u001b[0m".format("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
                        for p in managed_processes.values() if p.proc)
     print(running)
     cloudlog.debug(running)
@@ -244,17 +243,8 @@ def manager_thread() -> None:
 
 
 def main() -> None:
-  prepare_only = os.getenv("PREPAREONLY") is not None
-
   manager_init()
-
-  # Start UI early so prepare can happen in the background
-  if not prepare_only:
-    managed_processes['ui'].start()
-
-  manager_prepare()
-
-  if prepare_only:
+  if os.getenv("PREPAREONLY") is not None:
     return
 
   # SystemExit on sigterm
@@ -299,6 +289,8 @@ if __name__ == "__main__":
 
   try:
     main()
+  except KeyboardInterrupt:
+    print("got CTRL-C, exiting")
   except Exception:
     add_file_handler(cloudlog)
     cloudlog.exception("Manager failed to start")

selfdrive/manager/process.py

@@ -8,14 +8,14 @@ from typing import Optional, Callable, List, ValuesView
 from abc import ABC, abstractmethod
 from multiprocessing import Process
 
-from setproctitle import setproctitle  # pylint: disable=no-name-in-module
+from setproctitle import setproctitle
 
 from cereal import car, log
 import cereal.messaging as messaging
 import openpilot.selfdrive.sentry as sentry
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 
 WATCHDOG_FN = "/dev/shm/wd_"
 ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None
@@ -89,9 +89,7 @@ def join_process(process: Process, timeout: float) -> None:
 class ManagerProcess(ABC):
   daemon = False
   sigkill = False
-  onroad = True
-  offroad = False
-  callback: Optional[Callable[[bool, Params, car.CarParams], bool]] = None
+  should_run: Callable[[bool, Params, car.CarParams], bool]
   proc: Optional[Process] = None
   enabled = True
   name = ""
@@ -121,7 +119,7 @@ class ManagerProcess(ABC):
       fn = WATCHDOG_FN + str(self.proc.pid)
       with open(fn, "rb") as f:
         # TODO: why can't pylint find struct.unpack?
-        self.last_watchdog_time = struct.unpack('Q', f.read())[0] # pylint: disable=no-member
+        self.last_watchdog_time = struct.unpack('Q', f.read())[0]
     except Exception:
       pass
 
@@ -193,15 +191,12 @@ class ManagerProcess(ABC):
 
 
 class NativeProcess(ManagerProcess):
-  def __init__(self, name, cwd, cmdline, enabled=True, onroad=True, offroad=False, callback=None, unkillable=False, sigkill=False, watchdog_max_dt=None):
+  def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
     self.name = name
     self.cwd = cwd
     self.cmdline = cmdline
+    self.should_run = should_run
     self.enabled = enabled
-    self.onroad = onroad
-    self.offroad = offroad
-    self.callback = callback
-    self.unkillable = unkillable
     self.sigkill = sigkill
     self.watchdog_max_dt = watchdog_max_dt
     self.launcher = nativelauncher
@@ -226,14 +221,11 @@ class NativeProcess(ManagerProcess):
 
 
 class PythonProcess(ManagerProcess):
-  def __init__(self, name, module, enabled=True, onroad=True, offroad=False, callback=None, unkillable=False, sigkill=False, watchdog_max_dt=None):
+  def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
     self.name = name
     self.module = module
+    self.should_run = should_run
     self.enabled = enabled
-    self.onroad = onroad
-    self.offroad = offroad
-    self.callback = callback
-    self.unkillable = unkillable
     self.sigkill = sigkill
     self.watchdog_max_dt = watchdog_max_dt
     self.launcher = launcher
@@ -268,10 +260,12 @@ class DaemonProcess(ManagerProcess):
     self.module = module
     self.param_name = param_name
     self.enabled = enabled
-    self.onroad = True
-    self.offroad = True
     self.params = None
 
+  @staticmethod
+  def should_run(started, params, CP):
+    return True
+
   def prepare(self) -> None:
     pass
 
@@ -292,11 +286,11 @@ class DaemonProcess(ManagerProcess):
         pass
 
     cloudlog.info(f"starting daemon {self.name}")
-    proc = subprocess.Popen(['python', '-m', self.module],  # pylint: disable=subprocess-popen-preexec-fn
-                               stdin=open('/dev/null'),
-                               stdout=open('/dev/null', 'w'),
-                               stderr=open('/dev/null', 'w'),
-                               preexec_fn=os.setpgrp)
+    proc = subprocess.Popen(['python', '-m', self.module],
+                            stdin=open('/dev/null'),
+                            stdout=open('/dev/null', 'w'),
+                            stderr=open('/dev/null', 'w'),
+                            preexec_fn=os.setpgrp)
 
     self.params.put(self.param_name, str(proc.pid))
 
@@ -311,26 +305,14 @@ def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None
 
   running = []
   for p in procs:
-    # Conditions that make a process run
-    run = any((
-      p.offroad and not started,
-      p.onroad and started,
-    ))
-    if p.callback is not None and None not in (params, CP):
-      run = run or p.callback(started, params, CP)
-
-    # Conditions that block a process from starting
-    run = run and not any((
-      not p.enabled,
-      p.name in not_run,
-    ))
-
-    if run:
-      p.start()
+    if p.enabled and p.name not in not_run and p.should_run(started, params, CP):
       running.append(p)
     else:
       p.stop(block=False)
 
     p.check_watchdog(started)
 
+  for p in running:
+    p.start()
+
   return running

selfdrive/manager/process_config.py

@@ -10,22 +10,20 @@ NO_IR_CTRL = os.path.isfile('/data/media/0/no_ir_ctrl')
 WEBCAM = os.getenv("USE_WEBCAM") is not None
 
 def driverview(started: bool, params: Params, CP: car.CarParams) -> bool:
-  return params.get_bool("IsDriverViewEnabled")  # type: ignore
+  return started or params.get_bool("IsDriverViewEnabled")
 
 def notcar(started: bool, params: Params, CP: car.CarParams) -> bool:
-  return CP.notCar  # type: ignore
+  return started and CP.notCar
 
 def iscar(started: bool, params: Params, CP: car.CarParams) -> bool:
-  return not CP.notCar
+  return started and not CP.notCar
 
 def logging(started, params, CP: car.CarParams) -> bool:
   run = (not CP.notCar) or not params.get_bool("DisableLogging")
   return started and run
 
 def ublox_available() -> bool:
-  if EON:
-    return True
-  return os.path.exists('/dev/ttyHS0') and not os.path.exists('/persist/comma/use-quectel-gps')
+  return True if EON else os.path.exists('/dev/ttyHS0') and not os.path.exists('/persist/comma/use-quectel-gps')
 
 def ublox(started, params, CP: car.CarParams) -> bool:
   use_ublox = ublox_available()
@@ -36,65 +34,74 @@ def ublox(started, params, CP: car.CarParams) -> bool:
 def qcomgps(started, params, CP: car.CarParams) -> bool:
   return started and not ublox_available()
 
+def always_run(started, params, CP: car.CarParams) -> bool:
+  return True
+
+def only_onroad(started: bool, params, CP: car.CarParams) -> bool:
+  return started
+
+def only_offroad(started, params, CP: car.CarParams) -> bool:
+  return not started
+
 procs = [
-  NativeProcess("camerad", "selfdrive/camerad", ["./camerad"], callback=driverview),
-  NativeProcess("clocksd", "system/clocksd", ["./clocksd"]),
-  NativeProcess("logcatd", "system/logcatd", ["./logcatd"]),
-  NativeProcess("proclogd", "system/proclogd", ["./proclogd"]),
-  PythonProcess("logmessaged", "system.logmessaged", offroad=True),
+  NativeProcess("camerad", "selfdrive/camerad", ["./camerad"], driverview),
+  NativeProcess("clocksd", "system/clocksd", ["./clocksd"], only_onroad),
+  NativeProcess("logcatd", "system/logcatd", ["./logcatd"], only_onroad),
+  NativeProcess("proclogd", "system/proclogd", ["./proclogd"], only_onroad),
+  PythonProcess("logmessaged", "system.logmessaged", always_run),
   # PythonProcess("micd", "system.micd", callback=iscar),
   # PythonProcess("timezoned", "system.timezoned", enabled=not PC, offroad=True),
 
   DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"),
-  NativeProcess("dmonitoringmodeld", "selfdrive/hybrid_modeld", ["./dmonitoringmodeld"], enabled=(not PC or WEBCAM) and not NO_IR_CTRL, callback=driverview),
+  NativeProcess("dmonitoringmodeld", "selfdrive/hybrid_modeld", ["./dmonitoringmodeld"], driverview, enabled=(not PC or WEBCAM) and not NO_IR_CTRL),
   # NativeProcess("encoderd", "system/loggerd", ["./encoderd"]),
   # NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], onroad=False, callback=notcar),
-  NativeProcess("loggerd", "selfdrive/loggerd", ["./loggerd"], onroad=False, callback=logging),
-  NativeProcess("modeld", "selfdrive/hybrid_modeld" if not Params().get_bool("dp_0813") else "selfdrive/legacy_modeld", ["./modeld"]),
+  NativeProcess("loggerd", "selfdrive/loggerd", ["./loggerd"], logging),
+  NativeProcess("modeld", "selfdrive/hybrid_modeld" if not Params().get_bool("dp_0813") else "selfdrive/legacy_modeld", ["./modeld"], only_onroad),
   # NativeProcess("mapsd", "selfdrive/navd", ["./mapsd"]),
   # NativeProcess("navmodeld", "selfdrive/modeld", ["./navmodeld"]),
-  NativeProcess("sensord", "system/sensord", ["./sensord"], enabled=not PC, offroad=EON),
-  NativeProcess("ui", "selfdrive/ui", ["./ui"], offroad=True, watchdog_max_dt=(5 if not PC else None)),
-  NativeProcess("soundd", "selfdrive/ui/soundd", ["./soundd"]),
-  NativeProcess("locationd", "selfdrive/locationd", ["./locationd"]),
-  NativeProcess("boardd", "selfdrive/boardd", ["./boardd"], enabled=False),
-  PythonProcess("calibrationd", "selfdrive.locationd.calibrationd"),
-  PythonProcess("torqued", "selfdrive.locationd.torqued"),
-  PythonProcess("controlsd", "selfdrive.controls.controlsd"),
-  PythonProcess("deleter", "selfdrive.loggerd.deleter", offroad=True),
-  PythonProcess("dmonitoringd", "selfdrive.legacy_monitoring.dmonitoringd", enabled=(not PC or WEBCAM) and not NO_IR_CTRL, callback=driverview),
+  NativeProcess("sensord", "system/sensord", ["./sensord"], always_run if EON else only_onroad, enabled=not PC),
+  NativeProcess("ui", "selfdrive/ui", ["./ui"], always_run, watchdog_max_dt=(5 if not PC else None)),
+  NativeProcess("soundd", "selfdrive/ui/soundd", ["./soundd"], only_onroad),
+  NativeProcess("locationd", "selfdrive/locationd", ["./locationd"], only_onroad),
+  NativeProcess("boardd", "selfdrive/boardd", ["./boardd"], always_run, enabled=False),
+  PythonProcess("calibrationd", "selfdrive.locationd.calibrationd", only_onroad),
+  PythonProcess("torqued", "selfdrive.locationd.torqued", only_onroad),
+  PythonProcess("controlsd", "selfdrive.controls.controlsd", only_onroad),
+  PythonProcess("deleter", "selfdrive.loggerd.deleter", always_run),
+  PythonProcess("dmonitoringd", "selfdrive.legacy_monitoring.dmonitoringd", driverview, enabled=(not PC or WEBCAM) and not NO_IR_CTRL),
   # PythonProcess("laikad", "selfdrive.locationd.laikad"),
   # PythonProcess("rawgpsd", "system.sensord.rawgps.rawgpsd", enabled=TICI, onroad=False, callback=qcomgps),
   # PythonProcess("navd", "selfdrive.navd.navd"),
-  PythonProcess("pandad", "selfdrive.boardd.pandad", offroad=True),
-  PythonProcess("paramsd", "selfdrive.locationd.paramsd"),
-  NativeProcess("ubloxd", "system/ubloxd", ["./ubloxd"], enabled=not PC, onroad=False, callback=ublox),
+  PythonProcess("pandad", "selfdrive.boardd.pandad", always_run),
+  PythonProcess("paramsd", "selfdrive.locationd.paramsd", only_onroad),
+  NativeProcess("ubloxd", "system/ubloxd", ["./ubloxd"], ublox, enabled=not PC),
   # PythonProcess("pigeond", "system.sensord.pigeond", enabled=TICI, onroad=False, callback=ublox),
-  PythonProcess("plannerd", "selfdrive.controls.plannerd"),
-  PythonProcess("radard", "selfdrive.controls.radard"),
-  PythonProcess("thermald", "selfdrive.thermald.thermald", offroad=True),
-  PythonProcess("tombstoned", "selfdrive.tombstoned", enabled=not PC, offroad=True),
-  PythonProcess("updated", "selfdrive.updated", enabled=not PC, onroad=False, offroad=True),
-  # PythonProcess("uploader", "selfdrive.loggerd.uploader", offroad=True),
+  PythonProcess("plannerd", "selfdrive.controls.plannerd", only_onroad),
+  PythonProcess("radard", "selfdrive.controls.radard", only_onroad),
+  PythonProcess("thermald", "selfdrive.thermald.thermald", always_run),
+  PythonProcess("tombstoned", "selfdrive.tombstoned", always_run, enabled=not PC),
+  PythonProcess("updated", "selfdrive.updated", only_offroad, enabled=not PC),
+  PythonProcess("uploader", "selfdrive.loggerd.uploader", only_offroad),
   # PythonProcess("statsd", "selfdrive.statsd", offroad=True),
 
   # debug procs
-  NativeProcess("bridge", "cereal/messaging", ["./bridge"], onroad=False, callback=notcar),
+  NativeProcess("bridge", "cereal/messaging", ["./bridge"], notcar),
   # rick - webjoystick needs aiohttp, install additional modules manually: pip install aiohttp aiortc
   # PythonProcess("webjoystick", "tools.bodyteleop.web", onroad=False, callback=notcar),
 
   # EON only
-  PythonProcess("rtshield", "selfdrive.rtshield", enabled=EON),
-  PythonProcess("shutdownd", "system.hardware.eon.shutdownd", enabled=EON),
-  PythonProcess("androidd", "system.hardware.eon.androidd", enabled=EON, offroad=True),
+  PythonProcess("rtshield", "selfdrive.rtshield", only_onroad, enabled=EON),
+  PythonProcess("shutdownd", "system.hardware.eon.shutdownd", only_onroad, enabled=EON),
+  PythonProcess("androidd", "system.hardware.eon.androidd", always_run, enabled=EON),
 
   # mapd
-  PythonProcess("mapd", "selfdrive.mapd.mapd"),
+  PythonProcess("mapd", "selfdrive.mapd.mapd", only_onroad),
   # gpxd
   # PythonProcess("gpxd", "selfdrive.dragonpilot.gpxd"),
   # PythonProcess("gpx_uploader", "selfdrive.dragonpilot.gpx_uploader", offroad=True),
-  NativeProcess("otisserv", "selfdrive/dragonpilot", ['./otisserv'], offroad=True),
-  NativeProcess("fileserv", "selfdrive/dragonpilot", ['./fileserv'], offroad=True),
+  NativeProcess("otisserv", "selfdrive/dragonpilot", ['./otisserv'], only_offroad),
+  NativeProcess("fileserv", "selfdrive/dragonpilot", ['./fileserv'], only_offroad),
 ]
 
 managed_processes = {p.name: p for p in procs}

selfdrive/updated.py

@@ -17,8 +17,8 @@ from markdown_it import MarkdownIt
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
 from openpilot.common.time import system_time_valid
-from openpilot.system.hardware import AGNOS, HARDWARE
-from openpilot.system.swaglog import cloudlog
+from openpilot.system.hardware import AGNOS, HARDWARE, EON
+from openpilot.common.swaglog import cloudlog
 from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
 from openpilot.system.version import is_tested_branch
 
@@ -37,26 +37,42 @@ OVERLAY_INIT = Path(os.path.join(BASEDIR, ".overlay_init"))
 DAYS_NO_CONNECTIVITY_MAX = 14     # do not allow to engage after this many days
 DAYS_NO_CONNECTIVITY_PROMPT = 10  # send an offroad prompt after this many days
 
+class UserRequest:
+  NONE = 0
+  CHECK = 1
+  FETCH = 2
+
 class WaitTimeHelper:
   def __init__(self):
     self.ready_event = threading.Event()
-    self.only_check_for_update = False
+    self.user_request = UserRequest.NONE
     signal.signal(signal.SIGHUP, self.update_now)
     signal.signal(signal.SIGUSR1, self.check_now)
 
   def update_now(self, signum: int, frame) -> None:
     cloudlog.info("caught SIGHUP, attempting to downloading update")
-    self.only_check_for_update = False
+    self.user_request = UserRequest.FETCH
     self.ready_event.set()
 
   def check_now(self, signum: int, frame) -> None:
     cloudlog.info("caught SIGUSR1, checking for updates")
-    self.only_check_for_update = True
+    self.user_request = UserRequest.CHECK
     self.ready_event.set()
 
   def sleep(self, t: float) -> None:
     self.ready_event.wait(timeout=t)
 
+def write_time_to_param(params, param) -> None:
+  t = datetime.datetime.utcnow()
+  params.put(param, t.isoformat().encode('utf8'))
+
+def read_time_from_param(params, param) -> datetime.datetime:
+  t = params.get(param, encoding='utf8')
+  try:
+    return datetime.datetime.fromisoformat(t)
+  except (TypeError, ValueError):
+    pass
+  return None
 
 def run(cmd: List[str], cwd: Optional[str] = None) -> str:
   return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')
@@ -73,7 +89,8 @@ def set_consistent_flag(consistent: bool) -> None:
 
 def parse_release_notes(basedir: str) -> bytes:
   try:
-    with open(os.path.join(basedir, "RELEASES.md"), "rb") as f:
+    version_type = "D2" if os.path.isfile(os.path.join(basedir, "d2")) else "R2"
+    with open(os.path.join(basedir, f"CHANGELOGS-{version_type}.md"), "rb") as f:
       r = f.read().split(b'\n\n', 1)[0]  # Slice latest release notes
     try:
       return bytes(MarkdownIt().render(r.decode("utf-8")), encoding="utf-8")
@@ -135,8 +152,10 @@ def init_overlay() -> None:
   params.put_bool("UpdateAvailable", False)
   set_consistent_flag(False)
   dismount_overlay()
+  args = ["rm", "-rf", STAGING_ROOT]
   if AGNOS:
-    run(["sudo", "rm", "-rf", STAGING_ROOT])
+    args = ["sudo"] + args
+    run(args)
   if os.path.isdir(STAGING_ROOT):
     shutil.rmtree(STAGING_ROOT)
 
@@ -244,7 +263,7 @@ def handle_neos_update() -> None:
 class Updater:
   def __init__(self):
     self.params = Params()
-    self.branches = defaultdict(lambda: '')
+    self.branches = defaultdict(str)
     self._has_internet: bool = False
 
   @property
@@ -253,10 +272,9 @@ class Updater:
 
   @property
   def target_branch(self) -> str:
-    b: Union[str, None] = self.params.get("UpdaterTargetBranch", encoding='utf-8')
+    b: str | None = self.params.get("UpdaterTargetBranch", encoding='utf-8')
     if b is None:
       b = self.get_branch(BASEDIR)
-      self.params.put("UpdaterTargetBranch", b)
     return b
 
   @property
@@ -271,7 +289,7 @@ class Updater:
 
   @property
   def update_available(self) -> bool:
-    if os.path.isdir(OVERLAY_MERGED):
+    if os.path.isdir(OVERLAY_MERGED) and len(self.branches) > 0:
       hash_mismatch = self.get_commit_hash(OVERLAY_MERGED) != self.branches[self.target_branch]
       branch_mismatch = self.get_branch(OVERLAY_MERGED) != self.target_branch
       return hash_mismatch or branch_mismatch
@@ -285,20 +303,19 @@ class Updater:
 
   def set_params(self, update_success: bool, failed_count: int, exception: Optional[str]) -> None:
     self.params.put("UpdateFailedCount", str(failed_count))
+    self.params.put("UpdaterTargetBranch", self.target_branch)
 
     self.params.put_bool("UpdaterFetchAvailable", self.update_available)
-    self.params.put("UpdaterAvailableBranches", ','.join(self.branches.keys()))
+    if len(self.branches):
+      self.params.put("UpdaterAvailableBranches", ','.join(self.branches.keys()))
 
     last_update = datetime.datetime.utcnow()
     if update_success:
-      t = last_update.isoformat()
-      self.params.put("LastUpdateTime", t.encode('utf8'))
+      write_time_to_param(self.params, "LastUpdateTime")
     else:
-      try:
-        t = self.params.get("LastUpdateTime", encoding='utf8')
-        last_update = datetime.datetime.fromisoformat(t)
-      except (TypeError, ValueError):
-        pass
+      t = read_time_from_param(self.params, "LastUpdateTime")
+      if t is not None:
+        last_update = t
 
     if exception is None:
       self.params.remove("LastUpdateException")
@@ -344,17 +361,17 @@ class Updater:
       else:
         extra_text = exception
       set_offroad_alert("Offroad_UpdateFailed", True, extra_text=extra_text)
-    elif failed_count > 0:
-      if dt.days > DAYS_NO_CONNECTIVITY_MAX:
-        set_offroad_alert("Offroad_ConnectivityNeeded", True)
-      elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT:
-        remaining = max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 1)
-        set_offroad_alert("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining} day{'' if remaining == 1 else 's'}.")
+    # elif failed_count > 0:
+    #   if dt.days > DAYS_NO_CONNECTIVITY_MAX:
+    #     set_offroad_alert("Offroad_ConnectivityNeeded", True)
+    #   elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT:
+    #     remaining = max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 1)
+    #     set_offroad_alert("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining} day{'' if remaining == 1 else 's'}.")
 
   def check_for_update(self) -> None:
     cloudlog.info("checking for updates")
 
-    excluded_branches = ('release2', 'release2-staging', 'dashcam', 'dashcam-staging')
+    excluded_branches = ('beta3', 'r3', 'd3')
 
     try:
       run(["git", "ls-remote", "origin", "HEAD"], OVERLAY_MERGED)
@@ -422,97 +439,100 @@ class Updater:
 def main() -> None:
   params = Params()
 
-  updater = Updater()
-  update_failed_count = 0  # TODO: Load from param?
-  exception = None
-
   if params.get_bool("DisableUpdates"):
-    updater.set_params(False, update_failed_count, exception)
     cloudlog.warning("updates are disabled by the DisableUpdates param")
     exit(0)
 
-  ov_lock_fd = open(LOCK_FILE, 'w')
-  try:
-    fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except OSError as e:
-    raise RuntimeError("couldn't get overlay lock; is another instance running?") from e
-
-  # Set low io priority
-  proc = psutil.Process()
-  if psutil.LINUX:
-    proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)
-
-  # Check if we just performed an update
-  if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
-    cloudlog.event("update installed")
-
-  if not params.get("InstallDate"):
-    t = datetime.datetime.utcnow().isoformat()
-    params.put("InstallDate", t.encode('utf8'))
-
-  # updater = Updater()
-  # update_failed_count = 0 # TODO: Load from param?
-
-  # no fetch on the first time
-  wait_helper = WaitTimeHelper()
-  wait_helper.only_check_for_update = True
-
-  # invalidate old finalized update
-  set_consistent_flag(False)
-
-  # Run the update loop
-  while True:
-    wait_helper.ready_event.clear()
-
-    # Attempt an update
-    # exception = None
+  with open(LOCK_FILE, 'w') as ov_lock_fd:
     try:
-      # TODO: reuse overlay from previous updated instance if it looks clean
-      init_overlay()
+      fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError as e:
+      raise RuntimeError("couldn't get overlay lock; is another instance running?") from e
 
-      # ensure we have some params written soon after startup
-      updater.set_params(False, update_failed_count, exception)
+    # Set low io priority
+    proc = psutil.Process()
+    if psutil.LINUX:
+      proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)
 
-      if not system_time_valid():
-        wait_helper.sleep(60)
-        continue
+    # Check if we just performed an update
+    if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
+      cloudlog.event("update installed")
 
-      update_failed_count += 1
+    if not params.get("InstallDate"):
+      t = datetime.datetime.utcnow().isoformat()
+      params.put("InstallDate", t.encode('utf8'))
 
-      # check for update
-      params.put("UpdaterState", "checking...")
-      updater.check_for_update()
+    updater = Updater()
+    update_failed_count = 0 # TODO: Load from param?
+    wait_helper = WaitTimeHelper()
 
-      # download update
-      if wait_helper.only_check_for_update:
-        cloudlog.info("skipping fetch this cycle")
-      else:
-        updater.fetch_update()
-      update_failed_count = 0
-    except subprocess.CalledProcessError as e:
-      cloudlog.event(
-        "update process failed",
-        cmd=e.cmd,
-        output=e.output,
-        returncode=e.returncode
-      )
-      exception = f"command failed: {e.cmd}\n{e.output}"
-      OVERLAY_INIT.unlink(missing_ok=True)
-    except Exception as e:
-      cloudlog.exception("uncaught updated exception, shouldn't happen")
-      exception = str(e)
-      OVERLAY_INIT.unlink(missing_ok=True)
+    # invalidate old finalized update
+    set_consistent_flag(False)
 
-    try:
-      params.put("UpdaterState", "idle")
-      update_successful = (update_failed_count == 0)
-      updater.set_params(update_successful, update_failed_count, exception)
-    except Exception:
-      cloudlog.exception("uncaught updated exception while setting params, shouldn't happen")
+    # set initial state
+    params.put("UpdaterState", "idle")
 
-    # infrequent attempts if we successfully updated recently
-    wait_helper.only_check_for_update = False
-    wait_helper.sleep(5*60 if update_failed_count > 0 else 1.5*60*60)
+    # Run the update loop
+    first_run = True
+    while True:
+      wait_helper.ready_event.clear()
+
+      # Attempt an update
+      exception = None
+      try:
+        # TODO: reuse overlay from previous updated instance if it looks clean
+        init_overlay()
+
+        # ensure we have some params written soon after startup
+        updater.set_params(False, update_failed_count, exception)
+
+        if not system_time_valid() or first_run:
+          first_run = False
+          wait_helper.sleep(60)
+          continue
+
+        update_failed_count += 1
+
+        # check for update
+        params.put("UpdaterState", "checking...")
+        updater.check_for_update()
+
+        # download update
+        last_fetch = read_time_from_param(params, "UpdaterLastFetchTime")
+        timed_out = last_fetch is None or (datetime.datetime.utcnow() - last_fetch > datetime.timedelta(days=3))
+        user_requested_fetch = wait_helper.user_request == UserRequest.FETCH
+        if params.get_bool("NetworkMetered") and not timed_out and not user_requested_fetch:
+          cloudlog.info("skipping fetch, connection metered")
+        elif wait_helper.user_request == UserRequest.CHECK:
+          cloudlog.info("skipping fetch, only checking")
+        else:
+          updater.fetch_update()
+          write_time_to_param(params, "UpdaterLastFetchTime")
+        update_failed_count = 0
+      except subprocess.CalledProcessError as e:
+        cloudlog.event(
+          "update process failed",
+          cmd=e.cmd,
+          output=e.output,
+          returncode=e.returncode
+        )
+        exception = f"command failed: {e.cmd}\n{e.output}"
+        OVERLAY_INIT.unlink(missing_ok=True)
+      except Exception as e:
+        cloudlog.exception("uncaught updated exception, shouldn't happen")
+        exception = str(e)
+        OVERLAY_INIT.unlink(missing_ok=True)
+
+      try:
+        params.put("UpdaterState", "idle")
+        update_successful = (update_failed_count == 0)
+        updater.set_params(update_successful, update_failed_count, exception)
+      except Exception:
+        cloudlog.exception("uncaught updated exception while setting params, shouldn't happen")
+
+      # infrequent attempts if we successfully updated recently
+      wait_helper.user_request = UserRequest.NONE
+      wait_helper.sleep(5*60 if update_failed_count > 0 else 1.5*60*60)
 
 
 if __name__ == "__main__":

system/hardware/eon/shutdownd.py

@@ -20,7 +20,7 @@ def main():
       # 0 for shutdown, 1 for reboot
       prop = getprop("sys.shutdown.requested")
       if prop is not None and len(prop) > 0:
-        # os.system("pkill -9 loggerd")
+        os.system("pkill -9 loggerd")
         params.put("LastSystemShutdown", f"'{prop}' {datetime.datetime.now()}")
         os.sync()
 

system/hardware/hw.py

@@ -0,0 +1,58 @@
+import os
+from pathlib import Path
+
+from openpilot.system.hardware import PC
+
+DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache"
+
+class Paths:
+  @staticmethod
+  def comma_home() -> str:
+    return os.path.join(str(Path.home()), ".comma" + os.environ.get("OPENPILOT_PREFIX", ""))
+
+  @staticmethod
+  def log_root() -> str:
+    if os.environ.get('LOG_ROOT', False):
+      return os.environ['LOG_ROOT']
+    elif PC:
+      return str(Path(Paths.comma_home()) / "media" / "0" / "realdata")
+    else:
+      return '/data/media/0/realdata/'
+
+  @staticmethod
+  def swaglog_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "log")
+    else:
+      return "/data/log/"
+
+  @staticmethod
+  def swaglog_ipc() -> str:
+    return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "")
+
+  @staticmethod
+  def download_cache_root() -> str:
+    if os.environ.get('COMMA_CACHE', False):
+      return os.environ['COMMA_CACHE'] + "/"
+    return DEFAULT_DOWNLOAD_CACHE_ROOT + os.environ.get("OPENPILOT_PREFIX", "") + "/"
+
+  @staticmethod
+  def persist_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "persist")
+    else:
+      return "/persist/"
+
+  @staticmethod
+  def stats_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "stats")
+    else:
+      return "/data/stats/"
+
+  @staticmethod
+  def config_root() -> str:
+    if PC:
+      return Paths.comma_home()
+    else:
+      return "/tmp/.comma"

system/version.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 import os
 import subprocess
-from typing import List, Optional
+from typing import List, Optional, TypeVar
 from functools import lru_cache
 
 from openpilot.common.basedir import BASEDIR
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 
 RELEASE_BRANCHES = ['release3-staging', 'dashcam3-staging', 'release3', 'dashcam3', 'nightly']
 TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging']
@@ -13,7 +13,7 @@ TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging']
 training_version: bytes = b"0.2.0"
 terms_version: bytes = b"2"
 
-
+_RT = TypeVar("_RT")
 def cache(user_function, /):
   return lru_cache(maxsize=None)(user_function)
 
@@ -30,41 +30,45 @@ def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[s
 
 
 @cache
-def get_commit(branch: str = "HEAD", default: Optional[str] = None) -> Optional[str]:
-  return run_cmd_default(["git", "rev-parse", branch], default=default)
+def get_commit(branch: str = "HEAD") -> str:
+  return run_cmd_default(["git", "rev-parse", branch])
 
 
 @cache
-def get_short_branch(default: Optional[str] = None) -> Optional[str]:
-  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], default=default)
+def get_commit_date(commit: str = "HEAD") -> str:
+  return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit])
 
 
 @cache
-def get_branch(default: Optional[str] = None) -> Optional[str]:
-  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], default=default)
+def get_short_branch() -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"])
 
 
 @cache
-def get_origin(default: Optional[str] = None) -> Optional[str]:
+def get_branch() -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"])
+
+
+@cache
+def get_origin() -> str:
+  origin = ""
   try:
     local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"])
     tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"])
-    return run_cmd(["git", "config", "remote." + tracking_remote + ".url"])
+    origin = run_cmd(["git", "config", "remote." + tracking_remote + ".url"])
   except subprocess.CalledProcessError:  # Not on a branch, fallback
-    return run_cmd_default(["git", "config", "--get", "remote.origin.url"], default=default)
-
+    origin = run_cmd_default(["git", "config", "--get", "remote.origin.url"])
+  except TypeError:
+    pass
+  return "" if origin is None else origin
 
 @cache
-def get_normalized_origin(default: Optional[str] = None) -> Optional[str]:
-  origin: Optional[str] = get_origin()
-
-  if origin is None:
-    return default
-
-  return origin.replace("git@", "", 1) \
-               .replace(".git", "", 1) \
-               .replace("https://", "", 1) \
-               .replace(":", "/", 1)
+def get_normalized_origin() -> str:
+  return get_origin() \
+    .replace("git@", "", 1) \
+    .replace(".git", "", 1) \
+    .replace("https://", "", 1) \
+    .replace(":", "/", 1)
 
 
 @cache
@@ -75,7 +79,7 @@ def get_version() -> str:
 
 @cache
 def get_short_version() -> str:
-  return get_version().split('-')[0]  # type: ignore
+  return get_version().split('-')[0]
 
 @cache
 def is_prebuilt() -> bool:
@@ -86,12 +90,7 @@ def is_prebuilt() -> bool:
 def is_comma_remote() -> bool:
   # note to fork maintainers, this is used for release metrics. please do not
   # touch this to get rid of the orange startup alert. there's better ways to do that
-  origin: Optional[str] = get_origin()
-  if origin is None:
-    return False
-
-  return origin.startswith(('git@github.com:commaai', 'https://github.com/commaai'))
-
+  return get_normalized_origin() == "github.com/commaai/openpilot"
 
 @cache
 def is_tested_branch() -> bool:
@@ -105,7 +104,7 @@ def is_release_branch() -> bool:
 def is_dirty() -> bool:
   origin = get_origin()
   branch = get_branch()
-  if (origin is None) or (branch is None):
+  if not origin or not branch:
     return True
 
   dirty = False
@@ -141,3 +140,4 @@ if __name__ == "__main__":
   print(f"Branch: {get_branch()}")
   print(f"Short branch: {get_short_branch()}")
   print(f"Prebuilt: {is_prebuilt()}")
+  print(f"Commit date: {get_commit_date()}")