openpilot v0.9.7 release

date: 2024-06-11T01:36:39
master commit: f8cb04e4a8

system/hardware/.gitignore

@@ -0,0 +1 @@
+eon/rat

system/hardware/base.h

@@ -5,7 +5,7 @@
 #include <map>
 #include <string>
 
-#include "cereal/messaging/messaging.h"
+#include "cereal/gen/cpp/log.capnp.h"
 
 // no-op base hw class
 class HardwareNone {

system/hardware/base.py

@@ -1,6 +1,5 @@
 from abc import abstractmethod, ABC
 from collections import namedtuple
-from typing import Dict
 
 from cereal import log
 
@@ -10,7 +9,7 @@ NetworkType = log.DeviceState.NetworkType
 
 class HardwareBase(ABC):
   @staticmethod
-  def get_cmdline() -> Dict[str, str]:
+  def get_cmdline() -> dict[str, str]:
     with open('/proc/cmdline') as f:
       cmdline = f.read()
     return {kv[0]: kv[1] for kv in [s.split('=') for s in cmdline.split(' ')] if len(kv) == 2}

system/hardware/fan_controller.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+from abc import ABC, abstractmethod
+
+from openpilot.common.realtime import DT_HW
+from openpilot.common.numpy_fast import interp
+from openpilot.common.swaglog import cloudlog
+from openpilot.selfdrive.controls.lib.pid import PIDController
+
+class BaseFanController(ABC):
+  @abstractmethod
+  def update(self, cur_temp: float, ignition: bool) -> int:
+    pass
+
+
+class TiciFanController(BaseFanController):
+  def __init__(self) -> None:
+    super().__init__()
+    cloudlog.info("Setting up TICI fan handler")
+
+    self.last_ignition = False
+    self.controller = PIDController(k_p=0, k_i=4e-3, k_f=1, rate=(1 / DT_HW))
+
+  def update(self, cur_temp: float, ignition: bool) -> int:
+    self.controller.neg_limit = -(100 if ignition else 30)
+    self.controller.pos_limit = -(30 if ignition else 0)
+
+    if ignition != self.last_ignition:
+      self.controller.reset()
+
+    error = 70 - cur_temp
+    fan_pwr_out = -int(self.controller.update(
+                      error=error,
+                      feedforward=interp(cur_temp, [60.0, 100.0], [0, -100])
+                    ))
+
+    self.last_ignition = ignition
+    return fan_pwr_out
+

system/hardware/hardwared.py

@@ -0,0 +1,474 @@
+#!/usr/bin/env python3
+import os
+import json
+import queue
+import threading
+import time
+from collections import OrderedDict, namedtuple
+from pathlib import Path
+
+import psutil
+
+import cereal.messaging as messaging
+from cereal import log
+from cereal.services import SERVICE_LIST
+from openpilot.common.dict_helpers import strip_deprecated_keys
+from openpilot.common.filter_simple import FirstOrderFilter
+from openpilot.common.params import Params
+from openpilot.common.realtime import DT_HW
+from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
+from openpilot.system.hardware import HARDWARE, TICI, AGNOS
+from openpilot.system.loggerd.config import get_available_percent
+from openpilot.system.statsd import statlog
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.hardware.power_monitoring import PowerMonitoring
+from openpilot.system.hardware.fan_controller import TiciFanController
+from openpilot.system.version import terms_version, training_version
+
+ThermalStatus = log.DeviceState.ThermalStatus
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+CURRENT_TAU = 15.   # 15s time constant
+TEMP_TAU = 5.   # 5s time constant
+DISCONNECT_TIMEOUT = 5.  # wait 5 seconds before going offroad after disconnect so you get an alert
+PANDA_STATES_TIMEOUT = round(1000 / SERVICE_LIST['pandaStates'].frequency * 1.5)  # 1.5x the expected pandaState frequency
+
+ThermalBand = namedtuple("ThermalBand", ['min_temp', 'max_temp'])
+HardwareState = namedtuple("HardwareState", ['network_type', 'network_info', 'network_strength', 'network_stats',
+                                             'network_metered', 'nvme_temps', 'modem_temps'])
+
+# List of thermal bands. We will stay within this region as long as we are within the bounds.
+# When exiting the bounds, we'll jump to the lower or higher band. Bands are ordered in the dict.
+THERMAL_BANDS = OrderedDict({
+  ThermalStatus.green: ThermalBand(None, 80.0),
+  ThermalStatus.yellow: ThermalBand(75.0, 96.0),
+  ThermalStatus.red: ThermalBand(88.0, 107.),
+  ThermalStatus.danger: ThermalBand(94.0, None),
+})
+
+# Override to highest thermal band when offroad and above this temp
+OFFROAD_DANGER_TEMP = 75
+
+prev_offroad_states: dict[str, tuple[bool, str | None]] = {}
+
+tz_by_type: dict[str, int] | None = None
+def populate_tz_by_type():
+  global tz_by_type
+  tz_by_type = {}
+  for n in os.listdir("/sys/devices/virtual/thermal"):
+    if not n.startswith("thermal_zone"):
+      continue
+    with open(os.path.join("/sys/devices/virtual/thermal", n, "type")) as f:
+      tz_by_type[f.read().strip()] = int(n.removeprefix("thermal_zone"))
+
+def read_tz(x):
+  if x is None:
+    return 0
+
+  if isinstance(x, str):
+    if tz_by_type is None:
+      populate_tz_by_type()
+    x = tz_by_type[x]
+
+  try:
+    with open(f"/sys/devices/virtual/thermal/thermal_zone{x}/temp") as f:
+      return int(f.read())
+  except FileNotFoundError:
+    return 0
+
+
+def read_thermal(thermal_config):
+  dat = messaging.new_message('deviceState', valid=True)
+  dat.deviceState.cpuTempC = [read_tz(z) / thermal_config.cpu[1] for z in thermal_config.cpu[0]]
+  dat.deviceState.gpuTempC = [read_tz(z) / thermal_config.gpu[1] for z in thermal_config.gpu[0]]
+  dat.deviceState.memoryTempC = read_tz(thermal_config.mem[0]) / thermal_config.mem[1]
+  dat.deviceState.pmicTempC = [read_tz(z) / thermal_config.pmic[1] for z in thermal_config.pmic[0]]
+  return dat
+
+
+def set_offroad_alert_if_changed(offroad_alert: str, show_alert: bool, extra_text: str | None=None):
+  if prev_offroad_states.get(offroad_alert, None) == (show_alert, extra_text):
+    return
+  prev_offroad_states[offroad_alert] = (show_alert, extra_text)
+  set_offroad_alert(offroad_alert, show_alert, extra_text)
+
+
+def hw_state_thread(end_event, hw_queue):
+  """Handles non critical hardware state, and sends over queue"""
+  count = 0
+  prev_hw_state = None
+
+  modem_version = None
+  modem_nv = None
+  modem_configured = False
+  modem_restarted = False
+  modem_missing_count = 0
+
+  while not end_event.is_set():
+    # these are expensive calls. update every 10s
+    if (count % int(10. / DT_HW)) == 0:
+      try:
+        network_type = HARDWARE.get_network_type()
+        modem_temps = HARDWARE.get_modem_temperatures()
+        if len(modem_temps) == 0 and prev_hw_state is not None:
+          modem_temps = prev_hw_state.modem_temps
+
+        # Log modem version once
+        if AGNOS and ((modem_version is None) or (modem_nv is None)):
+          modem_version = HARDWARE.get_modem_version()
+          modem_nv = HARDWARE.get_modem_nv()
+
+          if (modem_version is not None) and (modem_nv is not None):
+            cloudlog.event("modem version", version=modem_version, nv=modem_nv)
+          else:
+            if not modem_restarted:
+              # TODO: we may be able to remove this with a MM update
+              # ModemManager's probing on startup can fail
+              # rarely, restart the service to probe again.
+              modem_missing_count += 1
+              if modem_missing_count > 3:
+                modem_restarted = True
+                cloudlog.event("restarting ModemManager")
+                os.system("sudo systemctl restart --no-block ModemManager")
+
+        tx, rx = HARDWARE.get_modem_data_usage()
+
+        hw_state = HardwareState(
+          network_type=network_type,
+          network_info=HARDWARE.get_network_info(),
+          network_strength=HARDWARE.get_network_strength(network_type),
+          network_stats={'wwanTx': tx, 'wwanRx': rx},
+          network_metered=HARDWARE.get_network_metered(network_type),
+          nvme_temps=HARDWARE.get_nvme_temperatures(),
+          modem_temps=modem_temps,
+        )
+
+        try:
+          hw_queue.put_nowait(hw_state)
+        except queue.Full:
+          pass
+
+        # TODO: remove this once the config is in AGNOS
+        if not modem_configured and len(HARDWARE.get_sim_info().get('sim_id', '')) > 0:
+          cloudlog.warning("configuring modem")
+          HARDWARE.configure_modem()
+          modem_configured = True
+
+        prev_hw_state = hw_state
+      except Exception:
+        cloudlog.exception("Error getting hardware state")
+
+    count += 1
+    time.sleep(DT_HW)
+
+
+def hardware_thread(end_event, hw_queue) -> None:
+  pm = messaging.PubMaster(['deviceState'])
+  sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "controlsState", "pandaStates"], poll="pandaStates")
+
+  count = 0
+
+  onroad_conditions: dict[str, bool] = {
+    "ignition": False,
+  }
+  startup_conditions: dict[str, bool] = {}
+  startup_conditions_prev: dict[str, bool] = {}
+
+  off_ts: float | None = None
+  started_ts: float | None = None
+  started_seen = False
+  startup_blocked_ts: float | None = None
+  thermal_status = ThermalStatus.yellow
+
+  last_hw_state = HardwareState(
+    network_type=NetworkType.none,
+    network_info=None,
+    network_metered=False,
+    network_strength=NetworkStrength.unknown,
+    network_stats={'wwanTx': -1, 'wwanRx': -1},
+    nvme_temps=[],
+    modem_temps=[],
+  )
+
+  all_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
+  offroad_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
+  should_start_prev = False
+  in_car = False
+  engaged_prev = False
+
+  params = Params()
+  power_monitor = PowerMonitoring()
+
+  HARDWARE.initialize_hardware()
+  thermal_config = HARDWARE.get_thermal_config()
+
+  fan_controller = None
+
+  while not end_event.is_set():
+    sm.update(PANDA_STATES_TIMEOUT)
+
+    pandaStates = sm['pandaStates']
+    peripheralState = sm['peripheralState']
+    peripheral_panda_present = peripheralState.pandaType != log.PandaState.PandaType.unknown
+
+    if sm.updated['pandaStates'] and len(pandaStates) > 0:
+
+      # Set ignition based on any panda connected
+      onroad_conditions["ignition"] = any(ps.ignitionLine or ps.ignitionCan for ps in pandaStates if ps.pandaType != log.PandaState.PandaType.unknown)
+
+      pandaState = pandaStates[0]
+
+      in_car = pandaState.harnessStatus != log.PandaState.HarnessStatus.notConnected
+
+      # Setup fan handler on first connect to panda
+      if fan_controller is None and peripheral_panda_present:
+        if TICI:
+          fan_controller = TiciFanController()
+
+    elif (time.monotonic() - sm.recv_time['pandaStates']) > DISCONNECT_TIMEOUT:
+      if onroad_conditions["ignition"]:
+        onroad_conditions["ignition"] = False
+        cloudlog.error("panda timed out onroad")
+
+    # Run at 2Hz, plus rising edge of ignition
+    ign_edge = started_ts is None and onroad_conditions["ignition"]
+    if (sm.frame % round(SERVICE_LIST['pandaStates'].frequency * DT_HW) != 0) and not ign_edge:
+      continue
+
+    msg = read_thermal(thermal_config)
+    msg.deviceState.deviceType = HARDWARE.get_device_type()
+
+    try:
+      last_hw_state = hw_queue.get_nowait()
+    except queue.Empty:
+      pass
+
+    msg.deviceState.freeSpacePercent = get_available_percent(default=100.0)
+    msg.deviceState.memoryUsagePercent = int(round(psutil.virtual_memory().percent))
+    msg.deviceState.gpuUsagePercent = int(round(HARDWARE.get_gpu_usage_percent()))
+    online_cpu_usage = [int(round(n)) for n in psutil.cpu_percent(percpu=True)]
+    offline_cpu_usage = [0., ] * (len(msg.deviceState.cpuTempC) - len(online_cpu_usage))
+    msg.deviceState.cpuUsagePercent = online_cpu_usage + offline_cpu_usage
+
+    msg.deviceState.networkType = last_hw_state.network_type
+    msg.deviceState.networkMetered = last_hw_state.network_metered
+    msg.deviceState.networkStrength = last_hw_state.network_strength
+    msg.deviceState.networkStats = last_hw_state.network_stats
+    if last_hw_state.network_info is not None:
+      msg.deviceState.networkInfo = last_hw_state.network_info
+
+    msg.deviceState.nvmeTempC = last_hw_state.nvme_temps
+    msg.deviceState.modemTempC = last_hw_state.modem_temps
+
+    msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness()
+
+    # this subset is only used for offroad
+    temp_sources = [
+      msg.deviceState.memoryTempC,
+      max(msg.deviceState.cpuTempC),
+      max(msg.deviceState.gpuTempC),
+    ]
+    offroad_comp_temp = offroad_temp_filter.update(max(temp_sources))
+
+    # this drives the thermal status while onroad
+    temp_sources.append(max(msg.deviceState.pmicTempC))
+    all_comp_temp = all_temp_filter.update(max(temp_sources))
+    msg.deviceState.maxTempC = all_comp_temp
+
+    if fan_controller is not None:
+      msg.deviceState.fanSpeedPercentDesired = fan_controller.update(all_comp_temp, onroad_conditions["ignition"])
+
+    is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (time.monotonic() - off_ts > 60 * 5))
+    if is_offroad_for_5_min and offroad_comp_temp > OFFROAD_DANGER_TEMP:
+      # if device is offroad and already hot without the extra onroad load,
+      # we want to cool down first before increasing load
+      thermal_status = ThermalStatus.danger
+    else:
+      current_band = THERMAL_BANDS[thermal_status]
+      band_idx = list(THERMAL_BANDS.keys()).index(thermal_status)
+      if current_band.min_temp is not None and all_comp_temp < current_band.min_temp:
+        thermal_status = list(THERMAL_BANDS.keys())[band_idx - 1]
+      elif current_band.max_temp is not None and all_comp_temp > current_band.max_temp:
+        thermal_status = list(THERMAL_BANDS.keys())[band_idx + 1]
+
+    # **** starting logic ****
+
+    startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate")
+    startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall")
+    startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version
+
+    # with 2% left, we killall, otherwise the phone will take a long time to boot
+    startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2
+    startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version
+    startup_conditions["not_driver_view"] = not params.get_bool("IsDriverViewEnabled")
+    startup_conditions["not_taking_snapshot"] = not params.get_bool("IsTakingSnapshot")
+
+    # must be at an engageable thermal band to go onroad
+    startup_conditions["device_temp_engageable"] = thermal_status < ThermalStatus.red
+
+    # ensure device is fully booted
+    startup_conditions["device_booted"] = startup_conditions.get("device_booted", False) or HARDWARE.booted()
+
+    # if the temperature enters the danger zone, go offroad to cool down
+    onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger
+    extra_text = f"{offroad_comp_temp:.1f}C"
+    show_alert = (not onroad_conditions["device_temp_good"] or not startup_conditions["device_temp_engageable"]) and onroad_conditions["ignition"]
+    set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", show_alert, extra_text=extra_text)
+
+    # TODO: this should move to TICI.initialize_hardware, but we currently can't import params there
+    if TICI:
+      if not os.path.isfile("/persist/comma/living-in-the-moment"):
+        if not Path("/data/media").is_mount():
+          set_offroad_alert_if_changed("Offroad_StorageMissing", True)
+        else:
+          # check for bad NVMe
+          try:
+            with open("/sys/block/nvme0n1/device/model") as f:
+              model = f.read().strip()
+            if not model.startswith("Samsung SSD 980") and params.get("Offroad_BadNvme") is None:
+              set_offroad_alert_if_changed("Offroad_BadNvme", True)
+              cloudlog.event("Unsupported NVMe", model=model, error=True)
+          except Exception:
+            pass
+
+    # Handle offroad/onroad transition
+    should_start = all(onroad_conditions.values())
+    if started_ts is None:
+      should_start = should_start and all(startup_conditions.values())
+
+    if should_start != should_start_prev or (count == 0):
+      params.put_bool("IsEngaged", False)
+      engaged_prev = False
+      HARDWARE.set_power_save(not should_start)
+
+    if sm.updated['controlsState']:
+      engaged = sm['controlsState'].enabled
+      if engaged != engaged_prev:
+        params.put_bool("IsEngaged", engaged)
+        engaged_prev = engaged
+
+      try:
+        with open('/dev/kmsg', 'w') as kmsg:
+          kmsg.write(f"<3>[hardware] engaged: {engaged}\n")
+      except Exception:
+        pass
+
+    if should_start:
+      off_ts = None
+      if started_ts is None:
+        started_ts = time.monotonic()
+        started_seen = True
+        if startup_blocked_ts is not None:
+          cloudlog.event("Startup after block", block_duration=(time.monotonic() - startup_blocked_ts),
+                         startup_conditions=startup_conditions, onroad_conditions=onroad_conditions,
+                         startup_conditions_prev=startup_conditions_prev, error=True)
+      startup_blocked_ts = None
+    else:
+      if onroad_conditions["ignition"] and (startup_conditions != startup_conditions_prev):
+        cloudlog.event("Startup blocked", startup_conditions=startup_conditions, onroad_conditions=onroad_conditions, error=True)
+        startup_conditions_prev = startup_conditions.copy()
+        startup_blocked_ts = time.monotonic()
+
+      started_ts = None
+      if off_ts is None:
+        off_ts = time.monotonic()
+
+    # Offroad power monitoring
+    voltage = None if peripheralState.pandaType == log.PandaState.PandaType.unknown else peripheralState.voltage
+    power_monitor.calculate(voltage, onroad_conditions["ignition"])
+    msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used()
+    msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity())
+    current_power_draw = HARDWARE.get_current_power_draw()
+    statlog.sample("power_draw", current_power_draw)
+    msg.deviceState.powerDrawW = current_power_draw
+
+    som_power_draw = HARDWARE.get_som_power_draw()
+    statlog.sample("som_power_draw", som_power_draw)
+    msg.deviceState.somPowerDrawW = som_power_draw
+
+    # Check if we need to shut down
+    if power_monitor.should_shutdown(onroad_conditions["ignition"], in_car, off_ts, started_seen):
+      cloudlog.warning(f"shutting device down, offroad since {off_ts}")
+      params.put_bool("DoShutdown", True)
+
+    msg.deviceState.started = started_ts is not None
+    msg.deviceState.startedMonoTime = int(1e9*(started_ts or 0))
+
+    last_ping = params.get("LastAthenaPingTime")
+    if last_ping is not None:
+      msg.deviceState.lastAthenaPingTime = int(last_ping)
+
+    msg.deviceState.thermalStatus = thermal_status
+    pm.send("deviceState", msg)
+
+    # Log to statsd
+    statlog.gauge("free_space_percent", msg.deviceState.freeSpacePercent)
+    statlog.gauge("gpu_usage_percent", msg.deviceState.gpuUsagePercent)
+    statlog.gauge("memory_usage_percent", msg.deviceState.memoryUsagePercent)
+    for i, usage in enumerate(msg.deviceState.cpuUsagePercent):
+      statlog.gauge(f"cpu{i}_usage_percent", usage)
+    for i, temp in enumerate(msg.deviceState.cpuTempC):
+      statlog.gauge(f"cpu{i}_temperature", temp)
+    for i, temp in enumerate(msg.deviceState.gpuTempC):
+      statlog.gauge(f"gpu{i}_temperature", temp)
+    statlog.gauge("memory_temperature", msg.deviceState.memoryTempC)
+    for i, temp in enumerate(msg.deviceState.pmicTempC):
+      statlog.gauge(f"pmic{i}_temperature", temp)
+    for i, temp in enumerate(last_hw_state.nvme_temps):
+      statlog.gauge(f"nvme_temperature{i}", temp)
+    for i, temp in enumerate(last_hw_state.modem_temps):
+      statlog.gauge(f"modem_temperature{i}", temp)
+    statlog.gauge("fan_speed_percent_desired", msg.deviceState.fanSpeedPercentDesired)
+    statlog.gauge("screen_brightness_percent", msg.deviceState.screenBrightnessPercent)
+
+    # report to server once every 10 minutes
+    rising_edge_started = should_start and not should_start_prev
+    if rising_edge_started or (count % int(600. / DT_HW)) == 0:
+      dat = {
+        'count': count,
+        'pandaStates': [strip_deprecated_keys(p.to_dict()) for p in pandaStates],
+        'peripheralState': strip_deprecated_keys(peripheralState.to_dict()),
+        'location': (strip_deprecated_keys(sm["gpsLocationExternal"].to_dict()) if sm.alive["gpsLocationExternal"] else None),
+        'deviceState': strip_deprecated_keys(msg.to_dict())
+      }
+      cloudlog.event("STATUS_PACKET", **dat)
+
+      # save last one before going onroad
+      if rising_edge_started:
+        try:
+          params.put("LastOffroadStatusPacket", json.dumps(dat))
+        except Exception:
+          cloudlog.exception("failed to save offroad status")
+
+    params.put_bool_nonblocking("NetworkMetered", msg.deviceState.networkMetered)
+
+    count += 1
+    should_start_prev = should_start
+
+
+def main():
+  hw_queue = queue.Queue(maxsize=1)
+  end_event = threading.Event()
+
+  threads = [
+    threading.Thread(target=hw_state_thread, args=(end_event, hw_queue)),
+    threading.Thread(target=hardware_thread, args=(end_event, hw_queue)),
+  ]
+
+  for t in threads:
+    t.start()
+
+  try:
+    while True:
+      time.sleep(1)
+      if not all(t.is_alive() for t in threads):
+        break
+  finally:
+    end_event.set()
+
+  for t in threads:
+    t.join()
+
+
+if __name__ == "__main__":
+  main()

system/hardware/power_monitoring.py

@@ -0,0 +1,128 @@
+import time
+import threading
+
+from openpilot.common.params import Params
+from openpilot.system.hardware import HARDWARE
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.statsd import statlog
+
+CAR_VOLTAGE_LOW_PASS_K = 0.011 # LPF gain for 45s tau (dt/tau / (dt/tau + 1))
+
+# While driving, a battery charges completely in about 30-60 minutes
+CAR_BATTERY_CAPACITY_uWh = 30e6
+CAR_CHARGING_RATE_W = 45
+
+VBATT_PAUSE_CHARGING = 11.8           # Lower limit on the LPF car battery voltage
+MAX_TIME_OFFROAD_S = 30*3600
+MIN_ON_TIME_S = 3600
+DELAY_SHUTDOWN_TIME_S = 300 # Wait at least DELAY_SHUTDOWN_TIME_S seconds after offroad_time to shutdown.
+VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 60
+
+class PowerMonitoring:
+  def __init__(self):
+    self.params = Params()
+    self.last_measurement_time = None           # Used for integration delta
+    self.last_save_time = 0                     # Used for saving current value in a param
+    self.power_used_uWh = 0                     # Integrated power usage in uWh since going into offroad
+    self.next_pulsed_measurement_time = None
+    self.car_voltage_mV = 12e3                  # Low-passed version of peripheralState voltage
+    self.car_voltage_instant_mV = 12e3          # Last value of peripheralState voltage
+    self.integration_lock = threading.Lock()
+
+    car_battery_capacity_uWh = self.params.get("CarBatteryCapacity")
+    if car_battery_capacity_uWh is None:
+      car_battery_capacity_uWh = 0
+
+    # Reset capacity if it's low
+    self.car_battery_capacity_uWh = max((CAR_BATTERY_CAPACITY_uWh / 10), int(car_battery_capacity_uWh))
+
+  # Calculation tick
+  def calculate(self, voltage: int | None, ignition: bool):
+    try:
+      now = time.monotonic()
+
+      # If peripheralState is None, we're probably not in a car, so we don't care
+      if voltage is None:
+        with self.integration_lock:
+          self.last_measurement_time = None
+          self.next_pulsed_measurement_time = None
+          self.power_used_uWh = 0
+        return
+
+      # Low-pass battery voltage
+      self.car_voltage_instant_mV = voltage
+      self.car_voltage_mV = ((voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K)))
+      statlog.gauge("car_voltage", self.car_voltage_mV / 1e3)
+
+      # Cap the car battery power and save it in a param every 10-ish seconds
+      self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0)
+      self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh)
+      if now - self.last_save_time >= 10:
+        self.params.put_nonblocking("CarBatteryCapacity", str(int(self.car_battery_capacity_uWh)))
+        self.last_save_time = now
+
+      # First measurement, set integration time
+      with self.integration_lock:
+        if self.last_measurement_time is None:
+          self.last_measurement_time = now
+          return
+
+      if ignition:
+        # If there is ignition, we integrate the charging rate of the car
+        with self.integration_lock:
+          self.power_used_uWh = 0
+          integration_time_h = (now - self.last_measurement_time) / 3600
+          if integration_time_h < 0:
+            raise ValueError(f"Negative integration time: {integration_time_h}h")
+          self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W * 1e6 * integration_time_h)
+          self.last_measurement_time = now
+      else:
+        # Get current power draw somehow
+        current_power = HARDWARE.get_current_power_draw()
+
+        # Do the integration
+        self._perform_integration(now, current_power)
+    except Exception:
+      cloudlog.exception("Power monitoring calculation failed")
+
+  def _perform_integration(self, t: float, current_power: float) -> None:
+    with self.integration_lock:
+      try:
+        if self.last_measurement_time:
+          integration_time_h = (t - self.last_measurement_time) / 3600
+          power_used = (current_power * 1000000) * integration_time_h
+          if power_used < 0:
+            raise ValueError(f"Negative power used! Integration time: {integration_time_h} h Current Power: {power_used} uWh")
+          self.power_used_uWh += power_used
+          self.car_battery_capacity_uWh -= power_used
+          self.last_measurement_time = t
+      except Exception:
+        cloudlog.exception("Integration failed")
+
+  # Get the power usage
+  def get_power_used(self) -> int:
+    return int(self.power_used_uWh)
+
+  def get_car_battery_capacity(self) -> int:
+    return int(self.car_battery_capacity_uWh)
+
+  # See if we need to shutdown
+  def should_shutdown(self, ignition: bool, in_car: bool, offroad_timestamp: float | None, started_seen: bool):
+    if offroad_timestamp is None:
+      return False
+
+    now = time.monotonic()
+    should_shutdown = False
+    offroad_time = (now - offroad_timestamp)
+    low_voltage_shutdown = (self.car_voltage_mV < (VBATT_PAUSE_CHARGING * 1e3) and
+                            offroad_time > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S)
+    should_shutdown |= offroad_time > MAX_TIME_OFFROAD_S
+    should_shutdown |= low_voltage_shutdown
+    should_shutdown |= (self.car_battery_capacity_uWh <= 0)
+    should_shutdown &= not ignition
+    should_shutdown &= (not self.params.get_bool("DisablePowerDown"))
+    should_shutdown &= in_car
+    should_shutdown &= offroad_time > DELAY_SHUTDOWN_TIME_S
+    should_shutdown |= self.params.get_bool("ForcePowerDown")
+    should_shutdown &= started_seen or (now > MIN_ON_TIME_S)
+    return should_shutdown

system/hardware/tests/test_fan_controller.py

@@ -0,0 +1,50 @@
+import pytest
+
+from openpilot.system.hardware.fan_controller import TiciFanController
+
+ALL_CONTROLLERS = [TiciFanController]
+
+def patched_controller(mocker, controller_class):
+  mocker.patch("os.system", new=mocker.Mock())
+  return controller_class()
+
+class TestFanController:
+  def wind_up(self, controller, ignition=True):
+    for _ in range(1000):
+      controller.update(100, ignition)
+
+  def wind_down(self, controller, ignition=False):
+    for _ in range(1000):
+      controller.update(10, ignition)
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_hot_onroad(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_up(controller)
+    assert controller.update(100, True) >= 70
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_offroad_limits(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_up(controller)
+    assert controller.update(100, False) <= 30
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_no_fan_wear(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_down(controller)
+    assert controller.update(10, False) == 0
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_limited(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_up(controller, True)
+    assert controller.update(100, True) == 100
+
+  @pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
+  def test_windup_speed(self, mocker, controller_class):
+    controller = patched_controller(mocker, controller_class)
+    self.wind_down(controller, True)
+    for _ in range(10):
+      controller.update(90, True)
+    assert controller.update(90, True) >= 60

system/hardware/tests/test_power_monitoring.py

@@ -0,0 +1,199 @@
+import pytest
+
+from openpilot.common.params import Params
+from openpilot.system.hardware.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \
+                                                CAR_CHARGING_RATE_W, VBATT_PAUSE_CHARGING, DELAY_SHUTDOWN_TIME_S
+
+# Create fake time
+ssb = 0.
+def mock_time_monotonic():
+  global ssb
+  ssb += 1.
+  return ssb
+
+TEST_DURATION_S = 50
+GOOD_VOLTAGE = 12 * 1e3
+VOLTAGE_BELOW_PAUSE_CHARGING = (VBATT_PAUSE_CHARGING - 1) * 1e3
+
+def pm_patch(mocker, name, value, constant=False):
+  if constant:
+    mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", value)
+  else:
+    mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", return_value=value)
+
+
+@pytest.fixture(autouse=True)
+def mock_time(mocker):
+  mocker.patch("time.monotonic", mock_time_monotonic)
+
+
+class TestPowerMonitoring:
+  def setup_method(self):
+    self.params = Params()
+
+  # Test to see that it doesn't do anything when pandaState is None
+  def test_panda_state_present(self):
+    pm = PowerMonitoring()
+    for _ in range(10):
+      pm.calculate(None, None)
+    assert pm.get_power_used() == 0
+    assert pm.get_car_battery_capacity() == (CAR_BATTERY_CAPACITY_uWh / 10)
+
+  # Test to see that it doesn't integrate offroad when ignition is True
+  def test_offroad_ignition(self):
+    pm = PowerMonitoring()
+    for _ in range(10):
+      pm.calculate(GOOD_VOLTAGE, True)
+    assert pm.get_power_used() == 0
+
+  # Test to see that it integrates with discharging battery
+  def test_offroad_integration_discharging(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    expected_power_usage = ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_power_used() - expected_power_usage) < 10
+
+  # Test to check positive integration of car_battery_capacity
+  def test_car_battery_integration_onroad(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 0
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, True)
+    expected_capacity = ((TEST_DURATION_S/3600) * CAR_CHARGING_RATE_W * 1e6)
+    assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
+
+  # Test to check positive integration upper limit
+  def test_car_battery_integration_upper_limit(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh - 1000
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, True)
+    estimated_capacity = CAR_BATTERY_CAPACITY_uWh + (CAR_CHARGING_RATE_W / 3600 * 1e6)
+    assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
+
+  # Test to check negative integration of car_battery_capacity
+  def test_car_battery_integration_offroad(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    expected_capacity = CAR_BATTERY_CAPACITY_uWh - ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
+
+  # Test to check negative integration lower limit
+  def test_car_battery_integration_lower_limit(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 1000
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    estimated_capacity = 0 - ((1/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
+
+  # Test to check policy of stopping charging after MAX_TIME_OFFROAD_S
+  def test_max_time_offroad(self, mocker):
+    MOCKED_MAX_OFFROAD_TIME = 3600
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    pm_patch(mocker, "MAX_TIME_OFFROAD_S", MOCKED_MAX_OFFROAD_TIME, constant=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    start_time = ssb
+    ignition = False
+    while ssb <= start_time + MOCKED_MAX_OFFROAD_TIME:
+      pm.calculate(GOOD_VOLTAGE, ignition)
+      if (ssb - start_time) % 1000 == 0 and ssb < start_time + MOCKED_MAX_OFFROAD_TIME:
+        assert not pm.should_shutdown(ignition, True, start_time, False)
+    assert pm.should_shutdown(ignition, True, start_time, False)
+
+  def test_car_voltage(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 350
+    VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 50
+    pm_patch(mocker, "VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S", VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S, constant=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = False
+    start_time = ssb
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert pm.should_shutdown(ignition, True, start_time, True) == \
+                          (pm.car_voltage_mV < VBATT_PAUSE_CHARGING * 1e3 and \
+                          (ssb - start_time) > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S and \
+                            (ssb - start_time) > DELAY_SHUTDOWN_TIME_S)
+    assert pm.should_shutdown(ignition, True, start_time, True)
+
+  # Test to check policy of not stopping charging when DisablePowerDown is set
+  def test_disable_power_down(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    self.params.put_bool("DisablePowerDown", True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = False
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, True, ssb, False)
+    assert not pm.should_shutdown(ignition, True, ssb, False)
+
+  # Test to check policy of not stopping charging when ignition
+  def test_ignition(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = True
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, True, ssb, False)
+    assert not pm.should_shutdown(ignition, True, ssb, False)
+
+  # Test to check policy of not stopping charging when harness is not connected
+  def test_harness_connection(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+
+    ignition = False
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, False, ssb, False)
+    assert not pm.should_shutdown(ignition, False, ssb, False)
+
+  def test_delay_shutdown_time(self):
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 0
+    ignition = False
+    in_car = True
+    offroad_timestamp = ssb
+    started_seen = True
+    pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+
+    while ssb < offroad_timestamp + DELAY_SHUTDOWN_TIME_S:
+      assert not pm.should_shutdown(ignition, in_car,
+                                          offroad_timestamp,
+                                          started_seen), \
+                       f"Should not shutdown before {DELAY_SHUTDOWN_TIME_S} seconds offroad time"
+    assert pm.should_shutdown(ignition, in_car,
+                                       offroad_timestamp,
+                                       started_seen), \
+                    f"Should shutdown after {DELAY_SHUTDOWN_TIME_S} seconds offroad time"

system/hardware/tici/agnos.json

@@ -1,10 +1,10 @@
 [
   {
     "name": "boot",
-    "url": "https://commadist.azureedge.net/agnosupdate/boot-f0de74e139b8b99224738d4e72a5b1831758f20b09ff6bb28f3aaaae1c4c1ebe.img.xz",
-    "hash": "f0de74e139b8b99224738d4e72a5b1831758f20b09ff6bb28f3aaaae1c4c1ebe",
-    "hash_raw": "f0de74e139b8b99224738d4e72a5b1831758f20b09ff6bb28f3aaaae1c4c1ebe",
-    "size": 15636480,
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-5674ea6767e7198cf1e7def3de66a57061f001ed76d43dc4b4f84de545c53c6f.img.xz",
+    "hash": "5674ea6767e7198cf1e7def3de66a57061f001ed76d43dc4b4f84de545c53c6f",
+    "hash_raw": "5674ea6767e7198cf1e7def3de66a57061f001ed76d43dc4b4f84de545c53c6f",
+    "size": 16029696,
     "sparse": false,
     "full_check": true,
     "has_ab": true
@@ -61,17 +61,17 @@
   },
   {
     "name": "system",
-    "url": "https://commadist.azureedge.net/agnosupdate/system-0f69173d5f3058f7197c139442a6556be59e52f15402a263215a329ba5ec41e2.img.xz",
-    "hash": "4858385ba6284bcaa179ab77ac4263486e4d8670df921e4ac400464dc1dde59c",
-    "hash_raw": "0f69173d5f3058f7197c139442a6556be59e52f15402a263215a329ba5ec41e2",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-1badfe72851628d6cf9200a53a6151bb4e797b49c717141409fc57138eae388a.img.xz",
+    "hash": "328e90c62068222dfd98f71dd3f6251fcb962f082b49c6be66ab2699f5db6f4f",
+    "hash_raw": "1badfe72851628d6cf9200a53a6151bb4e797b49c717141409fc57138eae388a",
     "size": 10737418240,
     "sparse": true,
     "full_check": false,
     "has_ab": true,
     "alt": {
-      "hash": "42658a6fff660d9b6abb9cb9fbb3481071259c9a9598718af6b1edff2b556009",
-      "url": "https://commadist.azureedge.net/agnosupdate/system-skip-chunks-0f69173d5f3058f7197c139442a6556be59e52f15402a263215a329ba5ec41e2.img.xz",
-      "size": 4548292756
+      "hash": "bc11d2148f29862ee1326aca2af1cf6bbf5fed831e3f8f6b8f7a0f110dfe8d26",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-skip-chunks-1badfe72851628d6cf9200a53a6151bb4e797b49c717141409fc57138eae388a.img.xz",
+      "size": 4548070000
     }
   }
 ]

system/hardware/tici/agnos.py

@@ -6,15 +6,17 @@ import os
 import struct
 import subprocess
 import time
-from typing import Dict, Generator, List, Tuple, Union
+from collections.abc import Generator
 
 import requests
 
-import openpilot.system.hardware.tici.casync as casync
+import openpilot.system.updated.casync.casync as casync
 
 SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
 CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/"
 
+AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json"
+
 
 class StreamingDecompressor:
   def __init__(self, url: str) -> None:
@@ -117,7 +119,7 @@ def get_raw_hash(path: str, partition_size: int) -> str:
   return raw_hash.hexdigest().lower()
 
 
-def verify_partition(target_slot_number: int, partition: Dict[str, Union[str, int]], force_full_check: bool = False) -> bool:
+def verify_partition(target_slot_number: int, partition: dict[str, str | int], force_full_check: bool = False) -> bool:
   full_check = partition['full_check'] or force_full_check
   path = get_partition_path(target_slot_number, partition)
 
@@ -184,7 +186,7 @@ def extract_casync_image(target_slot_number: int, partition: dict, cloudlog):
 
   target = casync.parse_caibx(partition['casync_caibx'])
 
-  sources: List[Tuple[str, casync.ChunkReader, casync.ChunkDict]] = []
+  sources: list[tuple[str, casync.ChunkReader, casync.ChunkDict]] = []
 
   # First source is the current partition.
   try:

system/hardware/tici/amplifier.py

@@ -2,7 +2,6 @@
 import time
 from smbus2 import SMBus
 from collections import namedtuple
-from typing import List
 
 # https://datasheets.maximintegrated.com/en/ds/MAX98089.pdf
 
@@ -110,7 +109,7 @@ class Amplifier:
   def _get_shutdown_config(self, amp_disabled: bool) -> AmpConfig:
     return AmpConfig("Global shutdown", 0b0 if amp_disabled else 0b1, 0x51, 7, 0b10000000)
 
-  def _set_configs(self, configs: List[AmpConfig]) -> None:
+  def _set_configs(self, configs: list[AmpConfig]) -> None:
     with SMBus(self.AMP_I2C_BUS) as bus:
       for config in configs:
         if self.debug:
@@ -123,7 +122,7 @@ class Amplifier:
         if self.debug:
           print(f"  Changed {hex(config.register)}: {hex(old_value)} -> {hex(new_value)}")
 
-  def set_configs(self, configs: List[AmpConfig]) -> bool:
+  def set_configs(self, configs: list[AmpConfig]) -> bool:
     # retry in case panda is using the amp
     tries = 15
     for i in range(15):

system/hardware/tici/casync.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-import io
-import lzma
-import os
-import struct
-import sys
-import time
-from abc import ABC, abstractmethod
-from collections import defaultdict, namedtuple
-from typing import Callable, Dict, List, Optional, Tuple
-
-import requests
-from Crypto.Hash import SHA512
-
-CA_FORMAT_INDEX = 0x96824d9c7b129ff9
-CA_FORMAT_TABLE = 0xe75b9e112f17417d
-CA_FORMAT_TABLE_TAIL_MARKER = 0xe75b9e112f17417
-FLAGS = 0xb000000000000000
-
-CA_HEADER_LEN = 48
-CA_TABLE_HEADER_LEN = 16
-CA_TABLE_ENTRY_LEN = 40
-CA_TABLE_MIN_LEN = CA_TABLE_HEADER_LEN + CA_TABLE_ENTRY_LEN
-
-CHUNK_DOWNLOAD_TIMEOUT = 60
-CHUNK_DOWNLOAD_RETRIES = 3
-
-CAIBX_DOWNLOAD_TIMEOUT = 120
-
-Chunk = namedtuple('Chunk', ['sha', 'offset', 'length'])
-ChunkDict = Dict[bytes, Chunk]
-
-
-class ChunkReader(ABC):
-  @abstractmethod
-  def read(self, chunk: Chunk) -> bytes:
-    ...
-
-
-class FileChunkReader(ChunkReader):
-  """Reads chunks from a local file"""
-  def __init__(self, fn: str) -> None:
-    super().__init__()
-    self.f = open(fn, 'rb')
-
-  def __del__(self):
-    self.f.close()
-
-  def read(self, chunk: Chunk) -> bytes:
-    self.f.seek(chunk.offset)
-    return self.f.read(chunk.length)
-
-
-class RemoteChunkReader(ChunkReader):
-  """Reads lzma compressed chunks from a remote store"""
-
-  def __init__(self, url: str) -> None:
-    super().__init__()
-    self.url = url
-    self.session = requests.Session()
-
-  def read(self, chunk: Chunk) -> bytes:
-    sha_hex = chunk.sha.hex()
-    url = os.path.join(self.url, sha_hex[:4], sha_hex + ".cacnk")
-
-    if os.path.isfile(url):
-      with open(url, 'rb') as f:
-        contents = f.read()
-    else:
-      for i in range(CHUNK_DOWNLOAD_RETRIES):
-        try:
-          resp = self.session.get(url, timeout=CHUNK_DOWNLOAD_TIMEOUT)
-          break
-        except Exception:
-          if i == CHUNK_DOWNLOAD_RETRIES - 1:
-            raise
-          time.sleep(CHUNK_DOWNLOAD_TIMEOUT)
-
-      resp.raise_for_status()
-      contents = resp.content
-
-    decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
-    return decompressor.decompress(contents)
-
-
-def parse_caibx(caibx_path: str) -> List[Chunk]:
-  """Parses the chunks from a caibx file. Can handle both local and remote files.
-  Returns a list of chunks with hash, offset and length"""
-  caibx: io.BufferedIOBase
-  if os.path.isfile(caibx_path):
-    caibx = open(caibx_path, 'rb')
-  else:
-    resp = requests.get(caibx_path, timeout=CAIBX_DOWNLOAD_TIMEOUT)
-    resp.raise_for_status()
-    caibx = io.BytesIO(resp.content)
-
-  caibx.seek(0, os.SEEK_END)
-  caibx_len = caibx.tell()
-  caibx.seek(0, os.SEEK_SET)
-
-  # Parse header
-  length, magic, flags, min_size, _, max_size = struct.unpack("<QQQQQQ", caibx.read(CA_HEADER_LEN))
-  assert flags == flags
-  assert length == CA_HEADER_LEN
-  assert magic == CA_FORMAT_INDEX
-
-  # Parse table header
-  length, magic = struct.unpack("<QQ", caibx.read(CA_TABLE_HEADER_LEN))
-  assert magic == CA_FORMAT_TABLE
-
-  # Parse chunks
-  num_chunks = (caibx_len - CA_HEADER_LEN - CA_TABLE_MIN_LEN) // CA_TABLE_ENTRY_LEN
-  chunks = []
-
-  offset = 0
-  for i in range(num_chunks):
-    new_offset = struct.unpack("<Q", caibx.read(8))[0]
-
-    sha = caibx.read(32)
-    length = new_offset - offset
-
-    assert length <= max_size
-
-    # Last chunk can be smaller
-    if i < num_chunks - 1:
-      assert length >= min_size
-
-    chunks.append(Chunk(sha, offset, length))
-    offset = new_offset
-
-  caibx.close()
-  return chunks
-
-
-def build_chunk_dict(chunks: List[Chunk]) -> ChunkDict:
-  """Turn a list of chunks into a dict for faster lookups based on hash.
-  Keep first chunk since it's more likely to be already downloaded."""
-  r = {}
-  for c in chunks:
-    if c.sha not in r:
-      r[c.sha] = c
-  return r
-
-
-def extract(target: List[Chunk],
-            sources: List[Tuple[str, ChunkReader, ChunkDict]],
-            out_path: str,
-            progress: Optional[Callable[[int], None]] = None):
-  stats: Dict[str, int] = defaultdict(int)
-
-  mode = 'rb+' if os.path.exists(out_path) else 'wb'
-  with open(out_path, mode) as out:
-    for cur_chunk in target:
-
-      # Find source for desired chunk
-      for name, chunk_reader, store_chunks in sources:
-        if cur_chunk.sha in store_chunks:
-          bts = chunk_reader.read(store_chunks[cur_chunk.sha])
-
-          # Check length
-          if len(bts) != cur_chunk.length:
-            continue
-
-          # Check hash
-          if SHA512.new(bts, truncate="256").digest() != cur_chunk.sha:
-            continue
-
-          # Write to output
-          out.seek(cur_chunk.offset)
-          out.write(bts)
-
-          stats[name] += cur_chunk.length
-
-          if progress is not None:
-            progress(sum(stats.values()))
-
-          break
-      else:
-        raise RuntimeError("Desired chunk not found in provided stores")
-
-  return stats
-
-
-def print_stats(stats: Dict[str, int]):
-  total_bytes = sum(stats.values())
-  print(f"Total size: {total_bytes / 1024 / 1024:.2f} MB")
-  for name, total in stats.items():
-    print(f"  {name}: {total / 1024 / 1024:.2f} MB ({total / total_bytes * 100:.1f}%)")
-
-
-def extract_simple(caibx_path, out_path, store_path):
-  # (name, callback, chunks)
-  target = parse_caibx(caibx_path)
-  sources = [
-    # (store_path, RemoteChunkReader(store_path), build_chunk_dict(target)),
-    (store_path, FileChunkReader(store_path), build_chunk_dict(target)),
-  ]
-
-  return extract(target, sources, out_path)
-
-
-if __name__ == "__main__":
-  caibx = sys.argv[1]
-  out = sys.argv[2]
-  store = sys.argv[3]
-
-  stats = extract_simple(caibx, out, store)
-  print_stats(stats)

system/hardware/tici/esim.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+import os
+import math
+import time
+import binascii
+import requests
+import serial
+import subprocess
+
+
+def post(url, payload):
+  print()
+  print("POST to", url)
+  r = requests.post(
+    url,
+    data=payload,
+    verify=False,
+    headers={
+      "Content-Type": "application/json",
+      "X-Admin-Protocol": "gsma/rsp/v2.2.0",
+      "charset": "utf-8",
+      "User-Agent": "gsma-rsp-lpad",
+    },
+  )
+  print("resp", r)
+  print("resp text", repr(r.text))
+  print()
+  r.raise_for_status()
+
+  ret = f"HTTP/1.1 {r.status_code}"
+  ret += ''.join(f"{k}: {v}" for k, v in r.headers.items() if k != 'Connection')
+  return ret.encode() + r.content
+
+
+class LPA:
+  def __init__(self):
+    self.dev = serial.Serial('/dev/ttyUSB2', baudrate=57600, timeout=1, bytesize=8)
+    self.dev.reset_input_buffer()
+    self.dev.reset_output_buffer()
+    assert "OK" in self.at("AT")
+
+  def at(self, cmd):
+    print(f"==> {cmd}")
+    self.dev.write(cmd.encode() + b'\r\n')
+
+    r = b""
+    cnt = 0
+    while b"OK" not in r and b"ERROR" not in r and cnt < 20:
+      r += self.dev.read(8192).strip()
+      cnt += 1
+    r = r.decode()
+    print(f"<== {repr(r)}")
+    return r
+
+  def download_ota(self, qr):
+    return self.at(f'AT+QESIM="ota","{qr}"')
+
+  def download(self, qr):
+    smdp = qr.split('$')[1]
+    out = self.at(f'AT+QESIM="download","{qr}"')
+    for _ in range(5):
+      line = out.split("+QESIM: ")[1].split("\r\n\r\nOK")[0]
+
+      parts = [x.strip().strip('"') for x in line.split(',', maxsplit=4)]
+      print(repr(parts))
+      trans, ret, url, payloadlen, payload = parts
+      assert trans == "trans" and ret == "0"
+      assert len(payload) == int(payloadlen)
+
+      r = post(f"https://{smdp}/{url}", payload)
+      to_send = binascii.hexlify(r).decode()
+
+      chunk_len = 1400
+      for i in range(math.ceil(len(to_send) / chunk_len)):
+        state = 1 if (i+1)*chunk_len < len(to_send) else 0
+        data = to_send[i * chunk_len : (i+1)*chunk_len]
+        out = self.at(f'AT+QESIM="trans",{len(to_send)},{state},{i},{len(data)},"{data}"')
+        assert "OK" in out
+
+      if '+QESIM:"download",1' in out:
+        raise Exception("profile install failed")
+      elif '+QESIM:"download",0' in out:
+        print("done, successfully loaded")
+        break
+
+  def enable(self, iccid):
+    self.at(f'AT+QESIM="enable","{iccid}"')
+
+  def disable(self, iccid):
+    self.at(f'AT+QESIM="disable","{iccid}"')
+
+  def delete(self, iccid):
+    self.at(f'AT+QESIM="delete","{iccid}"')
+
+  def list_profiles(self):
+    out = self.at('AT+QESIM="list"')
+    return out.strip().splitlines()[1:]
+
+
+if __name__ == "__main__":
+  import sys
+
+  if "RESTART" in os.environ:
+    subprocess.check_call("sudo systemctl stop ModemManager", shell=True)
+    subprocess.check_call("/usr/comma/lte/lte.sh stop_blocking", shell=True)
+    subprocess.check_call("/usr/comma/lte/lte.sh start", shell=True)
+    while not os.path.exists('/dev/ttyUSB2'):
+      time.sleep(1)
+    time.sleep(3)
+
+  lpa = LPA()
+  print(lpa.list_profiles())
+  if len(sys.argv) > 1:
+    lpa.download(sys.argv[1])
+    print(lpa.list_profiles())

system/hardware/tici/hardware.h

@@ -20,12 +20,12 @@ public:
   }
 
   static std::string get_name() {
-    std::string devicetree_model = util::read_file("/sys/firmware/devicetree/base/model");
-    return (devicetree_model.find("tizi") != std::string::npos) ? "tizi" : "tici";
+    std::string model = util::read_file("/sys/firmware/devicetree/base/model");
+    return model.substr(std::string("comma ").size());
   }
 
   static cereal::InitData::DeviceType get_device_type() {
-    return (get_name() == "tizi") ? cereal::InitData::DeviceType::TIZI : cereal::InitData::DeviceType::TICI;
+    return (get_name() == "tizi") ? cereal::InitData::DeviceType::TIZI : (get_name() == "mici" ? cereal::InitData::DeviceType::MICI : cereal::InitData::DeviceType::TICI);
   }
 
   static int get_voltage() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/in1_input").c_str()); }

system/hardware/tici/hardware.py

@@ -94,11 +94,7 @@ def get_device_type():
   # lru_cache and cache can cause memory leaks when used in classes
   with open("/sys/firmware/devicetree/base/model") as f:
     model = f.read().strip('\x00')
-  model = model.split('comma ')[-1]
-  # TODO: remove this with AGNOS 7+
-  if model.startswith('Qualcomm'):
-    model = 'tici'
-  return model
+  return model.split('comma ')[-1]
 
 class Tici(HardwareBase):
   @cached_property
@@ -116,6 +112,8 @@ class Tici(HardwareBase):
 
   @cached_property
   def amplifier(self):
+    if self.get_device_type() == "mici":
+      return None
     return Amplifier()
 
   def get_os_version(self):
@@ -374,29 +372,26 @@ class Tici(HardwareBase):
 
   def set_power_save(self, powersave_enabled):
     # amplifier, 100mW at idle
-    self.amplifier.set_global_shutdown(amp_disabled=powersave_enabled)
-    if not powersave_enabled:
-      self.amplifier.initialize_configuration(self.get_device_type())
+    if self.amplifier is not None:
+      self.amplifier.set_global_shutdown(amp_disabled=powersave_enabled)
+      if not powersave_enabled:
+        self.amplifier.initialize_configuration(self.get_device_type())
 
     # *** CPU config ***
 
-    # offline big cluster, leave core 4 online for boardd
-    for i in range(5, 8):
+    # offline big cluster, leave core 4 online for pandad
+    for i in range(4, 8):
       val = '0' if powersave_enabled else '1'
       sudo_write(val, f'/sys/devices/system/cpu/cpu{i}/online')
 
     for n in ('0', '4'):
+      if powersave_enabled and n == '4':
+        continue
       gov = 'ondemand' if powersave_enabled else 'performance'
       sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
 
     # *** IRQ config ***
 
-    # boardd core
-    affine_irq(4, "spi_geni")         # SPI
-    affine_irq(4, "xhci-hcd:usb3")    # aux panda USB (or potentially anything else on USB)
-    if "tici" in self.get_device_type():
-      affine_irq(4, "xhci-hcd:usb1")  # internal panda USB (also modem)
-
     # GPU
     affine_irq(5, "kgsl-3d0")
 
@@ -414,9 +409,10 @@ class Tici(HardwareBase):
       return 0
 
   def initialize_hardware(self):
-    self.amplifier.initialize_configuration(self.get_device_type())
+    if self.amplifier is not None:
+      self.amplifier.initialize_configuration(self.get_device_type())
 
-    # Allow thermald to write engagement status to kmsg
+    # Allow hardwared to write engagement status to kmsg
     os.system("sudo chmod a+w /dev/kmsg")
 
     # Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
@@ -453,6 +449,18 @@ class Tici(HardwareBase):
     sudo_write("N", "/sys/kernel/debug/msm_vidc/clock_scaling")
     sudo_write("Y", "/sys/kernel/debug/msm_vidc/disable_thermal_mitigation")
 
+    # pandad core
+    affine_irq(3, "spi_geni")         # SPI
+    if "tici" in self.get_device_type():
+      affine_irq(3, "xhci-hcd:usb3")  # aux panda USB (or potentially anything else on USB)
+      affine_irq(3, "xhci-hcd:usb1")  # internal panda USB (also modem)
+    try:
+      pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
+      subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
+      subprocess.call(["sudo", "taskset", "-pc", "3", pid])
+    except subprocess.CalledProcessException as e:
+      print(str(e))
+
   def configure_modem(self):
     sim_id = self.get_sim_info().get('sim_id', '')
 
@@ -468,8 +476,9 @@ class Tici(HardwareBase):
         # use sim slot
         'AT^SIMSWAP=1',
 
-        # configure ECM mode
-        'AT$QCPCFG=usbNet,1'
+        # ethernet config
+        'AT$QCPCFG=usbNet,0',
+        'AT$QCNETDEVCTL=3,1',
       ]
     else:
       cmds += [
@@ -478,6 +487,12 @@ class Tici(HardwareBase):
         'AT+QNVFW="/nv/item_files/ims/IMS_enable",00',
         'AT+QNVFW="/nv/item_files/modem/mmode/ue_usage_setting",01',
       ]
+      if self.get_device_type() == "tizi":
+        cmds += [
+          # SIM hot swap
+          'AT+QSIMDET=1,0',
+          'AT+QSIMSTAT=1',
+        ]
 
       # clear out old blue prime initial APN
       os.system('mmcli -m any --3gpp-set-initial-eps-bearer-settings="apn="')

system/hardware/tici/power_monitor.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import sys
+import time
+import datetime
+import numpy as np
+from collections import deque
+
+from openpilot.common.realtime import Ratekeeper
+from openpilot.common.filter_simple import FirstOrderFilter
+
+
+def read_power():
+  with open("/sys/bus/i2c/devices/0-0040/hwmon/hwmon1/power1_input") as f:
+    return int(f.read()) / 1e6
+
+def sample_power(seconds=5) -> list[float]:
+  rate = 123
+  rk = Ratekeeper(rate, print_delay_threshold=None)
+
+  pwrs = []
+  for _ in range(rate*seconds):
+    pwrs.append(read_power())
+    rk.keep_time()
+  return pwrs
+
+def get_power(seconds=5):
+  pwrs = sample_power(seconds)
+  return np.mean(pwrs)
+
+def wait_for_power(min_pwr, max_pwr, min_secs_in_range, timeout):
+  start_time = time.monotonic()
+  pwrs = deque([min_pwr - 1.]*min_secs_in_range, maxlen=min_secs_in_range)
+  while (time.monotonic() - start_time < timeout):
+    pwrs.append(get_power(1))
+    if all(min_pwr <= p <= max_pwr for p in pwrs):
+      break
+  return np.mean(pwrs)
+
+
+if __name__ == "__main__":
+  duration = None
+  if len(sys.argv) > 1:
+    duration = int(sys.argv[1])
+
+  rate = 23
+  rk = Ratekeeper(rate, print_delay_threshold=None)
+  fltr = FirstOrderFilter(0, 5, 1. / rate, initialized=False)
+
+  measurements = []
+  start_time = time.monotonic()
+
+  try:
+    while duration is None or time.monotonic() - start_time < duration:
+      fltr.update(read_power())
+      if rk.frame % rate == 0:
+        measurements.append(fltr.x)
+        t = datetime.timedelta(seconds=time.monotonic() - start_time)
+        avg = sum(measurements) / len(measurements)
+        print(f"Now: {fltr.x:.2f} W, Avg: {avg:.2f} W over {t}")
+      rk.keep_time()
+  except KeyboardInterrupt:
+    pass
+
+  t = datetime.timedelta(seconds=time.monotonic() - start_time)
+  avg = sum(measurements) / len(measurements)
+  print(f"\nAverage power: {avg:.2f}W over {t}")

system/hardware/tici/precise_power_measure.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+import numpy as np
+from openpilot.system.hardware.tici.power_monitor import sample_power
+
+if __name__ == '__main__':
+  print("measuring for 5 seconds")
+  for _ in range(3):
+    pwrs = sample_power()
+    print(f"mean {np.mean(pwrs):.2f} std {np.std(pwrs):.2f}")

system/hardware/tici/restart_modem.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/bash
+
+#nmcli connection modify --temporary lte gsm.home-only yes
+#nmcli connection modify --temporary lte gsm.auto-config yes
+#nmcli connection modify --temporary lte connection.autoconnect-retries 20
+sudo nmcli connection reload
+
+sudo systemctl stop ModemManager
+nmcli con down lte
+nmcli con down blue-prime
+
+# power cycle modem
+/usr/comma/lte/lte.sh stop_blocking
+/usr/comma/lte/lte.sh start
+
+sudo systemctl restart NetworkManager
+#sudo systemctl restart ModemManager
+sudo ModemManager --debug

system/hardware/tici/tests/compare_casync_manifest.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+import argparse
+import collections
+import multiprocessing
+import os
+
+import requests
+from tqdm import tqdm
+
+import openpilot.system.hardware.tici.casync as casync
+
+
+def get_chunk_download_size(chunk):
+  sha = chunk.sha.hex()
+  path = os.path.join(remote_url, sha[:4], sha + ".cacnk")
+  if os.path.isfile(path):
+    return os.path.getsize(path)
+  else:
+    r = requests.head(path, timeout=10)
+    r.raise_for_status()
+    return int(r.headers['content-length'])
+
+
+if __name__ == "__main__":
+
+  parser = argparse.ArgumentParser(description='Compute overlap between two casync manifests')
+  parser.add_argument('frm')
+  parser.add_argument('to')
+  args = parser.parse_args()
+
+  frm = casync.parse_caibx(args.frm)
+  to = casync.parse_caibx(args.to)
+  remote_url = args.to.replace('.caibx', '')
+
+  most_common = collections.Counter(t.sha for t in to).most_common(1)[0][0]
+
+  frm_dict = casync.build_chunk_dict(frm)
+
+  # Get content-length for each chunk
+  with multiprocessing.Pool() as pool:
+    szs = list(tqdm(pool.imap(get_chunk_download_size, to), total=len(to)))
+  chunk_sizes = {t.sha: sz for (t, sz) in zip(to, szs, strict=True)}
+
+  sources: dict[str, list[int]] = {
+    'seed': [],
+    'remote_uncompressed': [],
+    'remote_compressed': [],
+  }
+
+  for chunk in to:
+    # Assume most common chunk is the zero chunk
+    if chunk.sha == most_common:
+      continue
+
+    if chunk.sha in frm_dict:
+      sources['seed'].append(chunk.length)
+    else:
+      sources['remote_uncompressed'].append(chunk.length)
+      sources['remote_compressed'].append(chunk_sizes[chunk.sha])
+
+  print()
+  print("Update statistics (excluding zeros)")
+  print()
+  print("Download only with no seed:")
+  print(f"  Remote (uncompressed)\t\t{sum(sources['seed'] + sources['remote_uncompressed']) / 1000 / 1000:.2f} MB\tn = {len(to)}")
+  print(f"  Remote (compressed download)\t{sum(chunk_sizes.values()) / 1000 / 1000:.2f} MB\tn = {len(to)}")
+  print()
+  print("Upgrade with seed partition:")
+  print(f"  Seed   (uncompressed)\t\t{sum(sources['seed']) / 1000 / 1000:.2f} MB\t\t\t\tn = {len(sources['seed'])}")
+  sz, n = sum(sources['remote_uncompressed']), len(sources['remote_uncompressed'])
+  print(f"  Remote (uncompressed)\t\t{sz / 1000 / 1000:.2f} MB\t(avg {sz / 1000 / 1000 / n:4f} MB)\tn = {n}")
+  sz, n = sum(sources['remote_compressed']), len(sources['remote_compressed'])
+  print(f"  Remote (compressed download)\t{sz / 1000 / 1000:.2f} MB\t(avg {sz / 1000 / 1000 / n:4f} MB)\tn = {n}")

system/hardware/tici/tests/test_agnos_updater.py

@@ -0,0 +1,20 @@
+import json
+import os
+import requests
+
+TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
+
+
+class TestAgnosUpdater:
+
+  def test_manifest(self):
+    with open(MANIFEST) as f:
+      m = json.load(f)
+
+    for img in m:
+      r = requests.head(img['url'], timeout=10)
+      r.raise_for_status()
+      assert r.headers['Content-Type'] == "application/x-xz"
+      if not img['sparse']:
+        assert img['hash'] == img['hash_raw']

system/hardware/tici/tests/test_amplifier.py

@@ -0,0 +1,70 @@
+import pytest
+import time
+import random
+import subprocess
+
+from panda import Panda
+from openpilot.system.hardware import TICI, HARDWARE
+from openpilot.system.hardware.tici.hardware import Tici
+from openpilot.system.hardware.tici.amplifier import Amplifier
+
+
+class TestAmplifier:
+
+  @classmethod
+  def setup_class(cls):
+    if not TICI:
+      pytest.skip()
+
+  def setup_method(self):
+    # clear dmesg
+    subprocess.check_call("sudo dmesg -C", shell=True)
+
+    HARDWARE.reset_internal_panda()
+    Panda.wait_for_panda(None, 30)
+    self.panda = Panda()
+
+  def teardown_method(self):
+    HARDWARE.reset_internal_panda()
+
+  def _check_for_i2c_errors(self, expected):
+    dmesg = subprocess.check_output("dmesg", shell=True, encoding='utf8')
+    i2c_lines = [l for l in dmesg.strip().splitlines() if 'i2c_geni a88000.i2c' in l]
+    i2c_str = '\n'.join(i2c_lines)
+
+    if not expected:
+      return len(i2c_lines) == 0
+    else:
+      return "i2c error :-107" in i2c_str or "Bus arbitration lost" in i2c_str
+
+  def test_init(self):
+    amp = Amplifier(debug=True)
+    r = amp.initialize_configuration(Tici().get_device_type())
+    assert r
+    assert self._check_for_i2c_errors(False)
+
+  def test_shutdown(self):
+    amp = Amplifier(debug=True)
+    for _ in range(10):
+      r = amp.set_global_shutdown(True)
+      r = amp.set_global_shutdown(False)
+      # amp config should be successful, with no i2c errors
+      assert r
+      assert self._check_for_i2c_errors(False)
+
+  def test_init_while_siren_play(self):
+    for _ in range(10):
+      self.panda.set_siren(False)
+      time.sleep(0.1)
+
+      self.panda.set_siren(True)
+      time.sleep(random.randint(0, 5))
+
+      amp = Amplifier(debug=True)
+      r = amp.initialize_configuration(Tici().get_device_type())
+      assert r
+
+      if self._check_for_i2c_errors(True):
+        break
+    else:
+      pytest.fail("didn't hit any i2c errors")

system/hardware/tici/tests/test_hardware.py

@@ -0,0 +1,26 @@
+import pytest
+import time
+import numpy as np
+
+from openpilot.system.hardware.tici.hardware import Tici
+
+HARDWARE = Tici()
+
+
+@pytest.mark.tici
+class TestHardware:
+
+  def test_power_save_time(self):
+    ts = {True: [], False: []}
+    for _ in range(5):
+      for on in (True, False):
+        st = time.monotonic()
+        HARDWARE.set_power_save(on)
+        ts[on].append(time.monotonic() - st)
+
+    # disabling power save is the main time-critical one
+    assert 0.1 < np.mean(ts[False]) < 0.15
+    assert max(ts[False]) < 0.2
+
+    assert 0.1 < np.mean(ts[True]) < 0.35
+    assert max(ts[True]) < 0.4

system/hardware/tici/tests/test_power_draw.py

@@ -0,0 +1,127 @@
+from collections import defaultdict, deque
+import pytest
+import time
+import numpy as np
+from dataclasses import dataclass
+from tabulate import tabulate
+
+import cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
+from openpilot.common.mock import mock_messages
+from openpilot.selfdrive.car.car_helpers import write_car_param
+from openpilot.system.hardware.tici.power_monitor import get_power
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.system.manager.manager import manager_cleanup
+
+SAMPLE_TIME = 8       # seconds to sample power
+MAX_WARMUP_TIME = 30  # seconds to wait for SAMPLE_TIME consecutive valid samples
+
+@dataclass
+class Proc:
+  procs: list[str]
+  power: float
+  msgs: list[str]
+  rtol: float = 0.05
+  atol: float = 0.12
+
+  @property
+  def name(self):
+    return '+'.join(self.procs)
+
+
+PROCS = [
+  Proc(['camerad'], 2.1, msgs=['roadCameraState', 'wideRoadCameraState', 'driverCameraState']),
+  Proc(['modeld'], 1.12, atol=0.2, msgs=['modelV2']),
+  Proc(['dmonitoringmodeld'], 0.4, msgs=['driverStateV2']),
+  Proc(['encoderd'], 0.23, msgs=[]),
+]
+
+
+@pytest.mark.tici
+class TestPowerDraw:
+
+  def setup_method(self):
+    write_car_param()
+
+    # wait a bit for power save to disable
+    time.sleep(5)
+
+  def teardown_method(self):
+    manager_cleanup()
+
+  def get_expected_messages(self, proc):
+    return int(sum(SAMPLE_TIME * SERVICE_LIST[msg].frequency for msg in proc.msgs))
+
+  def valid_msg_count(self, proc, msg_counts):
+    msgs_received = sum(msg_counts[msg] for msg in proc.msgs)
+    msgs_expected = self.get_expected_messages(proc)
+    return np.core.numeric.isclose(msgs_expected, msgs_received, rtol=.02, atol=2)
+
+  def valid_power_draw(self, proc, used):
+    return np.core.numeric.isclose(used, proc.power, rtol=proc.rtol, atol=proc.atol)
+
+  def tabulate_msg_counts(self, msgs_and_power):
+    msg_counts = defaultdict(int)
+    for _, counts in msgs_and_power:
+      for msg, count in counts.items():
+        msg_counts[msg] += count
+    return msg_counts
+
+  def get_power_with_warmup_for_target(self, proc, prev):
+    socks = {msg: messaging.sub_sock(msg) for msg in proc.msgs}
+    for sock in socks.values():
+      messaging.drain_sock_raw(sock)
+
+    msgs_and_power = deque([], maxlen=SAMPLE_TIME)
+
+    start_time = time.monotonic()
+
+    while (time.monotonic() - start_time) < MAX_WARMUP_TIME:
+      power = get_power(1)
+      iteration_msg_counts = {}
+      for msg,sock in socks.items():
+        iteration_msg_counts[msg] = len(messaging.drain_sock_raw(sock))
+      msgs_and_power.append((power, iteration_msg_counts))
+
+      if len(msgs_and_power) < SAMPLE_TIME:
+        continue
+
+      msg_counts = self.tabulate_msg_counts(msgs_and_power)
+      now = np.mean([m[0] for m in msgs_and_power])
+
+      if self.valid_msg_count(proc, msg_counts) and self.valid_power_draw(proc, now - prev):
+        break
+
+    return now, msg_counts, time.monotonic() - start_time - SAMPLE_TIME
+
+  @mock_messages(['liveLocationKalman'])
+  def test_camera_procs(self, subtests):
+    baseline = get_power()
+
+    prev = baseline
+    used = {}
+    warmup_time = {}
+    msg_counts = {}
+
+    for proc in PROCS:
+      for p in proc.procs:
+        managed_processes[p].start()
+      now, local_msg_counts, warmup_time[proc.name] = self.get_power_with_warmup_for_target(proc, prev)
+      msg_counts.update(local_msg_counts)
+
+      used[proc.name] = now - prev
+      prev = now
+
+    manager_cleanup()
+
+    tab = [['process', 'expected (W)', 'measured (W)', '# msgs expected', '# msgs received', "warmup time (s)"]]
+    for proc in PROCS:
+      cur = used[proc.name]
+      expected = proc.power
+      msgs_received = sum(msg_counts[msg] for msg in proc.msgs)
+      tab.append([proc.name, round(expected, 2), round(cur, 2), self.get_expected_messages(proc), msgs_received, round(warmup_time[proc.name], 2)])
+      with subtests.test(proc=proc.name):
+        assert self.valid_msg_count(proc, msg_counts), f"expected {self.get_expected_messages(proc)} msgs, got {msgs_received} msgs"
+        assert self.valid_power_draw(proc, cur), f"expected {expected:.2f}W, got {cur:.2f}W"
+    print(tabulate(tab))
+    print(f"Baseline {baseline:.2f}W\n")