log chestnut ASM at 10Hz (#38609)

* modeld: only read the smu while the big model runs

* chestnutState at 10Hz

* smu polling

openpilot/cereal/services.py

@@ -25,7 +25,7 @@ _services: dict[str, tuple] = {
   "accelerometer": (True, 104., 104),
   "temperatureSensor": (True, 2., 200),
   "deviceState": (True, 2., 1),
-  "chestnutState": (True, 0.1, 1),
+  "chestnutState": (True, 10., 10),
   "touch": (True, 20., 1),
   "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
   "controlsState": (True, 100., 10, QueueSize.MEDIUM),

openpilot/selfdrive/modeld/modeld.py

@@ -71,9 +71,12 @@ def get_action_from_model(model_output: dict[str, np.ndarray], prev_action: log.
 
 class ChestnutState:
   # only modeld can access chestnut
-  def __init__(self, pm: PubMaster):
+  def __init__(self, pm: PubMaster, big: bool):
     self.pm = pm
+    self.big = big
     self.valid = True
+    self.sends = 0
+    self.metrics = {}
 
   @cached_property
   def power_limit(self) -> int:
@@ -83,33 +86,41 @@ class ChestnutState:
   def send(self) -> None:
     msg = messaging.new_message('chestnutState')
     state = msg.chestnutState
-    valid = False
-    if "AMD" in Device._opened_devices:
+    self.sends += 1
+    if self.big and "AMD" in Device._opened_devices and self.sends % 100 == 1:
       try:
         smu = Device["AMD"].iface.dev_impl.smu
         smu._send_msg(smu.smu_mod.PPSMC_MSG_TransferTableSmu2Dram, smu.smu_mod.TABLE_SMU_METRICS, timeout=100)
         metrics = smu.read_table(smu.smu_mod.SmuMetricsExternal_t, smu.smu_mod.TABLE_SMU_METRICS).SmuMetrics
-        state.tempC = metrics.AvgTemperature[smu.smu_mod.TEMP_HOTSPOT]
-        state.memoryTempC = metrics.AvgTemperature[smu.smu_mod.TEMP_MEM]
-        state.powerDrawW = metrics.AverageSocketPower
-        state.powerLimitW = self.power_limit
-        state.gpuUsagePercent = metrics.AverageGfxActivity
-        state.gpuClockMhz = metrics.AverageGfxclkFrequencyPostDs
-        state.fanSpeedRpm = metrics.AvgFanRpm
-        valid = True
+        self.metrics = {'tempC': metrics.AvgTemperature[smu.smu_mod.TEMP_HOTSPOT],
+                        'memoryTempC': metrics.AvgTemperature[smu.smu_mod.TEMP_MEM],
+                        'powerDrawW': metrics.AverageSocketPower,
+                        'powerLimitW': self.power_limit,
+                        'gpuUsagePercent': metrics.AverageGfxActivity,
+                        'gpuClockMhz': metrics.AverageGfxclkFrequencyPostDs,
+                        'fanSpeedRpm': metrics.AvgFanRpm}
+        self.valid = True
       except Exception:
         if self.valid:
           cloudlog.exception("chestnut state read failed")
+        self.valid = False
+        self.metrics.clear()
+    if self.big:
+      for k, v in self.metrics.items():
+        setattr(state, k, v)
+
+    asm_valid = False
+    if "AMD" in Device._opened_devices:
       try:
         # ASM runs on USB-C power, these still read without a gpu
         asm = Device["AMD"].iface.pci_dev.usb
         state.pcieLtssm = asm.read(0xB450, 1)[0]
         state.supplyVoltage, state.supplyCurrent = struct.unpack('<Hh', bytes(asm.usb.control_read(0xC0, 5))[:4])
+        asm_valid = True
       except Exception:
         pass
 
-    self.valid = valid
-    msg.valid = valid
+    msg.valid = asm_valid and (not self.big or self.valid)
     self.pm.send('chestnutState', msg)
 
 
@@ -266,7 +277,7 @@ def main(demo=False):
 
   publish_state = PublishState()
   params = Params()
-  chestnut_state = ChestnutState(pm) if USBGPU else None
+  chestnut_state = ChestnutState(pm, model.usbgpu) if USBGPU else None
 
   # setup filter to track dropped frames
   frame_dropped_filter = FirstOrderFilter(0., 10., 1. / ModelConstants.MODEL_RUN_FREQ)
@@ -382,6 +393,8 @@ def main(demo=False):
       cloudlog.exception("big model failed, fall back to small")
       params.put_bool("UsbGpuActive", False)
       model = small_model
+      if chestnut_state is not None:
+        chestnut_state.big = False
       run_count = 0
       model_output = None
     mt2 = time.perf_counter()