cereal: name the new panda faults (#39133)

panda/python/__init__.py

@@ -1,6 +1,7 @@
 # python library to interface with panda
 import os
 import re
+import math
 import sys
 import time
 import usb1
@@ -17,7 +18,7 @@ from opendbc.car.structs import CarParams
 from .base import BaseHandle
 from .constants import BASEDIR, FW_PATH, McuType, compute_version_hash
 from .dfu import PandaDFU
-from .spi import PandaSpiHandle, PandaSpiException, PandaProtocolMismatch
+from .spi import PandaSpiHandle, PandaSpiException, PandaSpiNackResponse, PandaProtocolMismatch
 from .usb import PandaUsbHandle
 from .utils import logger
 
@@ -149,6 +150,7 @@ class Panda:
   HEALTH_FLAG_SOM_RESET_TRIGGERED = 1 << 6
   HEALTH_FLAG_NMI_RESET = 1 << 7
   HEALTH_FLAG_HARDFAULT_RESET = 1 << 8
+  HEALTH_FLAG_HEARTBEAT_DISABLED = 1 << 9
 
   H7_DEVICES = [HW_TYPE_RED_PANDA, HW_TYPE_TRES, HW_TYPE_CUATRO, HW_TYPE_BODY]
   F4_DEVICES = [HW_TYPE_DOS]
@@ -556,6 +558,7 @@ class Panda:
       "som_reset_triggered": bool(flags & self.HEALTH_FLAG_SOM_RESET_TRIGGERED),
       "nmi_reset": bool(flags & self.HEALTH_FLAG_NMI_RESET),
       "hardfault_reset": bool(flags & self.HEALTH_FLAG_HARDFAULT_RESET),
+      "heartbeat_disabled": bool(flags & self.HEALTH_FLAG_HEARTBEAT_DISABLED),
       "sound_output_level": a[19],
       "temperature": a[20] - 40.0,
     }
@@ -690,6 +693,46 @@ class Panda:
   def enter_stop_mode(self):
     self._handle.controlWrite(Panda.REQUEST_OUT, 0xb5, 0, 0, b'', expect_disconnect=True)
 
+  def _require_fault_injection(self, *, h7=False):
+    if self.get_type() == Panda.HW_TYPE_BODY:
+      raise NotImplementedError("Body firmware does not implement these diagnostics")
+    if h7 and self.get_mcu_type() != McuType.H7:
+      raise NotImplementedError("This diagnostic requires an H7 Panda")
+    if not self.get_version().endswith("-FAULTTEST"):
+      raise RuntimeError("Fault injection requires a dedicated bench firmware build")
+
+  def inject_fault(self, fault_bit):
+    self._require_fault_injection()
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xea, fault_bit, 0, b'')
+
+  def inject_hardfault(self):
+    self._require_fault_injection()
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xeb, 0, 0, b'', expect_disconnect=True)
+
+  def inject_interrupt_fault(self, test):
+    self._require_fault_injection(h7=True)
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xef, test, 0, b'')
+
+  def inject_nmi(self):
+    self._require_fault_injection()
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xec, 0, 0, b'', expect_disconnect=True)
+
+  def inject_mpu_fault(self):
+    self._require_fault_injection(h7=True)
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xee, 0, 0, b'', expect_disconnect=True)
+
+  def inject_stack_overflow(self):
+    self._require_fault_injection(h7=True)
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xf0, 0, 0, b'', expect_disconnect=True)
+
+  def inject_self_test_failure(self, test):
+    self._require_fault_injection(h7=True)
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xed, test, 0, b'')
+
+  def inject_readback_fault(self, test):
+    self._require_fault_injection(h7=True)
+    self._handle.controlWrite(Panda.REQUEST_OUT, 0xe9, test, 0, b'')
+
   def set_safety_mode(self, mode=CarParams.SafetyModel.silent, param=0):
     self._handle.controlWrite(Panda.REQUEST_OUT, 0xdc, mode, param, b'')
 
@@ -722,16 +765,27 @@ class Panda:
   # libusb will try to send it again, with a max timeout.
   # Timeout is in ms. If set to 0, the timeout is infinite.
   CAN_SEND_TIMEOUT_MS = 10
+  # over SPI, a NACK comes back right away. this is how long we wait before trying that chunk again
+  CAN_SEND_RETRY_S = 1e-3
 
   def can_reset_communications(self):
     self._handle.controlWrite(Panda.REQUEST_OUT, 0xc0, 0, 0, b'')
 
   @ensure_can_packet_version
   def can_send_many(self, arr, *, fd=False, timeout=CAN_SEND_TIMEOUT_MS):
+    deadline = time.monotonic() + (timeout * 1e-3 if timeout != 0 else math.inf)
     snds = pack_can_buffer(arr, chunk=True, fd=fd)
     for tx in snds:
       while len(tx) > 0:
-        bs = self._handle.bulkWrite(3, tx, timeout=timeout)
+        try:
+          bs = self._handle.bulkWrite(3, tx, timeout=timeout)
+        except PandaSpiNackResponse:
+          # the panda's CAN TX buffers are full. try this chunk again if there's time for it,
+          # so a timeout of one retry period or less means a single attempt
+          if time.monotonic() + self.CAN_SEND_RETRY_S > deadline:
+            raise
+          time.sleep(self.CAN_SEND_RETRY_S)
+          continue
         tx = tx[bs:]
 
   def can_send(self, addr, dat, bus, *, fd=False, timeout=CAN_SEND_TIMEOUT_MS):

panda/python/spi.py

@@ -207,35 +207,19 @@ class PandaSpiHandle(BaseHandle):
 
     # timeout of 0 means forever
     deadline = time.monotonic() + (max(MIN_TIMEOUT_MS, timeout) * 1e-3 if timeout != 0 else math.inf)
-    nack = None
-    nack_count = 0
     # random, so it's never the same as another request the panda just handled, even from another client.
     # it stays the same when we send the request again, so the panda never runs it twice
     req_id = int.from_bytes(os.urandom(4), "little")
     while True:
-      try:
-        with self.dev.acquire() as spi:
-          dat = self._transfer_spidev(spi, endpoint, req_id, data, deadline, max_rx_len, expect_disconnect)
-        if dat is not None:
-          return dat
-        # the panda went quiet. let go of the bus for a bit, someone else might need it
-        logger.warning("SPI: panda isn't responding")
-        time.sleep(0.01)
-      except PandaSpiNackResponse as e:
-        # the panda NACKs while its CAN TX buffers are full, don't busy wait on it
-        if time.monotonic() > deadline:
-          raise
-        # a new id, since the panda would answer the same id with the same NACK
-        req_id = int.from_bytes(os.urandom(4), "little")
-        nack = e
-        nack_count += 1
-        if nack_count > 3:
-          time.sleep(min(max(nack_count * 10, 200), 2000) * 1e-6)
-      except PandaSpiMissingAck:
-        # only raised once the deadline passed. if the panda was refusing, that's why
-        if nack is not None:
-          raise nack from None
-        raise
+      # a NACK is the panda's answer (unknown endpoint, or its CAN TX buffers are full), it raises right away.
+      # whether to try again is the caller's call.
+      with self.dev.acquire() as spi:
+        dat = self._transfer_spidev(spi, endpoint, req_id, data, deadline, max_rx_len, expect_disconnect)
+      if dat is not None:
+        return dat
+      # the panda went quiet. let go of the bus for a bit, someone else might need it
+      logger.warning("SPI: panda isn't responding")
+      time.sleep(0.01)
 
   def get_info(self) -> bytes:
     # serial, HW type, bootstub