Increase Fault Resilience

panda/python/__init__.py

@@ -804,7 +804,8 @@ class Panda:
   # The panda will NAK CAN writes when there is CAN congestion.
   # libusb will try to send it again, with a max timeout.
   # Timeout is in ms. If set to 0, the timeout is infinite.
-  CAN_SEND_TIMEOUT_MS = 10
+  CAN_SEND_TIMEOUT_MS = 5
+  CAN_MAX_RETRIES = 3
 
   def can_reset_communications(self):
     self._handle.controlWrite(Panda.REQUEST_OUT, 0xc0, 0, 0, b'')
@@ -812,18 +813,18 @@ class Panda:
   @ensure_can_packet_version
   def can_send_many(self, arr, timeout=CAN_SEND_TIMEOUT_MS):
     snds = pack_can_buffer(arr)
-    while True:
-      try:
-        for tx in snds:
-          while True:
-            bs = self._handle.bulkWrite(3, tx, timeout=timeout)
-            tx = tx[bs:]
-            if len(tx) == 0:
-              break
-            logging.error("CAN: PARTIAL SEND MANY, RETRYING")
-        break
-      except (usb1.USBErrorIO, usb1.USBErrorOverflow):
-        logging.error("CAN: BAD SEND MANY, RETRYING")
+    for tx in snds:
+      retries = 0
+      while len(tx) > 0:
+        bs = self._handle.bulkWrite(3, tx, timeout=timeout)
+        if bs == 0:
+          retries += 1
+          if retries > self.CAN_MAX_RETRIES:
+            logging.warning("CAN send: no progress after retries, dropping")
+            break
+        else:
+          retries = 0
+        tx = tx[bs:]
 
   def can_send(self, addr, dat, bus, timeout=CAN_SEND_TIMEOUT_MS):
     self.can_send_many([[addr, None, dat, bus]], timeout=timeout)
@@ -831,13 +832,16 @@ class Panda:
   @ensure_can_packet_version
   def can_recv(self):
     dat = bytearray()
-    while True:
+    for _ in range(self.CAN_MAX_RETRIES):
       try:
         dat = self._handle.bulkRead(1, 16384) # Max receive batch size + 2 extra reserve frames
         break
       except (usb1.USBErrorIO, usb1.USBErrorOverflow):
         logging.error("CAN: BAD RECV, RETRYING")
-        time.sleep(0.1)
+        time.sleep(0.01)
+    else:
+      logging.error("CAN: recv failed after retries")
+      return []
     msgs, self.can_rx_overflow_buffer = unpack_can_buffer(self.can_rx_overflow_buffer + dat)
     return msgs