From 12088b9e537b59e4aa3fc369800b2554c8b15658 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Thu, 5 Feb 2026 00:04:15 -0600 Subject: [PATCH] Increase Fault Resilience --- panda/board/obj/bootstub.panda.bin | Bin 14876 -> 14876 bytes panda/board/obj/bootstub.panda_h7.bin | Bin 16944 -> 16944 bytes panda/board/obj/panda.bin.signed | Bin 60924 -> 60924 bytes panda/board/obj/panda_h7.bin.signed | Bin 69756 -> 69756 bytes panda/python/__init__.py | 34 ++++++++++++++------------ panda/python/spi.py | 13 +++++++++- 6 files changed, 31 insertions(+), 16 deletions(-) diff --git a/panda/board/obj/bootstub.panda.bin b/panda/board/obj/bootstub.panda.bin index ac228d7a0f75698e95a3b8214dc67b5755e43367..2c98ec2ac62276f5fa90b796a1f2d9c84ae8e805 100755 GIT binary patch delta 21 ccmbPJGN)w2En^Ndlhh;wBV*&ukBqI@0AaKT=Kufz delta 21 ccmbPJGN)w2En|*E^Q4rtq(t-0kBqI@0BE`htN;K2 diff --git a/panda/board/obj/bootstub.panda_h7.bin b/panda/board/obj/bootstub.panda_h7.bin index 89be15112966e60bf26383b796202ac7fa8d7d36..23dc5f2e85e3acee63e696d9ac56eb332f558cec 100755 GIT binary patch delta 23 dcmdnc!ngs5X4!CTO6luQJbBj<1sK+_`( zGt~Za_p0A_Eqde0VrG`8?N04gBf>+)1HIKuTJr(eZDuB}`qB zfTH>)R9N54LZw><1CiiUdET8pdld^9p+>(LIzq%k z1trSlG6~O>9(_e0`;}K8c`2+a W6)A3*jA(X~2izkPYxgTDiUAij@JoLH delta 168 zcmV;Z09XI~palG&1h71(0stJ7KBp=INFB3kr=AN4VK-uAW@2GCvkRV~R%q;@s6EUJJB@F0k@TbCJUn3;7bdeDgk4m6|h-+6#mr*IQ`Vhcw;{yg?^d^Pq zUm>6$A5rnD`~;V0`=_Nm@nw}TzA{->FxLsx{LtWWqZY?Tev9#Jk{fZl1)W~HIjPp( W$ys>_cUWh1n=>eC;#xOly#Y*zbxfN8 diff --git a/panda/python/__init__.py b/panda/python/__init__.py index 9339bfcf8..55a794d54 100644 --- a/panda/python/__init__.py +++ b/panda/python/__init__.py @@ -804,7 +804,8 @@ class Panda: # The panda will NAK CAN writes when there is CAN congestion. # libusb will try to send it again, with a max timeout. # Timeout is in ms. If set to 0, the timeout is infinite. - CAN_SEND_TIMEOUT_MS = 10 + CAN_SEND_TIMEOUT_MS = 5 + CAN_MAX_RETRIES = 3 def can_reset_communications(self): self._handle.controlWrite(Panda.REQUEST_OUT, 0xc0, 0, 0, b'') @@ -812,18 +813,18 @@ class Panda: @ensure_can_packet_version def can_send_many(self, arr, timeout=CAN_SEND_TIMEOUT_MS): snds = pack_can_buffer(arr) - while True: - try: - for tx in snds: - while True: - bs = self._handle.bulkWrite(3, tx, timeout=timeout) - tx = tx[bs:] - if len(tx) == 0: - break - logging.error("CAN: PARTIAL SEND MANY, RETRYING") - break - except (usb1.USBErrorIO, usb1.USBErrorOverflow): - logging.error("CAN: BAD SEND MANY, RETRYING") + for tx in snds: + retries = 0 + while len(tx) > 0: + bs = self._handle.bulkWrite(3, tx, timeout=timeout) + if bs == 0: + retries += 1 + if retries > self.CAN_MAX_RETRIES: + logging.warning("CAN send: no progress after retries, dropping") + break + else: + retries = 0 + tx = tx[bs:] def can_send(self, addr, dat, bus, timeout=CAN_SEND_TIMEOUT_MS): self.can_send_many([[addr, None, dat, bus]], timeout=timeout) @@ -831,13 +832,16 @@ class Panda: @ensure_can_packet_version def can_recv(self): dat = bytearray() - while True: + for _ in range(self.CAN_MAX_RETRIES): try: dat = self._handle.bulkRead(1, 16384) # Max receive batch size + 2 extra reserve frames break except (usb1.USBErrorIO, usb1.USBErrorOverflow): logging.error("CAN: BAD RECV, RETRYING") - time.sleep(0.1) + time.sleep(0.01) + else: + logging.error("CAN: recv failed after retries") + return [] msgs, self.can_rx_overflow_buffer = unpack_can_buffer(self.can_rx_overflow_buffer + dat) return msgs diff --git a/panda/python/spi.py b/panda/python/spi.py index be4f7dcf4..681f44a56 100644 --- a/panda/python/spi.py +++ b/panda/python/spi.py @@ -27,7 +27,10 @@ NACK = 0x1F CHECKSUM_START = 0xAB MIN_ACK_TIMEOUT_MS = 100 +MAX_ACK_TIMEOUT_MS = 500 # like C++ SPI_ACK_TIMEOUT +DEFAULT_TIMEOUT_MS = 500 # default when timeout=0 MAX_XFER_RETRY_COUNT = 5 +MAX_TIMEOUT_RETRIES = 5 # like C++ XFER_SIZE = 0x40*31 @@ -152,6 +155,8 @@ class PandaSpiHandle(BaseHandle): return cksum def _wait_for_ack(self, spi, ack_val: int, timeout: int, tx: int, length: int = 1) -> bytes: + # Original behavior preserved - timeout=0 means wait forever within this function + # The caller (_transfer) handles the overall timeout timeout_s = max(MIN_ACK_TIMEOUT_MS, timeout) * 1e-3 start = time.monotonic() @@ -225,10 +230,15 @@ class PandaSpiHandle(BaseHandle): logging.debug("starting transfer: endpoint=%d, max_rx_len=%d", endpoint, max_rx_len) logging.debug("==============================================") + # Fix timeout=0 infinite loop: default to DEFAULT_TIMEOUT_MS + if timeout == 0: + timeout = DEFAULT_TIMEOUT_MS + n = 0 start_time = time.monotonic() exc = PandaSpiException() - while (timeout == 0) or (time.monotonic() - start_time) < timeout*1e-3: + # Use the timeout for the overall loop, matching original behavior but with timeout=0 fixed + while (time.monotonic() - start_time) < timeout * 1e-3: n += 1 logging.debug("\ntry #%d", n) with self.dev.acquire() as spi: @@ -238,6 +248,7 @@ class PandaSpiHandle(BaseHandle): exc = e logging.debug("SPI transfer failed, retrying", exc_info=True) + logging.error("SPI transfer failed after %d tries, %.2fms", n, (time.monotonic() - start_time) * 1000) raise exc def get_protocol_version(self) -> bytes: