From 7ec34af976288b72796d3536d0efcd5d877f4364 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Thu, 5 Feb 2026 00:04:15 -0600 Subject: [PATCH] Increase Fault Resilience --- panda/board/obj/bootstub.panda.bin | Bin 14876 -> 14876 bytes panda/board/obj/bootstub.panda_h7.bin | Bin 16944 -> 16944 bytes panda/board/obj/panda.bin.signed | Bin 60816 -> 60820 bytes panda/board/obj/panda_h7.bin.signed | Bin 69648 -> 69652 bytes panda/python/__init__.py | 34 ++++++++++++++------------ panda/python/spi.py | 13 +++++++++- 6 files changed, 31 insertions(+), 16 deletions(-) diff --git a/panda/board/obj/bootstub.panda.bin b/panda/board/obj/bootstub.panda.bin index baa42ef1684f2919c2d9a211ccc97876ed15fb73..657034f2eff1e26717863b3c85582a4e12355efa 100755 GIT binary patch delta 21 ccmbPJGN)w2En^O2(?m-%GYgB&kBqI@0AOkd%m4rY delta 21 ccmbPJGN)w2En^N#19Q_v3yajvkBqI@0AVKw`2YX_ diff --git a/panda/board/obj/bootstub.panda_h7.bin b/panda/board/obj/bootstub.panda_h7.bin index 87d7f4cbca99d9df39f37a772a330af012276fe6..88f36e1f8be0e9cb07235d1ea31ca653e497f91e 100755 GIT binary patch delta 23 dcmdnc!ngs5X4!BUn=?P8J!VU;Do@69J3KhR8)|FOOUXDh=@dqq7bnZ(RiE?wa|k-gw-gg zCR#!@dT{4!RZ3CO<;E9v5;gI8U=&Xj1E@rZQ6Zp&!<<^&w_Uf2+~c3ys?Yo${p;>u z|GH3-%ff+moo!|hd01BuAXTe2K z1M0vX&*dCj{;k;0A2o^z%_ zc&3*f5mO?U$57qNN4)Ry7`9;G70(M8`~{{Rc87>v(ET7Mf@vUy zMT>o!3}~ytX0Ss^ka8*3dTmfHp0V~DG)Le*18PANYqpPbZHK0gb2?!+*|BE*4_MA* zr|MTM%waxtU^^YUwY(lJ`Ixc+(}9U#8c1Q~8WTnxIe00a+XhF`40f06O0?rrDy>F{ z1+=m%r^AB#T_XCjbmuHIH#>)mG>dr-4WvdTVW=Wdy&}ja*MRT2)v3JiyIn1~bn@`}(aKwG zi=R!49+M^L-?UbbnM-Ff?buLh5x2n?Y~I*Dfos7^>Np*xe^J)?YgEq1;5iPez#(u; zF^`{4H8yuAceB@l2k0iH>^+Zxf_)dpQ!a*1pmY^`JuwU`2@TsO7_k0SRv-2SN|pM- zDQP|IVz8g-!pG6$Nj>u*xo2=^}VH<0@cm7hV9=NKLE;}Zs% z1hN#@jp2e%0aP~!Rb9B4}|oFZ3meX{MKew+n1P} zo)U2Yyi@#%%w;ibsh>^S@n!C3&w{P#XD1-Xg85! zICy|86PXX>_YnVC{cHfV6O({J#z+OvMJwx9O>rESn!ccB=g@^ox=6KuX*{*&7xIj6A2L9>WjLG@tEkf!%2@$n&gj|V12SX0S+J`{$f!!c85YIPr zBm5qaDHxb1vj&?s2=WCs`yI$yw7Ww7gm!1hSC|h1Kjja9+9imAL8&3Z!FoOO+v6u) z`;ds`;3Z4llNA&IZC#`-+}eph9+lG79R9sguWjbWNVl3;VAtP}! z8ncnlN5m+&lVkBB0}0*un(FiaN4sJ^&`3S)5n6YUhy{Ca37U>Xrx39S#Dh|$v^<MpRs;qm7og-@FfzQ? zQ0z9~i5YlOVNpo*ZF^Y%cBQ3agnMi&{*U?DYS$B)pS4BWRtJhhR+WRwV3?3}d{04Q zB5r#bxVg>6kpXUKGaGR+gTDymv4l%*CgHb8ViplkpvPk{5BmAp*kwfXUTq)a077>K z7g5~Hzytb)P$FY;{8}CO^Zy)&J-Xynu(kWuw-@b(~w4&>0h6#pUpGT(_9SH$of z%#b_>Z+v=r1#CGi{uIST-w2FT0Lq~IV4V7Xg5<8)2Pz4 zQ73~BX){Oqd=uIAWOlE9-bXY)MLN1VVr=5@J=%*oqvx(Wxlhc<%&$&3E41GBa{ORo z_d>f5<1Yt4@EPhM?>amoJ9_J{vhT+l{cru_c2Cp7&uhlN)Aq1-q?1cP%Jr7tk2ZaO hN*Ezs+w+s9K<9PEJ+tDE(@!2ds&882nD~JD{tHG2o5BD9 delta 3215 zcmb7`3s6+o8OP7r<;q!M1qKMauEe`hMvP!UKtUm_5Cudls6>3#SMU)+)ln*h;A*NB zn=nF-D8$f!N(E&Vd*Y}~h^VRHC{~iHONa^@e1aH_gBi`x|8Zpo)0xh6XMXp4ob#RY zoyWcVZEMjMv}mpForK)^7hz!+{u}a}UvD4j?$yK62pb2+gE%k+Ob4^Te2@mx!78v8 zF4xoo~;=|t9hv=7SA@#OgtmBE9h6eO1oKi^jpHhor!AU5l;8KtDBHo zmUfa!_uX8>TbAsVapX99E60t?{!Lho3sGG!5th`SsCIXvs%}J`!9Dstnp*&G8K?oC ziQnhavGnynU4^nvo>(MWRmId-a zu}IP7P^`15e-55gTsG?~;XVo)fStFwBzoV5rcHD|>@?Bs?8e(z%0&0tCw#o?e9Cya z>m#Gwi?(%`(vImsIEV+S+@`l;)W8Ahc$N%^rOWsgsfz@+bgIijmuk?;Yu)!NUd@CJ z-&`_aM)2U)^z@-t{=F5~$gg ziNHB@D)${ZPPahXU^8DhvPY5HD3SJtnbI%B<{-VsV?R7K;4t_bxF9M^F8$;<@PNSk|DvypTQX;YbdAvF5BWzXLn@4o(VHbch?ic+A ztr02Faq^r>E&G)^xG(T2k<>T*hj6#H#Y%MxVS9K+>Iy~ufiMGaOMRr@2JK^DiNFo!ZL^N(Laz`O z0IEg#?7dDpJ5+KNcT6PDA4YX+puPt>#D)beRByQE2(>_l^W}@SDqldW7QZesc_3Tk zniY6%Sz|)76>F|hoi{e4!%EZSbCp&_%vX%i%jOIyZIGuuyFNbu?}Kma3U> ze+o{6oby!egnS0nhmI=~0XYM#5XQV{#cu=DQcwxxJvs(imyIuq%n36x!WNtqx%r72 zj!Uq5K*NpfUWCeKZ7l!d)t#b@Q2aIo`H4+cKAp~ zeAyjG|6coN*mB%nJM!*J?N(9%hwPxL*nUX=8-%&^v8!QAzuoL;zYSX^)N4on=x8?# zCaM>nP{4-$gax1*H-V@*|gr$>rfKKAeIS>izdX#1@S7l0F|bVL0~}g^$e(f8uRu1S-5c@&+C3njU_J=^ z#9u!BtHL6J#)K^l4^QJkWdXV~zay*ybn~>b)ijVdmzj)V$%IWfgm<(?{wH9;=^gt2j-LV?ud`wuhk1*dN zR7;akgkbVC$R?3n(M6;DQ11-S!_1tpXt+}n@bUr$VR8eS*8Pumo?PiEzS{c@^>s!_ zV=cDL?w=o8-z=#=Sa#68J_(gkE-*(Y%8)l%bTz}dp>~I$5 zMNrssd^tgK3M#kvP=(FGWdc5EE6qlc1%E!s2J?9NY0qH#E*3O{utSJy2eY8pzKz5X zAtT#ng6f>rp>-JWcRG-&l>OTF!y&E+ga{EX?EQP-zV4lMIK4|aHewboyr^2C!f0c z#Cp?rY;^yU_uUReb|ur3TFshsQFiv{fWMn+Vn_Hc@_y>Jc-HR+PH3*)L3)-hv5cPm p!Q+16v6uEJUKvT2h%KMjCVn!yqhZ^&XZicb)^|@Dbn_Cu_#bGtkI(=B diff --git a/panda/board/obj/panda_h7.bin.signed b/panda/board/obj/panda_h7.bin.signed index 349400f899d2b4c6c0245480529d45b268514d48..d95c992c738bf54755d6c19e8e8b7a0a1f4d0e59 100644 GIT binary patch delta 3493 zcmb7{dr(x@9mnrEyI1alD@zb@0}Q(&KA@B(s9+EaA_5UBMlfQef(Fq#h9ydjF}N5o z>L|Vrj&?>ytX4pwif2)%@=|QfsEH()fUhxA9bsAqQbo;#mVS@RZPA)P!pvuX&+|Ly z{O-Be`J8X4^btk^#74w`crd+Th)F1sNx4Ef?zNiM1W`z2j+nJAQdbD znP3&j0qgvzP052&0JeecW2v$q>;WZU7kvAGnCviQ8K?pr)PY8D30wiK;5%>!{0(%0 z9?%DV0%FkwHJyMSxC7%@+y8d$3x_`l0bw8##DD}a7o>o9K?YEFe-gSVJoGOMzSUq2 zSPyJqGpGUQKr{FpJg>22NZYRsIs?_V` z1-hm`QYG9;GQ~3=`R^X~!&K3Hqv6(DHq8!@)%W; zoF!AcK{AZ zvs|wfc48B=4E0huJFfp7F)-bT0i$R`VmoFo$E+X>$OuAs`VnBILE!a42_UC1{q)i;!+@gmgAgCSX#%6V<6fkt+}@<8tIhqSHs>gJZxu&s(RjbCO%0N9N25Bjg0z92ZG;u#0i~cSZ$oVZbb7v&&gI{q z)asH?QRNe`me;>Op6Gg^c7heWJNp*V7gsx~orkou8#%jV%W0}i;443Hcf$tq3bOIr zUEqcF>k8ee?rZcf`Hp1jWd5IBgtO~2FA5=V8-QT)jz=EG9r!1qh$_C2X^{Sq8C+j9 z8^@cc?tDt*0o0hnQ+9_@GM*RheMlyL+(MOpUn%@)z*hGfDzB&nGyT8NRcs?8dX zEw}xnfhxn`zCT?u32AMlif)K~8@3p(#w{qTA7W2JT!!5?%`rZ|-jT1`ZLmdM)%M1G zqTZlwldOXaIR9(OBnl~Ksp0~u+B;wi|5}S9ei^nXpxWNGj<`KYGHKwydIkp)vOmy~ zUkF?HX9hTIZ`fi%eb4{cnDV#DLY7+|N)E6)QD~;<%6)X-6*39C90B?=d5}ew`nParG~f&Ej1LFbb9|&?VVXe!i?pvH6|CCsWxVqm zQuC?egCQR7xR$z7B@4u`2lg!Ex!JgyVdwtKsZxZ@6lA!Kz!!*R*6h-E>rrnb-uN7< zR5NmBiM|A$+&tVlGx4Ic)n~%=>)|Q+1lL-4=CSHC!A2P#pAD$H3^NbMB0(O=znjvzaMYhNQsG1tB03HkAnbs@}*f4P*Bb2 zCcVq%-Smv!Hp4MWKMMT@VpNUs#Zei^Ss9Aw7L5NzASV_TBuN`34tzhaxS2gEBsRuT zrW@fPOkM}6mdV28xj-!=Cf~L{!Ii|>R>XR?rEdM#`Fk}>2ipIlm1{m-zwBYjx5fU2 z<%w+@k9~J~@y;)|TP>T)4lU?v>)4bO?(?iAXXB+s8B6|o5e>ZeUP_v^}}_CZpHLJc@Wd|pL0abkVK delta 3552 zcmb7{dr(!!9mmh^y+_XhZr}#5atDxmMMMM@iWC&_t%zX7;EM*L1kpw@fUk&xS4Yt} zK7w7J9UU?F00q5dL9~K^k0PTsI?)cO*knq_I7-zRtEpJ~T`s2uWB%x!`JCVW9{bzf z-`TS)I|}sq1$tY)E0LNHP$i(?=SXk;v>0L5+Gby>_y9i;2tq(Ohyr6l4449@gAYLp zm=6|#CBD?AWTRmNd0=B-sxYt>>;;=)+YZEF`ymTKF*pe-z*$fS>cKVe4QK*?1%C$* z!Bg-8h)KVO)FH15T!E>t?SHHGgux01f>1CNj0EFBESL&rf@Gkr{y5qqapzwuY@dJ( zunepMYe5;P1l8aIXwNabQ`s9jjQ*OllTRB^45YQ+mYpY|#jzWuUbc4yzHQTVt)nJ| zy>nZO`#4go2rXT#TZMjt>y`E2 zV*wM~s@smT6NZn73p2d;#wCTa^VH4^y9bu_7d_(p5bpI z#LV;hGpZgDec`;ut$~$#@`yDJenukrwcuzS>CR0<5_HA@xPw7B=mS3B+2Nb%f6a?J z%$vAr*hoTB*!0Maq<}X^`V;vue1%+%x3E_m4vQ5!Z zWE-oG-YFLyqe=rynYbAhYn@oBK7!r`Tv%z$VB*D_arDVS$5CV^$6c}RLn{s>fpm}q zwty;70Nr8Gqv=Lxdze)04a0ZqOMC@;7o_|)2IEf+wJ8z!vLk*2+ zUI%o<8YzjZ3r_0Qcso^AfJ|QZsXtL4q1giF@~0Vhh#|ksk?l03oi$~BAqRa$mA-t@ zVs|4Juy$Pro=?|#V*ZBp?o|CV>Nl-fvb3;(4HsZ+-QY=ym-yyxLAqRWEI>@^D#H3eFXXK|-DyYxUZA8%!GmB* z05;HFl7f#wR)8iT_TUeYhCgAy^N>9{*>4z{Ge8Ou?!}NnUrUxz$R!Go=bTkMaepe6 z3*l;Dbq8jXHQZXTkof%oLuftz#(-%+^lv;ZOD#2$Wz&94r&ankjh7vKtnc%rmMR1v zO34_>f>}vz(Djf##%nq|e`KER`*T$326M{@$s%NW4gTbH(i@?R?wUUcX$+n8VEAR| zH-|g==Ttl5HT^ntkyq0_FrLWQq1&YNjtNu^k}M*TzLF}=kedEEbYWi_?C@WNE)r&esSZJkB1hEWQY4ZWbrh!@Z?gyV^#~B>^juU!Y^UKU zrl`udsJ<&?99H=reL3`isSSo8Cu)HB>PpksvL*84vSVHi;a%;CH|71uRVK*ct2E;X$7vu8C3ZUU8X|M0t3NNo?ShX zs4)sItjno#i`Sj&Mdiaw(PYiFsphSV(KnqcKXpS9K!c3JK(?~Py3nwOcwD-Fi)F!D zfnr|+)p!M>2R?s!27w&tY*@UhhR0$h^ zmm7c`2VZxfg6SyDOyG+4^=QdL2$3$Eag<>D=8bXWQG259K;$Vicg>TGn6yy5zrpzb z12N-u)Cu#|j}nLMOI~_AV@Sy8v5q_s;19rjDj>ByshCd|(DI1+ 0: + bs = self._handle.bulkWrite(3, tx, timeout=timeout) + if bs == 0: + retries += 1 + if retries > self.CAN_MAX_RETRIES: + logging.warning("CAN send: no progress after retries, dropping") + break + else: + retries = 0 + tx = tx[bs:] def can_send(self, addr, dat, bus, timeout=CAN_SEND_TIMEOUT_MS): self.can_send_many([[addr, None, dat, bus]], timeout=timeout) @@ -831,13 +832,16 @@ class Panda: @ensure_can_packet_version def can_recv(self): dat = bytearray() - while True: + for _ in range(self.CAN_MAX_RETRIES): try: dat = self._handle.bulkRead(1, 16384) # Max receive batch size + 2 extra reserve frames break except (usb1.USBErrorIO, usb1.USBErrorOverflow): logging.error("CAN: BAD RECV, RETRYING") - time.sleep(0.1) + time.sleep(0.01) + else: + logging.error("CAN: recv failed after retries") + return [] msgs, self.can_rx_overflow_buffer = unpack_can_buffer(self.can_rx_overflow_buffer + dat) return msgs diff --git a/panda/python/spi.py b/panda/python/spi.py index be4f7dcf4..681f44a56 100644 --- a/panda/python/spi.py +++ b/panda/python/spi.py @@ -27,7 +27,10 @@ NACK = 0x1F CHECKSUM_START = 0xAB MIN_ACK_TIMEOUT_MS = 100 +MAX_ACK_TIMEOUT_MS = 500 # like C++ SPI_ACK_TIMEOUT +DEFAULT_TIMEOUT_MS = 500 # default when timeout=0 MAX_XFER_RETRY_COUNT = 5 +MAX_TIMEOUT_RETRIES = 5 # like C++ XFER_SIZE = 0x40*31 @@ -152,6 +155,8 @@ class PandaSpiHandle(BaseHandle): return cksum def _wait_for_ack(self, spi, ack_val: int, timeout: int, tx: int, length: int = 1) -> bytes: + # Original behavior preserved - timeout=0 means wait forever within this function + # The caller (_transfer) handles the overall timeout timeout_s = max(MIN_ACK_TIMEOUT_MS, timeout) * 1e-3 start = time.monotonic() @@ -225,10 +230,15 @@ class PandaSpiHandle(BaseHandle): logging.debug("starting transfer: endpoint=%d, max_rx_len=%d", endpoint, max_rx_len) logging.debug("==============================================") + # Fix timeout=0 infinite loop: default to DEFAULT_TIMEOUT_MS + if timeout == 0: + timeout = DEFAULT_TIMEOUT_MS + n = 0 start_time = time.monotonic() exc = PandaSpiException() - while (timeout == 0) or (time.monotonic() - start_time) < timeout*1e-3: + # Use the timeout for the overall loop, matching original behavior but with timeout=0 fixed + while (time.monotonic() - start_time) < timeout * 1e-3: n += 1 logging.debug("\ntry #%d", n) with self.dev.acquire() as spi: @@ -238,6 +248,7 @@ class PandaSpiHandle(BaseHandle): exc = e logging.debug("SPI transfer failed, retrying", exc_info=True) + logging.error("SPI transfer failed after %d tries, %.2fms", n, (time.monotonic() - start_time) * 1000) raise exc def get_protocol_version(self) -> bytes: