IQ.Pilot Release Commit @ 087daed

artifacts/iqpilot_alc_private/manifest.json

@@ -16,7 +16,7 @@
   },
   "python/iqpilot_private/konn3kt/iqlvbs/alc.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "b7a1d9c89f0fb20719ed03104c150361ead436f4eaf758213c34270f857fce29",
+    "sha256": "a7ac44d7ce568d335b6cd5e31c940c993c3febfbf150dbc5b625fec930789d30",
     "size": 401688
   },
   "runtime": {
@@ -28,6 +28,6 @@
     }
   },
   "signatures": {
-    "python/iqpilot_private/konn3kt/iqlvbs/alc.cpython-312-aarch64-linux-gnu.so": "3aPgCbOFw97bTasyTd2NnljzjuHrF9qEzuZX1X2Vu1Nu0dz+btrOzgiwV7lekZ6SyJbCpIUYeI+5ifNW2MzaBQ=="
+    "python/iqpilot_private/konn3kt/iqlvbs/alc.cpython-312-aarch64-linux-gnu.so": "OgV+x72Bjg9Gxxqv8pGyaTEee7bJK2xuCcfgmeKN34YKg6jjlyBgvvsx69UVrHlJx8It6udCEfDqfGjtRbzRCA=="
   }
 }

artifacts/iqpilot_commander_private/manifest.json

@@ -16,7 +16,7 @@
   },
   "python/iqpilot_private/konn3kt/iqlvbs/iqlvbs_commander.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "f452c3278cd5f8cc40db981cae2bd91f4796a91abaa76b6001dbb5dad80480d6",
+    "sha256": "384571df8380e5eef92806f3fdb6c9d1bc879af2ebdce9259bcfc486d506a4d0",
     "size": 68080
   },
   "runtime": {
@@ -28,6 +28,6 @@
     }
   },
   "signatures": {
-    "python/iqpilot_private/konn3kt/iqlvbs/iqlvbs_commander.cpython-312-aarch64-linux-gnu.so": "7DvAdsDSBnDIh3f7PqVWudkB+Njn6o7HAvtiZPrWpO0GW40B/DXhRLDvJ9rT+O2Lw3pLJBq04qA1szhbcFBhCw=="
+    "python/iqpilot_private/konn3kt/iqlvbs/iqlvbs_commander.cpython-312-aarch64-linux-gnu.so": "cH88ob+s+ZRJaNgbrRN2CQqCiQMBlo9yXdBvN3gteSC7OYlJ5z8OZND7eZSEfeeq2O7E8ZeZDs9WDU0WfP8DCw=="
   }
 }

artifacts/iqpilot_hephaestusd_private/manifest.json

@@ -16,27 +16,27 @@
   },
   "python/iqpilot_private/konn3kt/backups/archive_codec.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "ec590435888e03a83c9fbfb01102089b077de0840ae0bb2e0969e76213d633aa",
+    "sha256": "fddb1a432475585812d5d3cea8fbb2db3770df19d205b372b0df0463b2ec0840",
     "size": 135536
   },
   "python/iqpilot_private/konn3kt/backups/backup_keys.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "7188e3de16f358a1dfb7130cf8aa7f12d2861bd68f056bff9a805c14883f3016",
+    "sha256": "a996e5ef6a6451e18f7dc49bc70a1223455278215c8d96092701e4ecbb68a538",
     "size": 67664
   },
   "python/iqpilot_private/konn3kt/backups/backup_orchestrator.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "7218ad7ac087c68fe7d466e4c3468a6ab3b48c0fe45711e546e53823039c365b",
+    "sha256": "59c62ac7fe547df307e5f73054e00f1f650b18b8f95adf3ca0e79de5540d7427",
     "size": 204624
   },
   "python/iqpilot_private/konn3kt/backups/cbc_vault.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "56d60b113a982e2d4fee8b4f00eaf111fd09834fbe8bf95f382b49e0bf13c121",
+    "sha256": "ad3c2e37af910f38af41a6866e8485cbdf81b3b9bb68def3a378d3960f925d75",
     "size": 69752
   },
   "python/iqpilot_private/konn3kt/backups/imahelper.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "75f101e930d93044d2e074129207c8246ff7d94df77b0f83db7de665eef62ab6",
+    "sha256": "1f6909f98ae72ee50e56f797846d4c5623c0d60487c63f07a2f94945f735f45b",
     "size": 203072
   },
   "python/iqpilot_private/konn3kt/hephaestus/__init__.py": {
@@ -46,67 +46,67 @@
   },
   "python/iqpilot_private/konn3kt/hephaestus/_vendor/localapi_runtime.zip": {
     "mode": 420,
-    "sha256": "acc64ce2fe59583738ab5951f64e79d8043856317eb015dc1d0e6a4bffc51ee2",
+    "sha256": "732160f1baf7f0161925bad753d1f577d4a1d8c78623f229484849c2052945c9",
     "size": 2100238
   },
   "python/iqpilot_private/konn3kt/hephaestus/ble_auth.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "ee2330e3ba4f488336d18d887fc19291f76a1733a24c203a642634b7106a85a1",
+    "sha256": "7b92999c3119cf76e73c0847a26961210c52fcaafc1d6d2877e2efd0a4456795",
     "size": 268064
   },
   "python/iqpilot_private/konn3kt/hephaestus/ble_gatt.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "71568e01b8c983ca80d825bea610334b140c87da9e6ac3047b21d9c56c136423",
+    "sha256": "ae928487c4e757cefe1f9881207c1137dc8b106563bcd78b9d9e6e10fb40b851",
     "size": 340584
   },
   "python/iqpilot_private/konn3kt/hephaestus/ble_rpc_dispatch.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "7f47eb50931e9df19116736ef6564b2d5480a7424c24cd1d599424dc0da66537",
+    "sha256": "f4adfd7598757a54205fa1ad91641f02d98a7df290378c74108f557bae2627d5",
     "size": 68048
   },
   "python/iqpilot_private/konn3kt/hephaestus/ble_transportd.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "e4c0882530ab3097ecadbb586ac7e35955b587f81d9af2c57fea1b5450c2e5ef",
+    "sha256": "174978b5f138a3c49734dcbb0e1353098ca2bdf3f2d2c81fbccd08db87ace417",
     "size": 335728
   },
   "python/iqpilot_private/konn3kt/hephaestus/bt_gamepad.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "62bbf38d79bece455bdb399d7124b37347c9ec2ebdf35ff2ce4b696b39c59462",
+    "sha256": "ca3feeb930cdd5b7f6a32fdb11acd6ecc5aabe29bd67d58ace6c9ffc15e545eb",
     "size": 271680
   },
   "python/iqpilot_private/konn3kt/hephaestus/cloud_routes.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "b59b5d1bc50e5064206752cb30da118f2fa42f48c4cc2e6529ebbe04b28200a4",
+    "sha256": "c523dacac84c4ad80181ccfa3e886252cb964fe71796d4d3c82935f68d6705d0",
     "size": 134408
   },
   "python/iqpilot_private/konn3kt/hephaestus/hephaestusd.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "1d7b19f9d33649e91e5707574bdf436bc0e5b6dafd63d4b1aa63949bb6f7be20",
+    "sha256": "f6154067c4885232d0cc72c346a1e6462941d8956c70eaf7341457b302b13c4a",
     "size": 3448272
   },
   "python/iqpilot_private/konn3kt/hephaestus/kwp2000.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "8a5e6e08eb15a8295f08bd4629f80f6bf2539ec7eea7b6a787af3ed5b2546e35",
+    "sha256": "94b222b5bea69ad307349be5328a5043f90b1e69c30e60670c034117c52f0c6a",
     "size": 136448
   },
   "python/iqpilot_private/konn3kt/hephaestus/manage_hephaestusd.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "ead654070e296ae23fd8afdbc0a1c2f3eb8d1ff04a905ca3f2fbd8e20316ed6b",
+    "sha256": "00d9d730c09439d8b23145e38afc657486ebf5daf11501711ad4713d1ecb9f59",
     "size": 68136
   },
   "python/iqpilot_private/konn3kt/hephaestus/motd.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "38c55fe55500aca574ab5bf3ed693e54ca192db276c247a31209470319d6578a",
+    "sha256": "d9e1b7bd29123482d9883e4bc47f3fe45c84fad326c06f2e861da07cbfdee13a",
     "size": 67840
   },
   "python/iqpilot_private/konn3kt/hephaestus/tp20.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "0547db3c95f399b9721ae4f7306b57d4124283adda0213f7df5069a3837a7f98",
+    "sha256": "b838d67347e4983886d3c366a6461cb349cbe27064928fbf25f643253ca68c86",
     "size": 135648
   },
   "python/iqpilot_private/konn3kt/hephaestus/vw_pq_flasher.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "f7788e2023012537b40d6aa5109e9d913b7cb27e1de80eadc0ebe12a2ab7be1a",
+    "sha256": "f44609d1974b2ec11cda57db2c9c09bd4e54a224bd96a743af6a9f507dc56478",
     "size": 269568
   },
   "python/iqpilot_private/konn3kt/uploaderd/__init__.py": {
@@ -116,7 +116,7 @@
   },
   "python/iqpilot_private/konn3kt/uploaderd/iquploaderd.cpython-312-aarch64-linux-gnu.so": {
     "mode": 493,
-    "sha256": "efbef0e4b63a65d846c4cd50f89cb3fdfe0d12da3301e8b1c9137f9a5f02c753",
+    "sha256": "bb9f65c0d0baf02144971d2c721f37b118007df9c5f45a8983400941ec7d68a6",
     "size": 204184
   },
   "runtime": {
@@ -148,23 +148,23 @@
     }
   },
   "signatures": {
-    "python/iqpilot_private/konn3kt/backups/archive_codec.cpython-312-aarch64-linux-gnu.so": "tMSz9Dm9XQQkQzePUaDhhcM065JH2fcqAE3f+dlA2/+264m/j2ayWMYFMXTWZMTDm1YaQT8e9lvBe8sc4JSVDQ==",
-    "python/iqpilot_private/konn3kt/backups/backup_keys.cpython-312-aarch64-linux-gnu.so": "fvD5Ii/rNZuynr8Nh+K+yNC+AMji7v53spO7rj7Fs61CvO604fWnKVUO7A/2d7KHh26YpUcjbv8wRb8tRTzfCg==",
-    "python/iqpilot_private/konn3kt/backups/backup_orchestrator.cpython-312-aarch64-linux-gnu.so": "KcxQfc5HVD8QOLY/KHWsrhWpbYETAxsxR9WjFOqqmYe0aEC7K4njmJPtU1eCU3B2S6MGMmUdABBHs8kr32ViDQ==",
-    "python/iqpilot_private/konn3kt/backups/cbc_vault.cpython-312-aarch64-linux-gnu.so": "9jTowLTfwEw9Iu3wgfFordJmJ+/+O+nPvq4TP+3CDddW1jISzSMhiw418iyH1f0m7rwB3hGyZYgGLoJszSHpBw==",
-    "python/iqpilot_private/konn3kt/backups/imahelper.cpython-312-aarch64-linux-gnu.so": "YvM513CnlMKSrIrCgrCNG/k7XyFjSlnEupgWH0Jhj0zKrE8dkeCiXek7iU9W+VpgXa6O8P3jTJUWchqLhnONBw==",
-    "python/iqpilot_private/konn3kt/hephaestus/ble_auth.cpython-312-aarch64-linux-gnu.so": "bhNLMkcIgi0F/CZsEDHRQsNHyPBD+pVh3DPowBz7mRYxQffVGzk1FeE2Gp3P7xxHsnyKsy635sAkdwKOlZLtBQ==",
-    "python/iqpilot_private/konn3kt/hephaestus/ble_gatt.cpython-312-aarch64-linux-gnu.so": "Huy5oGlX55p5bhoMQipY9/doTBBAfwnQhgPdM6aeYTrzBS/QOUqwjyk3qjanftHA4m0V4Jab8cVvjEt1OiTsBw==",
-    "python/iqpilot_private/konn3kt/hephaestus/ble_rpc_dispatch.cpython-312-aarch64-linux-gnu.so": "2BmMUg8uOE4eoIW/OTbuwrZCMA+Rx4SOZWLBYs4GUbD8JR5Ei0nQ6VKLar31Ue7NYk7qBfpgqFdGPnV1uST/Dw==",
-    "python/iqpilot_private/konn3kt/hephaestus/ble_transportd.cpython-312-aarch64-linux-gnu.so": "6nFN6dJAmGczneUwRtJr+1TLNrD9reuMQcoa2G2j0np9xaSU401MdBssdVFSFBpse95iqQX1uV8ojQD327z1CA==",
-    "python/iqpilot_private/konn3kt/hephaestus/bt_gamepad.cpython-312-aarch64-linux-gnu.so": "8nKxC5LPkwPihlIdJzcD6OTp42avONqDkNgZrmTaQr8oueczFwXnHsgJ9aZ8jrpDK/nN+GLJrniSdFxZjrUUDA==",
-    "python/iqpilot_private/konn3kt/hephaestus/cloud_routes.cpython-312-aarch64-linux-gnu.so": "y8dA+6ahkX+Evzn8oSQEPyDB6+d+t//v5rsEdg4zRbjdeLeXs5kqraMEaAALVH80CvNJ6Gba1CcO1ebrK00EBQ==",
-    "python/iqpilot_private/konn3kt/hephaestus/hephaestusd.cpython-312-aarch64-linux-gnu.so": "jczf5QX+Xm0tOLPBrmvJoPZmcKar5zTrxoX7RAOR1JQgHu3jWPFW+sWOTK8c8ByiVF5PdLeYOPdQgu+0KQvdCg==",
-    "python/iqpilot_private/konn3kt/hephaestus/kwp2000.cpython-312-aarch64-linux-gnu.so": "s4HcwSXfBowW/vfY5M4GKg7gEueRJnbmq6zO1dAXx6HrTXYsL9iDZTeGHi7dNu/rw8mUFSkkpWsaFTlaYdK0DA==",
-    "python/iqpilot_private/konn3kt/hephaestus/manage_hephaestusd.cpython-312-aarch64-linux-gnu.so": "LVgImFZgV/hgoBwMfMVWSEKuXUvc2jR/EPl6SOfmMncoyuIZUkpQa18jjdMTWqXbRGpFvAe+0KocmroFTrbqDg==",
-    "python/iqpilot_private/konn3kt/hephaestus/motd.cpython-312-aarch64-linux-gnu.so": "vXffOAHGUAdEkv321fidnhZvUMaX73l2kO5/Y5Sk43+mr9V95Wcv4TR54/2Y2N4UfgtxnGFz8N0llK/DsQLEAw==",
-    "python/iqpilot_private/konn3kt/hephaestus/tp20.cpython-312-aarch64-linux-gnu.so": "wXVM74ZUEwSVnM7cMqzXQYCWhQwVovszTjxYyMhDFjwZlWtNPPGeq4WghHoghGolGSG9Q4SOLxh8uF67BU2RDw==",
-    "python/iqpilot_private/konn3kt/hephaestus/vw_pq_flasher.cpython-312-aarch64-linux-gnu.so": "1cBBo9W9EBxW7XF9o2MimKA9Xm26zaCEsUNHLF81zm2zWxkoSkeVlLyrxHE9ZxGhgMRMfW3lz4XUJbWRJpjGDA==",
-    "python/iqpilot_private/konn3kt/uploaderd/iquploaderd.cpython-312-aarch64-linux-gnu.so": "vqZyPMmER7bN7AvE4gwZVF4HypLwTxBfk4tidjRhVDZ2eKYbbG2yROd2z2xtk5Shf8GhdLsT7UhVrN+BKNnkBA=="
+    "python/iqpilot_private/konn3kt/backups/archive_codec.cpython-312-aarch64-linux-gnu.so": "njPmYch3VDP4JuJv/57vY03J1darpkL5QqjL7w07XYgkF6vzflZYFJq2ANSE0gqS3hWio2IZxOaVG16MJvxRCw==",
+    "python/iqpilot_private/konn3kt/backups/backup_keys.cpython-312-aarch64-linux-gnu.so": "eTwlMQbC5Z+Hplu3X00ewYQT7ftqnLCgComW9n54KIPSUiZbxwjiw4G2CeFhe28JfJUBr6sDUDlVoVNnZ9OqBA==",
+    "python/iqpilot_private/konn3kt/backups/backup_orchestrator.cpython-312-aarch64-linux-gnu.so": "k09CSvePXfMH7+q5uTsdIt69+STam/crgEXWvp5WGqAh3Hl0ZDVnRNHh8f+SKmppSTzU8gtjE9Yx/wzgjjYeBw==",
+    "python/iqpilot_private/konn3kt/backups/cbc_vault.cpython-312-aarch64-linux-gnu.so": "j0EBLQA9VD3hBRi8eiXZlB4/ZcruECmsppEr+BP9NrzLskdlKAysN83o4YDUAMMinLNq0VYOx9Ix4PHksQmJDg==",
+    "python/iqpilot_private/konn3kt/backups/imahelper.cpython-312-aarch64-linux-gnu.so": "iqWiLWOAPNTB9FV/NqnFLImvPn3xEvuAqcjrQsY9v6GYBev51+uBUgVEwZhEW5VOaK3tGLjErLvhczu1s205Bg==",
+    "python/iqpilot_private/konn3kt/hephaestus/ble_auth.cpython-312-aarch64-linux-gnu.so": "/RVYVsF0U1n5ZjoMMSqmrHvv0THuRauCmVMZPEDOMU5EIWW+ZYiQrgO24xPUILSV/og3S1wKpEBpUgXFm4sCBg==",
+    "python/iqpilot_private/konn3kt/hephaestus/ble_gatt.cpython-312-aarch64-linux-gnu.so": "PHdyy9tyV0wrwHvawZqHZv3W7MkJ6uaqluURusZF3BbIV4hdbok6i4bPyUUHIaiztFAclc7ktqm1D7nY4k7oBg==",
+    "python/iqpilot_private/konn3kt/hephaestus/ble_rpc_dispatch.cpython-312-aarch64-linux-gnu.so": "scXkv1O9w1hFvZmfoLWBSghQW3d7stQW/Tp8u09MaCaUyzIxEMySFMGWyuLsK/+Q72neAN9PfUD75l8CAZrzAA==",
+    "python/iqpilot_private/konn3kt/hephaestus/ble_transportd.cpython-312-aarch64-linux-gnu.so": "REH0sYrMI7XDjqxdVt+1odlWJa/xA+47aMZ/ow/tF6N6c8RTdEAQ3bse4ylh4zONQk+ozP1FP8KESPjCx9ImDQ==",
+    "python/iqpilot_private/konn3kt/hephaestus/bt_gamepad.cpython-312-aarch64-linux-gnu.so": "IFpGbURNm++Q+0+w/c26C7rGEA0aExvptQADdwGFpzOey8qn7Lm88w0dS4XrLmwYi46BjtbWm/nkN61steemBA==",
+    "python/iqpilot_private/konn3kt/hephaestus/cloud_routes.cpython-312-aarch64-linux-gnu.so": "lqzf8FZk6+1wSIHBIhRgzzv0Vd1DjvDOfVz0NdIV23y3l+2Eif/gkzUjuZpsXpIgIw9bBqh8L3PKXWW4jJFkBQ==",
+    "python/iqpilot_private/konn3kt/hephaestus/hephaestusd.cpython-312-aarch64-linux-gnu.so": "JbVCZ+G97IMYDzWnWP87Dk9mWnaWmxQXV41RWuDEwJWnM4HBi3+7HCziPo/GdEW9YpORyu1WBqg6Y5y0ijXlDA==",
+    "python/iqpilot_private/konn3kt/hephaestus/kwp2000.cpython-312-aarch64-linux-gnu.so": "1AOr8xUf6+iUX853cpCYEsdxKpeJom/u+neR4fh3pW/xMscieI7Y5I9DJgjdU4/7Zor2M1Lstxi5FDI45W8gDQ==",
+    "python/iqpilot_private/konn3kt/hephaestus/manage_hephaestusd.cpython-312-aarch64-linux-gnu.so": "lkHdhndsENTYxHNskHwva2FFmkQXs5vKKdSOcSbBmNWKDiYUSxNYzYBdUlYkYlBQqMZU6I57Dk54erq8Muj8AQ==",
+    "python/iqpilot_private/konn3kt/hephaestus/motd.cpython-312-aarch64-linux-gnu.so": "8ch+yvT6IdR63R6oSiqJnF6EVVZRiaNFk2QwHqYDNoT9o8h0i4/vssb6n86tVCj01ZLyT9rHvyznL36BittuAQ==",
+    "python/iqpilot_private/konn3kt/hephaestus/tp20.cpython-312-aarch64-linux-gnu.so": "aRWcvotSk/hilzlmj867MNro2kNalEd5ySGdqJkE7DKmGHxxCaG+x9E0AYTyvbzQs0ar59o9kQe31vL4Mr9ZDw==",
+    "python/iqpilot_private/konn3kt/hephaestus/vw_pq_flasher.cpython-312-aarch64-linux-gnu.so": "4SjG/+6RRSYaEVhsMnHTXRfnjnVstF2J8ttWQyfDYHhvFqmrshUf2d6r3aAZtNTDuqU3MgopfxZw7Rs/4OlqAw==",
+    "python/iqpilot_private/konn3kt/uploaderd/iquploaderd.cpython-312-aarch64-linux-gnu.so": "3U+IiMq2SN7NDiqwA8yFtM1eMq/64HnxDYVeeQWa7hdhwXoetfteI05VqUOG90GHgaGob026MP8f+v7rKd2WCA=="
   }
 }

system/hardware/hardwared.py

@@ -34,6 +34,10 @@ TEMP_TAU = 5.   # 5s time constant
 DISCONNECT_TIMEOUT = 5.  # wait 5 seconds before going offroad after disconnect so you get an alert
 PANDA_STATES_TIMEOUT = round(1000 / SERVICE_LIST['pandaStates'].frequency * 1.5)  # 1.5x the expected pandaState frequency
 ONROAD_CYCLE_TIME = 1  # seconds to wait offroad after requesting an onroad cycle
+CAN_STARTUP_RECOVERY_DELAY = 3.  # require a persistent CAN timeout before cycling onroad processes
+CAN_STARTUP_RECOVERY_WINDOW = 30.  # only recover shortly after ignition turns on
+CAN_STARTUP_RECOVERY_COOLDOWN = 5.  # allow the restarted car stack time to initialize
+CAN_STARTUP_RECOVERY_MAX_ATTEMPTS = 2
 
 ThermalBand = namedtuple("ThermalBand", ['min_temp', 'max_temp'])
 HardwareState = namedtuple("HardwareState", ['network_type', 'network_info', 'network_strength', 'network_stats',
@@ -55,6 +59,54 @@ prev_offroad_states: dict[str, tuple[bool, str | None]] = {}
 ALLOWED_TICI_BRANCHES = {"release-new", "release-tici", "master-mici", "beta", "beta-pq", "release-prebuilt"}
 
 
+class CanStartupRecovery:
+  """Bounded recovery for a car stack that starts without a usable CAN stream."""
+
+  def __init__(self) -> None:
+    self.ignition_on_ts: float | None = None
+    self.timeout_started_ts: float | None = None
+    self.last_attempt_ts: float | None = None
+    self.attempts = 0
+
+  def update(self, now: float, ignition: bool, started: bool, engaged: bool,
+             car_state_alive: bool, can_timeout: bool, v_ego: float) -> bool:
+    if not ignition:
+      self.ignition_on_ts = None
+      self.timeout_started_ts = None
+      self.last_attempt_ts = None
+      self.attempts = 0
+      return False
+
+    if self.ignition_on_ts is None:
+      self.ignition_on_ts = now
+
+    eligible = (
+      started
+      and not engaged
+      and car_state_alive
+      and can_timeout
+      and abs(v_ego) < 0.1
+      and (now - self.ignition_on_ts) <= CAN_STARTUP_RECOVERY_WINDOW
+      and self.attempts < CAN_STARTUP_RECOVERY_MAX_ATTEMPTS
+      and (self.last_attempt_ts is None or (now - self.last_attempt_ts) >= CAN_STARTUP_RECOVERY_COOLDOWN)
+    )
+    if not eligible:
+      self.timeout_started_ts = None
+      return False
+
+    if self.timeout_started_ts is None:
+      self.timeout_started_ts = now
+      return False
+
+    if (now - self.timeout_started_ts) < CAN_STARTUP_RECOVERY_DELAY:
+      return False
+
+    self.attempts += 1
+    self.last_attempt_ts = now
+    self.timeout_started_ts = None
+    return True
+
+
 def get_top_memory_processes(limit: int = 5) -> list[dict[str, object]]:
   procs: list[dict[str, object]] = []
   for proc in psutil.process_iter(['pid', 'name', 'memory_info', 'memory_percent']):
@@ -212,7 +264,7 @@ def hw_state_thread(end_event, hw_queue):
 
 def hardware_thread(end_event, hw_queue) -> None:
   pm = messaging.PubMaster(['deviceState', 'iqPerfTrace'])
-  sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "selfdriveState", "pandaStates"], poll="pandaStates")
+  sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "selfdriveState", "pandaStates", "carState"], poll="pandaStates")
   perf = PerfTraceEmitter("hardwared", pubmaster=pm)
 
   count = 0
@@ -250,6 +302,7 @@ def hardware_thread(end_event, hw_queue) -> None:
   low_power = False
   low_power_prev = False
   offroad_cycle_count = 0
+  can_startup_recovery = CanStartupRecovery()
 
   params = Params()
   power_monitor = PowerMonitoring()
@@ -274,6 +327,19 @@ def hardware_thread(end_event, hw_queue) -> None:
     if params.get_bool("OnroadCycleRequested"):
       params.put_bool("OnroadCycleRequested", False)
       offroad_cycle_count = sm.frame
+
+    car_state = sm['carState']
+    if can_startup_recovery.update(
+      time.monotonic(),
+      ignition=onroad_conditions["ignition"],
+      started=started_ts is not None,
+      engaged=sm['selfdriveState'].enabled,
+      car_state_alive=sm.alive['carState'],
+      can_timeout=car_state.canTimeout,
+      v_ego=car_state.vEgo,
+    ):
+      offroad_cycle_count = sm.frame
+      cloudlog.event("automatic CAN startup recovery", attempt=can_startup_recovery.attempts, error=True)
     onroad_conditions["not_onroad_cycle"] = (sm.frame - offroad_cycle_count) >= ONROAD_CYCLE_TIME * SERVICE_LIST['pandaStates'].frequency
 
     if sm.updated['pandaStates'] and len(pandaStates) > 0:

system/hardware/tests/test_hardwared.py

@@ -1,6 +1,13 @@
 from types import SimpleNamespace
 
-from openpilot.system.hardware.hardwared import ALLOWED_TICI_BRANCHES, is_supported_tici_branch
+from openpilot.system.hardware.hardwared import (
+  ALLOWED_TICI_BRANCHES,
+  CAN_STARTUP_RECOVERY_COOLDOWN,
+  CAN_STARTUP_RECOVERY_DELAY,
+  CAN_STARTUP_RECOVERY_MAX_ATTEMPTS,
+  CanStartupRecovery,
+  is_supported_tici_branch,
+)
 
 
 def test_beta_pq_allowed_for_tici():
@@ -17,3 +24,52 @@ def test_tici_channel_type_allowed():
 def test_unsupported_branch_rejected_for_tici():
   metadata = SimpleNamespace(channel="random-branch", channel_type="dev")
   assert not is_supported_tici_branch(metadata)
+
+
+def recovery_update(recovery: CanStartupRecovery, now: float, **kwargs) -> bool:
+  defaults = {
+    "ignition": True,
+    "started": True,
+    "engaged": False,
+    "car_state_alive": True,
+    "can_timeout": True,
+    "v_ego": 0.,
+  }
+  return recovery.update(now, **(defaults | kwargs))
+
+
+def test_can_startup_recovery_requires_persistent_timeout():
+  recovery = CanStartupRecovery()
+  assert not recovery_update(recovery, 10.)
+  assert not recovery_update(recovery, 10. + CAN_STARTUP_RECOVERY_DELAY - 0.1)
+  assert recovery_update(recovery, 10. + CAN_STARTUP_RECOVERY_DELAY)
+
+
+def test_can_startup_recovery_only_when_safe():
+  for unsafe_state in (
+    {"started": False},
+    {"engaged": True},
+    {"car_state_alive": False},
+    {"can_timeout": False},
+    {"v_ego": 0.2},
+  ):
+    recovery = CanStartupRecovery()
+    assert not recovery_update(recovery, 10., **unsafe_state)
+    assert not recovery_update(recovery, 10. + CAN_STARTUP_RECOVERY_DELAY, **unsafe_state)
+
+
+def test_can_startup_recovery_is_bounded_and_resets_next_ignition():
+  recovery = CanStartupRecovery()
+  now = 10.
+  for _ in range(CAN_STARTUP_RECOVERY_MAX_ATTEMPTS):
+    assert not recovery_update(recovery, now)
+    now += CAN_STARTUP_RECOVERY_DELAY
+    assert recovery_update(recovery, now)
+    now += CAN_STARTUP_RECOVERY_COOLDOWN
+
+  assert not recovery_update(recovery, now)
+  assert not recovery_update(recovery, now + CAN_STARTUP_RECOVERY_DELAY)
+
+  assert not recovery_update(recovery, now + 10., ignition=False)
+  assert not recovery_update(recovery, now + 11.)
+  assert recovery_update(recovery, now + 11. + CAN_STARTUP_RECOVERY_DELAY)