Panda logs (#28511)

* log in pandaStates

* Revert "log in pandaStates"

This reverts commit 87ee283ac45c2eeed94d9a958927e3072ec4460f.

* log to param

* dumps datetime

* cloudlog instead

* add truncation

* fix

* try isotimestamp

* duh

* more fixes

* should be __class__

* add some time to account for log gathering

* use new fast interface

* Revert "add some time to account for log gathering"

This reverts commit 2c08ad48052dfc0546530059a0cda03b04c4939a.

* cleanup

* simple test

* fix wrong serial

---------

Co-authored-by: Comma Device <device@comma.ai>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

common/params.cc

@@ -177,6 +177,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
     {"OpenpilotEnabledToggle", PERSISTENT},
     {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaLogState", PERSISTENT},
     {"PandaSignatures", CLEAR_ON_MANAGER_START},
     {"Passive", PERSISTENT},
     {"PrimeType", PERSISTENT},

selfdrive/boardd/pandad.py

@@ -3,6 +3,7 @@
 import os
 import usb1
 import time
+import json
 import subprocess
 from typing import List, NoReturn
 from functools import cmp_to_key
@@ -23,6 +24,48 @@ def get_expected_signature(panda: Panda) -> bytes:
     cloudlog.exception("Error computing expected signature")
     return b""
 
+def read_panda_logs(panda: Panda) -> None:
+  """
+    Forward panda logs to the cloud
+  """
+
+  params = Params()
+  serial = panda.get_usb_serial()
+
+  log_state = {}
+  try:
+    l = json.loads(params.get("PandaLogState"))
+    for k, v in l.items():
+      if isinstance(k, str) and isinstance(v, int):
+        log_state[k] = v
+  except (TypeError, json.JSONDecodeError):
+    cloudlog.exception("failed to parse PandaLogState")
+
+  try:
+    if serial in log_state:
+      logs = panda.get_logs(last_id=log_state[serial])
+    else:
+      logs = panda.get_logs(get_all=True)
+
+    # truncate logs to 100 entries if needed
+    MAX_LOGS = 100
+    if len(logs) > MAX_LOGS:
+      cloudlog.warning(f"Panda {serial} has {len(logs)} logs, truncating to {MAX_LOGS}")
+      logs = logs[-MAX_LOGS:]
+
+    # update log state
+    if len(logs) > 0:
+      log_state[serial] = logs[-1]["id"]
+
+    for log in logs:
+      if log['timestamp'] is not None:
+        log['timestamp'] = log['timestamp'].isoformat()
+      cloudlog.event("panda_log", **log, serial=serial)
+
+    params.put("PandaLogState", json.dumps(log_state))
+  except Exception:
+    cloudlog.exception(f"Error getting logs for panda {serial}")
+
 
 def flash_panda(panda_serial: str) -> Panda:
   try:
@@ -47,7 +90,6 @@ def flash_panda(panda_serial: str) -> Panda:
   if panda.bootstub:
     bootstub_version = panda.get_version()
     cloudlog.info(f"Flashed firmware not booting, flashing development bootloader. {bootstub_version=}, {internal_panda=}")
-
     if internal_panda:
       HARDWARE.recover_internal_panda()
     panda.recover(reset=(not internal_panda))
@@ -139,6 +181,8 @@ def main() -> NoReturn:
           params.put_bool("PandaHeartbeatLost", True)
           cloudlog.event("heartbeat lost", deviceState=health, serial=panda.get_usb_serial())
 
+        read_panda_logs(panda)
+
         if first_run:
           if panda.is_internal():
             # update time from RTC

selfdrive/boardd/tests/test_pandad.py

@@ -6,6 +6,7 @@ import unittest
 import cereal.messaging as messaging
 from cereal import log
 from common.gpio import gpio_set, gpio_init
+from common.params import Params
 from panda import Panda, PandaDFU, PandaProtocolMismatch
 from selfdrive.test.helpers import phone_only
 from selfdrive.manager.process_config import managed_processes
@@ -17,6 +18,10 @@ HERE = os.path.dirname(os.path.realpath(__file__))
 
 class TestPandad(unittest.TestCase):
 
+  def setUp(self):
+    self.params = Params()
+    self.start_log_state = self.params.get("PandaLogState")
+
   def tearDown(self):
     managed_processes['pandad'].stop()
 
@@ -30,6 +35,10 @@ class TestPandad(unittest.TestCase):
     if sm['peripheralState'].pandaType == log.PandaState.PandaType.unknown:
       raise Exception("boardd failed to start")
 
+    # simple check that we did something with the panda logs
+    cur_log_state = self.params.get("PandaLogState")
+    assert cur_log_state != self.start_log_state
+
   def _go_to_dfu(self):
     HARDWARE.recover_internal_panda()
     assert Panda.wait_for_dfu(None, 10)