plotjuggler: auto fallback to qlogs with prompt (#31110)

* qlog fallback

* show a warning for that

* qlog fallback

* fix

* wording

tools/lib/helpers.py

@@ -11,7 +11,7 @@ class RE:
   SEGMENT_NAME = r'{}(?:--|/)(?P<segment_num>[0-9]+)'.format(ROUTE_NAME)
   INDEX = r'-?[0-9]+'
   SLICE = r'(?P<start>{})?:?(?P<end>{})?:?(?P<step>{})?'.format(INDEX, INDEX, INDEX)
-  SEGMENT_RANGE = r'{}(?:--|/)?(?P<slice>({}))?/?(?P<selector>([qr]))?'.format(ROUTE_NAME, SLICE)
+  SEGMENT_RANGE = r'{}(?:--|/)?(?P<slice>({}))?/?(?P<selector>([qra]))?'.format(ROUTE_NAME, SLICE)
   BOOTLOG_NAME = ROUTE_NAME
 
   EXPLORER_FILE = r'^(?P<segment_name>{})--(?P<file_name>[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME)

tools/lib/logreader.py

@@ -16,6 +16,7 @@ from typing import Iterable, Iterator, List, Type
 from urllib.parse import parse_qs, urlparse
 
 from cereal import log as capnp_log
+from openpilot.common.swaglog import cloudlog
 from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url
 from openpilot.tools.lib.openpilotci import get_url
 from openpilot.tools.lib.filereader import FileReader, file_exists
@@ -71,8 +72,8 @@ class _LogFileReader:
 class ReadMode(enum.StrEnum):
   RLOG = "r" # only read rlogs
   QLOG = "q" # only read qlogs
-  #AUTO = "a" # default to rlogs, fallback to qlogs, not supported yet
-
+  AUTO = "a" # default to rlogs, fallback to qlogs
+  AUTO_INTERACIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user
 
 def create_slice_from_string(s: str):
   m = re.fullmatch(RE.SLICE, s)
@@ -86,31 +87,60 @@ def create_slice_from_string(s: str):
     return start
   return slice(start, end, step)
 
+def auto_strategy(rlog_paths, qlog_paths, interactive):
+  # auto select logs based on availability
+  if any(rlog is None or not file_exists(rlog) for rlog in rlog_paths):
+    if interactive:
+      if input("Some rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y":
+        return rlog_paths
+    else:
+      cloudlog.warning("Some rlogs were not found, falling back to qlogs for those segments...")
+
+    return [rlog if (rlog is not None and file_exists(rlog)) else (qlog if (qlog is not None and file_exists(qlog)) else None)
+                                                                for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)]
+  return rlog_paths
+
+def apply_strategy(mode: ReadMode, rlog_paths, qlog_paths):
+  if mode == ReadMode.RLOG:
+    return rlog_paths
+  elif mode == ReadMode.QLOG:
+    return qlog_paths
+  elif mode == ReadMode.AUTO:
+    return auto_strategy(rlog_paths, qlog_paths, False)
+  elif mode == ReadMode.AUTO_INTERACIVE:
+    return auto_strategy(rlog_paths, qlog_paths, True)
+
 def parse_slice(sr: SegmentRange, route: Route):
   segs = np.arange(route.max_seg_number+1)
   s = create_slice_from_string(sr._slice)
   return segs[s] if isinstance(s, slice) else [segs[s]]
 
-def comma_api_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG):
+def comma_api_source(sr: SegmentRange, route: Route, mode: ReadMode):
   segs = parse_slice(sr, route)
 
-  log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths()
+  rlog_paths = [route.log_paths()[seg] for seg in segs]
+  qlog_paths = [route.log_paths()[seg] for seg in segs]
 
-  invalid_segs = [seg for seg in segs if log_paths[seg] is None]
+  return apply_strategy(mode, rlog_paths, qlog_paths)
 
-  assert not len(invalid_segs), f"Some of the requested segments are not available: {invalid_segs}"
-
-  return [(log_paths[seg]) for seg in segs]
-
-def internal_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG):
+def internal_source(sr: SegmentRange, route: Route, mode: ReadMode):
   segs = parse_slice(sr, route)
 
-  return [f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2" for seg in segs]
+  def get_internal_url(sr: SegmentRange, seg, file):
+    return f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{file}.bz2"
 
-def openpilotci_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG):
+  rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in segs]
+  qlog_paths = [get_internal_url(sr, seg, "qlog")  for seg in segs]
+
+  return apply_strategy(mode, rlog_paths, qlog_paths)
+
+def openpilotci_source(sr: SegmentRange, route: Route, mode: ReadMode):
   segs = parse_slice(sr, route)
 
-  return [get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog') for seg in segs]
+  rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in segs]
+  qlog_paths = [get_url(sr.route_name, seg, "qlog")  for seg in segs]
+
+  return apply_strategy(mode, rlog_paths, qlog_paths)
 
 def comma_car_segments_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG):
   segs = parse_slice(sr, route)
@@ -120,10 +150,13 @@ def comma_car_segments_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG
 def direct_source(file_or_url):
   return [file_or_url]
 
+def get_invalid_files(files):
+  return [f for f in files if f is None or not file_exists(f)]
+
 def check_source(source, *args):
   try:
     files = source(*args)
-    assert all(file_exists(f) for f in files)
+    assert len(get_invalid_files(files)) == 0
     return True, files
   except Exception:
     return False, None
@@ -209,6 +242,9 @@ class LogReader:
 
   def reset(self):
     self.logreader_identifiers = self._parse_identifiers(self.identifier)
+    invalid_count = len(get_invalid_files(self.logreader_identifiers))
+    assert invalid_count == 0, f"{invalid_count}/{len(self.logreader_identifiers)} invalid log(s) found, please ensure all logs \
+are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name."
 
   @staticmethod
   def from_bytes(dat):

tools/plotjuggler/juggle.py

@@ -13,7 +13,7 @@ from functools import partial
 from openpilot.common.basedir import BASEDIR
 from openpilot.tools.lib.helpers import save_log
 
-from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.logreader import LogReader, ReadMode
 
 juggle_dir = os.path.dirname(os.path.realpath(__file__))
 
@@ -73,7 +73,7 @@ def process(can, lr):
   return [d for d in lr if can or d.which() not in ['can', 'sendcan']]
 
 def juggle_route(route_or_segment_name, can, layout, dbc=None):
-  sr = LogReader(route_or_segment_name)
+  sr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACIVE)
 
   all_data = sr.run_across_segments(24, partial(process, can))