Merge branch 'master' into hkg-angle-steering-2025

# Conflicts:
#	common/params_keys.h
#	opendbc_repo
This commit is contained in:
DevTekVE
2025-08-25 08:01:51 +02:00
84 changed files with 1424 additions and 962 deletions
+4 -3
View File
@@ -107,7 +107,7 @@ def decoder(addr, vipc_server, vst, nvidia, W, H, debug=False):
class CompressedVipc:
def __init__(self, addr, vision_streams, nvidia=False, debug=False):
def __init__(self, addr, vision_streams, server_name, nvidia=False, debug=False):
print("getting frame sizes")
os.environ["ZMQ"] = "1"
messaging.reset_context()
@@ -117,7 +117,7 @@ class CompressedVipc:
os.environ.pop("ZMQ")
messaging.reset_context()
self.vipc_server = VisionIpcServer("camerad")
self.vipc_server = VisionIpcServer(server_name)
for vst in vision_streams:
ed = sm[ENCODE_SOCKETS[vst]]
self.vipc_server.create_buffers(vst, 4, ed.width, ed.height)
@@ -144,6 +144,7 @@ if __name__ == "__main__":
parser.add_argument("addr", help="Address of comma three")
parser.add_argument("--nvidia", action="store_true", help="Use nvidia instead of ffmpeg")
parser.add_argument("--cams", default="0,1,2", help="Cameras to decode")
parser.add_argument("--server", default="camerad", help="choose vipc server name")
parser.add_argument("--silent", action="store_true", help="Suppress debug output")
args = parser.parse_args()
@@ -154,7 +155,7 @@ if __name__ == "__main__":
]
vsts = [vision_streams[int(x)] for x in args.cams.split(",")]
cvipc = CompressedVipc(args.addr, vsts, args.nvidia, debug=(not args.silent))
cvipc = CompressedVipc(args.addr, vsts, args.server, args.nvidia, debug=(not args.silent))
# register exit handler
signal.signal(signal.SIGINT, lambda sig, frame: cvipc.kill())
+34 -53
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python3
import atexit
import logging
import os
import platform
@@ -11,13 +10,14 @@ from argparse import ArgumentParser, ArgumentTypeError
from collections.abc import Sequence
from pathlib import Path
from random import randint
from subprocess import Popen, PIPE
from subprocess import Popen
from typing import Literal
from cereal.messaging import SubMaster
from openpilot.common.basedir import BASEDIR
from openpilot.common.params import Params, UnknownKeyName
from openpilot.common.prefix import OpenpilotPrefix
from openpilot.common.run import managed_proc
from openpilot.tools.lib.route import Route
from openpilot.tools.lib.logreader import LogReader
@@ -38,22 +38,23 @@ UI = str(Path(BASEDIR, 'selfdrive/ui/ui').resolve())
logger = logging.getLogger('clip.py')
def check_for_failure(proc: Popen):
exit_code = proc.poll()
if exit_code is not None and exit_code != 0:
cmd = str(proc.args)
if isinstance(proc.args, str):
cmd = proc.args
elif isinstance(proc.args, Sequence):
cmd = str(proc.args[0])
msg = f'{cmd} failed, exit code {exit_code}'
logger.error(msg)
stdout, stderr = proc.communicate()
if stdout:
logger.error(stdout.decode())
if stderr:
logger.error(stderr.decode())
raise ChildProcessError(msg)
def check_for_failure(procs: list[Popen]):
for proc in procs:
exit_code = proc.poll()
if exit_code is not None and exit_code != 0:
cmd = str(proc.args)
if isinstance(proc.args, str):
cmd = proc.args
elif isinstance(proc.args, Sequence):
cmd = str(proc.args[0])
msg = f'{cmd} failed, exit code {exit_code}'
logger.error(msg)
stdout, stderr = proc.communicate()
if stdout:
logger.error(stdout.decode())
if stderr:
logger.error(stderr.decode())
raise ChildProcessError(msg)
def escape_ffmpeg_text(value: str):
@@ -137,10 +138,6 @@ def populate_car_params(lr: LogReader):
logger.debug('persisted CarParams')
def start_proc(args: list[str], env: dict[str, str]):
return Popen(args, env=env, stdout=PIPE, stderr=PIPE)
def validate_env(parser: ArgumentParser):
if platform.system() not in ['Linux']:
parser.exit(1, f'clip.py: error: {platform.system()} is not a supported operating system\n')
@@ -176,8 +173,7 @@ def wait_for_frames(procs: list[Popen]):
while no_frames_drawn:
sm.update()
no_frames_drawn = sm['uiDebug'].drawTimeMillis == 0.
for proc in procs:
check_for_failure(proc)
check_for_failure(procs)
def clip(
@@ -253,35 +249,22 @@ def clip(
with OpenpilotPrefix(prefix, shared_download_cache=True):
populate_car_params(lr)
env = os.environ.copy()
env['DISPLAY'] = display
xvfb_proc = start_proc(xvfb_cmd, env)
atexit.register(lambda: xvfb_proc.terminate())
ui_proc = start_proc(ui_cmd, env)
atexit.register(lambda: ui_proc.terminate())
replay_proc = start_proc(replay_cmd, env)
atexit.register(lambda: replay_proc.terminate())
procs = [replay_proc, ui_proc, xvfb_proc]
logger.info('waiting for replay to begin (loading segments, may take a while)...')
wait_for_frames(procs)
logger.debug(f'letting UI warm up ({SECONDS_TO_WARM}s)...')
time.sleep(SECONDS_TO_WARM)
for proc in procs:
check_for_failure(proc)
ffmpeg_proc = start_proc(ffmpeg_cmd, env)
procs.append(ffmpeg_proc)
atexit.register(lambda: ffmpeg_proc.terminate())
logger.info(f'recording in progress ({duration}s)...')
ffmpeg_proc.wait(duration + PROC_WAIT_SECONDS)
for proc in procs:
check_for_failure(proc)
logger.info(f'recording complete: {Path(out).resolve()}')
with managed_proc(xvfb_cmd, env) as xvfb_proc, managed_proc(ui_cmd, env) as ui_proc, managed_proc(replay_cmd, env) as replay_proc:
procs = [xvfb_proc, ui_proc, replay_proc]
logger.info('waiting for replay to begin (loading segments, may take a while)...')
wait_for_frames(procs)
logger.debug(f'letting UI warm up ({SECONDS_TO_WARM}s)...')
time.sleep(SECONDS_TO_WARM)
check_for_failure(procs)
with managed_proc(ffmpeg_cmd, env) as ffmpeg_proc:
procs.append(ffmpeg_proc)
logger.info(f'recording in progress ({duration}s)...')
ffmpeg_proc.wait(duration + PROC_WAIT_SECONDS)
check_for_failure(procs)
logger.info(f'recording complete: {Path(out).resolve()}')
def main():
@@ -319,9 +302,7 @@ def main():
logger.exception('interrupted by user', exc_info=e)
except Exception as e:
logger.exception('encountered error', exc_info=e)
finally:
atexit._run_exitfuncs()
sys.exit(exit_code)
sys.exit(exit_code)
if __name__ == '__main__':
+57
View File
@@ -0,0 +1,57 @@
from collections.abc import Callable
from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url
from openpilot.tools.lib.openpilotci import get_url
from openpilot.tools.lib.filereader import DATA_ENDPOINT, file_exists, internal_source_available
from openpilot.tools.lib.route import Route, SegmentRange, FileName
# When passed a tuple of file names, each source will return the first that exists (rlog.zst, rlog.bz2)
FileNames = tuple[str, ...]
Source = Callable[[SegmentRange, list[int], FileNames], dict[int, str]]
InternalUnavailableException = Exception("Internal source not available")
def comma_api_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> dict[int, str]:
route = Route(sr.route_name)
# comma api will have already checked if the file exists
if fns == FileName.RLOG:
return {seg: route.log_paths()[seg] for seg in seg_idxs if route.log_paths()[seg] is not None}
else:
return {seg: route.qlog_paths()[seg] for seg in seg_idxs if route.qlog_paths()[seg] is not None}
def internal_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames, endpoint_url: str = DATA_ENDPOINT) -> dict[int, str]:
if not internal_source_available(endpoint_url):
raise InternalUnavailableException
def get_internal_url(sr: SegmentRange, seg, file):
return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}"
return eval_source({seg: [get_internal_url(sr, seg, fn) for fn in fns] for seg in seg_idxs})
def openpilotci_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> dict[int, str]:
return eval_source({seg: [get_url(sr.route_name, seg, fn) for fn in fns] for seg in seg_idxs})
def comma_car_segments_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> dict[int, str]:
return eval_source({seg: get_comma_segments_url(sr.route_name, seg) for seg in seg_idxs})
def eval_source(files: dict[int, list[str] | str]) -> dict[int, str]:
# Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst)
valid_files: dict[int, str] = {}
for seg_idx, urls in files.items():
if isinstance(urls, str):
urls = [urls]
# Add first valid file URL
for url in urls:
if file_exists(url):
valid_files[seg_idx] = url
break
return valid_files
+22 -74
View File
@@ -12,16 +12,15 @@ import urllib.parse
import warnings
import zstandard as zstd
from collections.abc import Callable, Iterable, Iterator
from collections.abc import Iterable, Iterator
from typing import cast
from urllib.parse import parse_qs, urlparse
from cereal import log as capnp_log
from openpilot.common.swaglog import cloudlog
from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url
from openpilot.tools.lib.openpilotci import get_url
from openpilot.tools.lib.filereader import DATA_ENDPOINT, FileReader, file_exists, internal_source_available
from openpilot.tools.lib.route import Route, SegmentRange, FileName
from openpilot.tools.lib.filereader import FileReader
from openpilot.tools.lib.file_sources import comma_api_source, internal_source, openpilotci_source, comma_car_segments_source, Source
from openpilot.tools.lib.route import SegmentRange, FileName
from openpilot.tools.lib.log_time_series import msgs_to_time_series
LogMessage = type[capnp._DynamicStructReader]
@@ -40,6 +39,7 @@ def save_log(dest, log_msgs, compress=True):
with open(dest, "wb") as f:
f.write(dat)
def decompress_stream(data: bytes):
dctx = zstd.ZstdDecompressor()
decompressed_data = b""
@@ -139,73 +139,22 @@ class ReadMode(enum.StrEnum):
AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user
LogPath = str | None
LogFileName = tuple[str, ...]
Source = Callable[[SegmentRange, LogFileName], list[LogPath]]
InternalUnavailableException = Exception("Internal source not available")
class LogsUnavailable(Exception):
pass
def comma_api_source(sr: SegmentRange, fns: LogFileName) -> list[LogPath]:
route = Route(sr.route_name)
# comma api will have already checked if the file exists
if fns == FileName.RLOG:
return [route.log_paths()[seg] for seg in sr.seg_idxs]
else:
return [route.qlog_paths()[seg] for seg in sr.seg_idxs]
def internal_source(sr: SegmentRange, fns: LogFileName, endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]:
if not internal_source_available(endpoint_url):
raise InternalUnavailableException
def get_internal_url(sr: SegmentRange, seg, file):
return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}"
return eval_source([[get_internal_url(sr, seg, fn) for fn in fns] for seg in sr.seg_idxs])
def openpilotci_source(sr: SegmentRange, fns: LogFileName) -> list[LogPath]:
return eval_source([[get_url(sr.route_name, seg, fn) for fn in fns] for seg in sr.seg_idxs])
def comma_car_segments_source(sr: SegmentRange, fns: LogFileName) -> list[LogPath]:
return eval_source([get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs])
def direct_source(file_or_url: str) -> list[str]:
return [file_or_url]
def eval_source(files: list[list[str] | str]) -> list[LogPath]:
# Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst)
valid_files: list[LogPath] = []
for urls in files:
if isinstance(urls, str):
urls = [urls]
for url in urls:
if file_exists(url):
valid_files.append(url)
break
else:
valid_files.append(None)
return valid_files
# TODO this should apply to camera files as well
def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) -> list[str]:
exceptions = {}
sr = SegmentRange(identifier)
mode = default_mode if sr.selector is None else ReadMode(sr.selector)
needed_seg_idxs = sr.seg_idxs
mode = default_mode if sr.selector is None else ReadMode(sr.selector)
if mode == ReadMode.QLOG:
try_fns = [FileName.QLOG]
else:
@@ -217,37 +166,35 @@ def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode)
# Build a dict of valid files as we evaluate each source. May contain mix of rlogs, qlogs, and None.
# This function only returns when we've sourced all files, or throws an exception
valid_files: dict[int, LogPath] = {}
valid_files: dict[int, str] = {}
for fn in try_fns:
for source in sources:
try:
files = source(sr, fn)
# Check every source returns an expected number of files
assert len(files) == len(valid_files) or len(valid_files) == 0, f"Source {source.__name__} returned unexpected number of files"
files = source(sr, needed_seg_idxs, fn)
# Build a dict of valid files
for idx, f in enumerate(files):
if valid_files.get(idx) is None:
valid_files[idx] = f
valid_files |= files
# Don't check for segment files that have already been found
needed_seg_idxs = [idx for idx in needed_seg_idxs if idx not in valid_files]
# We've found all files, return them
if all(f is not None for f in valid_files.values()):
if len(needed_seg_idxs) == 0:
return cast(list[str], list(valid_files.values()))
except Exception as e:
exceptions[source.__name__] = e
if fn == try_fns[0]:
missing_logs = list(valid_files.values()).count(None)
missing_logs = len(needed_seg_idxs)
if mode == ReadMode.AUTO:
cloudlog.warning(f"{missing_logs}/{len(valid_files)} rlogs were not found, falling back to qlogs for those segments...")
cloudlog.warning(f"{missing_logs}/{len(sr.seg_idxs)} rlogs were not found, falling back to qlogs for those segments...")
elif mode == ReadMode.AUTO_INTERACTIVE:
if input(f"{missing_logs}/{len(valid_files)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y":
if input(f"{missing_logs}/{len(sr.seg_idxs)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y":
break
missing_logs = list(valid_files.values()).count(None)
raise LogsUnavailable(f"{missing_logs}/{len(valid_files)} logs were not found, please ensure all logs " +
missing_logs = len(needed_seg_idxs)
raise LogsUnavailable(f"{missing_logs}/{len(sr.seg_idxs)} logs were not found, please ensure all logs " +
"are uploaded. You can fall back to qlogs with '/a' selector at the end of the route name.\n\n" +
"Exceptions for sources:\n - " + "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()]))
@@ -298,7 +245,7 @@ class LogReader:
def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG,
sources: list[Source] = None, sort_by_time=False, only_union_types=False):
if sources is None:
sources = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source]
sources = [internal_source, comma_api_source, openpilotci_source, comma_car_segments_source]
self.default_mode = default_mode
self.sources = sources
@@ -351,6 +298,7 @@ class LogReader:
def time_series(self):
return msgs_to_time_series(self)
if __name__ == "__main__":
import codecs
+3 -4
View File
@@ -231,7 +231,6 @@ class RouteName:
def __str__(self) -> str: return self._canonical_name
class SegmentName:
# TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
# of this class instead of manually constructing a segment name (use canonical_name prop instead)
@@ -252,7 +251,7 @@ class SegmentName:
@property
def canonical_name(self) -> str: return self._canonical_name
#TODO should only use one name
# TODO should only use one name
@property
def data_name(self) -> str: return f"{self._route_name.canonical_name}/{self._num}"
@@ -283,7 +282,7 @@ class SegmentName:
@staticmethod
def from_file_name(file_name):
# ??????/xxxxxxxxxxxxxxxx|1111-11-11-11--11-11-11/1/rlog.bz2
dongle_id, route_name, segment_num = file_name.replace('|','/').split('/')[-4:-1]
dongle_id, route_name, segment_num = file_name.replace('|', '/').split('/')[-4:-1]
return SegmentName(dongle_id + "|" + route_name + "--" + segment_num)
@staticmethod
@@ -304,6 +303,7 @@ class SegmentName:
dongle_id, route_name, segment_num = prefix.split("/")
return SegmentName(dongle_id + "|" + route_name + "--" + segment_num)
@cache
def get_max_seg_number_cached(sr: 'SegmentRange') -> int:
try:
@@ -365,4 +365,3 @@ class SegmentRange:
def __repr__(self) -> str:
return self.__str__()
+7 -7
View File
@@ -10,7 +10,8 @@ import requests
from parameterized import parameterized
from cereal import log as capnp_log
from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException
from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, parse_indirect, ReadMode
from openpilot.tools.lib.file_sources import comma_api_source, InternalUnavailableException
from openpilot.tools.lib.route import SegmentRange
from openpilot.tools.lib.url_file import URLFileException
@@ -36,12 +37,12 @@ def setup_source_scenario(mocker, is_internal=False):
comma_api_source_mock.__name__ = comma_api_source_mock._mock_name
if is_internal:
internal_source_mock.return_value = [QLOG_FILE]
internal_source_mock.return_value = {3: QLOG_FILE}
else:
internal_source_mock.side_effect = InternalUnavailableException
openpilotci_source_mock.return_value = [None]
comma_api_source_mock.return_value = [QLOG_FILE]
openpilotci_source_mock.return_value = {}
comma_api_source_mock.return_value = {3: QLOG_FILE}
yield
@@ -90,7 +91,7 @@ class TestLogReader:
@pytest.mark.parametrize("cache_enabled", [True, False])
def test_direct_parsing(self, mocker, cache_enabled):
file_exists_mock = mocker.patch("openpilot.tools.lib.logreader.file_exists")
file_exists_mock = mocker.patch("openpilot.tools.lib.filereader.file_exists")
os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False)
@@ -208,13 +209,12 @@ class TestLogReader:
assert qlog_len == log_len
@pytest.mark.parametrize("is_internal", [True, False])
@pytest.mark.slow
def test_auto_source_scenarios(self, mocker, is_internal):
lr = LogReader(QLOG_FILE)
qlog_len = len(list(lr))
with setup_source_scenario(mocker, is_internal=is_internal):
lr = LogReader(f"{TEST_ROUTE}/0/q")
lr = LogReader(f"{TEST_ROUTE}/3/q")
log_len = len(list(lr))
assert qlog_len == log_len
+10 -8
View File
@@ -9,12 +9,14 @@ from urllib3.util import Timeout
from openpilot.common.file_helpers import atomic_write_in_dir
from openpilot.system.hardware.hw import Paths
# Cache chunk size
K = 1000
CHUNK_SIZE = 1000 * K
logging.getLogger("urllib3").setLevel(logging.WARNING)
def hash_256(link: str) -> str:
return sha256((link.split("?")[0]).encode('utf-8')).hexdigest()
@@ -24,7 +26,7 @@ class URLFileException(Exception):
class URLFile:
_pool_manager: PoolManager|None = None
_pool_manager: PoolManager | None = None
@staticmethod
def reset() -> None:
@@ -33,16 +35,16 @@ class URLFile:
@staticmethod
def pool_manager() -> PoolManager:
if URLFile._pool_manager is None:
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),]
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[409, 429, 503, 504])
URLFile._pool_manager = PoolManager(num_pools=10, maxsize=100, socket_options=socket_options, retries=retries)
return URLFile._pool_manager
def __init__(self, url: str, timeout: int=10, debug: bool=False, cache: bool|None=None):
def __init__(self, url: str, timeout: int = 10, debug: bool = False, cache: bool | None = None):
self._url = url
self._timeout = Timeout(connect=timeout, read=timeout)
self._pos = 0
self._length: int|None = None
self._length: int | None = None
self._debug = debug
# True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input
self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0"))
@@ -58,7 +60,7 @@ class URLFile:
def __exit__(self, exc_type, exc_value, traceback) -> None:
pass
def _request(self, method: str, url: str, headers: dict[str, str]|None=None) -> BaseHTTPResponse:
def _request(self, method: str, url: str, headers: dict[str, str] | None = None) -> BaseHTTPResponse:
return URLFile.pool_manager().request(method, url, timeout=self._timeout, headers=headers)
def get_length_online(self) -> int:
@@ -85,7 +87,7 @@ class URLFile:
file_length.write(str(self._length))
return self._length
def read(self, ll: int|None=None) -> bytes:
def read(self, ll: int | None = None) -> bytes:
if self._force_download:
return self.read_aux(ll=ll)
@@ -117,7 +119,7 @@ class URLFile:
self._pos = file_end
return response
def read_aux(self, ll: int|None=None) -> bytes:
def read_aux(self, ll: int | None = None) -> bytes:
download_range = False
headers = {}
if self._pos != 0 or ll is not None:
@@ -152,7 +154,7 @@ class URLFile:
self._pos += len(ret)
return ret
def seek(self, pos:int) -> None:
def seek(self, pos: int) -> None:
self._pos = pos
@property