mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-08-21 07:03:44 +08:00
Merge branch 'master' into hkg-angle-steering-2025
# Conflicts: # common/params_keys.h # opendbc_repo
This commit is contained in:
@@ -107,7 +107,7 @@ def decoder(addr, vipc_server, vst, nvidia, W, H, debug=False):
|
||||
|
||||
|
||||
class CompressedVipc:
|
||||
def __init__(self, addr, vision_streams, nvidia=False, debug=False):
|
||||
def __init__(self, addr, vision_streams, server_name, nvidia=False, debug=False):
|
||||
print("getting frame sizes")
|
||||
os.environ["ZMQ"] = "1"
|
||||
messaging.reset_context()
|
||||
@@ -117,7 +117,7 @@ class CompressedVipc:
|
||||
os.environ.pop("ZMQ")
|
||||
messaging.reset_context()
|
||||
|
||||
self.vipc_server = VisionIpcServer("camerad")
|
||||
self.vipc_server = VisionIpcServer(server_name)
|
||||
for vst in vision_streams:
|
||||
ed = sm[ENCODE_SOCKETS[vst]]
|
||||
self.vipc_server.create_buffers(vst, 4, ed.width, ed.height)
|
||||
@@ -144,6 +144,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument("addr", help="Address of comma three")
|
||||
parser.add_argument("--nvidia", action="store_true", help="Use nvidia instead of ffmpeg")
|
||||
parser.add_argument("--cams", default="0,1,2", help="Cameras to decode")
|
||||
parser.add_argument("--server", default="camerad", help="choose vipc server name")
|
||||
parser.add_argument("--silent", action="store_true", help="Suppress debug output")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -154,7 +155,7 @@ if __name__ == "__main__":
|
||||
]
|
||||
|
||||
vsts = [vision_streams[int(x)] for x in args.cams.split(",")]
|
||||
cvipc = CompressedVipc(args.addr, vsts, args.nvidia, debug=(not args.silent))
|
||||
cvipc = CompressedVipc(args.addr, vsts, args.server, args.nvidia, debug=(not args.silent))
|
||||
|
||||
# register exit handler
|
||||
signal.signal(signal.SIGINT, lambda sig, frame: cvipc.kill())
|
||||
|
||||
+34
-53
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
@@ -11,13 +10,14 @@ from argparse import ArgumentParser, ArgumentTypeError
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from random import randint
|
||||
from subprocess import Popen, PIPE
|
||||
from subprocess import Popen
|
||||
from typing import Literal
|
||||
|
||||
from cereal.messaging import SubMaster
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params, UnknownKeyName
|
||||
from openpilot.common.prefix import OpenpilotPrefix
|
||||
from openpilot.common.run import managed_proc
|
||||
from openpilot.tools.lib.route import Route
|
||||
from openpilot.tools.lib.logreader import LogReader
|
||||
|
||||
@@ -38,22 +38,23 @@ UI = str(Path(BASEDIR, 'selfdrive/ui/ui').resolve())
|
||||
logger = logging.getLogger('clip.py')
|
||||
|
||||
|
||||
def check_for_failure(proc: Popen):
|
||||
exit_code = proc.poll()
|
||||
if exit_code is not None and exit_code != 0:
|
||||
cmd = str(proc.args)
|
||||
if isinstance(proc.args, str):
|
||||
cmd = proc.args
|
||||
elif isinstance(proc.args, Sequence):
|
||||
cmd = str(proc.args[0])
|
||||
msg = f'{cmd} failed, exit code {exit_code}'
|
||||
logger.error(msg)
|
||||
stdout, stderr = proc.communicate()
|
||||
if stdout:
|
||||
logger.error(stdout.decode())
|
||||
if stderr:
|
||||
logger.error(stderr.decode())
|
||||
raise ChildProcessError(msg)
|
||||
def check_for_failure(procs: list[Popen]):
|
||||
for proc in procs:
|
||||
exit_code = proc.poll()
|
||||
if exit_code is not None and exit_code != 0:
|
||||
cmd = str(proc.args)
|
||||
if isinstance(proc.args, str):
|
||||
cmd = proc.args
|
||||
elif isinstance(proc.args, Sequence):
|
||||
cmd = str(proc.args[0])
|
||||
msg = f'{cmd} failed, exit code {exit_code}'
|
||||
logger.error(msg)
|
||||
stdout, stderr = proc.communicate()
|
||||
if stdout:
|
||||
logger.error(stdout.decode())
|
||||
if stderr:
|
||||
logger.error(stderr.decode())
|
||||
raise ChildProcessError(msg)
|
||||
|
||||
|
||||
def escape_ffmpeg_text(value: str):
|
||||
@@ -137,10 +138,6 @@ def populate_car_params(lr: LogReader):
|
||||
logger.debug('persisted CarParams')
|
||||
|
||||
|
||||
def start_proc(args: list[str], env: dict[str, str]):
|
||||
return Popen(args, env=env, stdout=PIPE, stderr=PIPE)
|
||||
|
||||
|
||||
def validate_env(parser: ArgumentParser):
|
||||
if platform.system() not in ['Linux']:
|
||||
parser.exit(1, f'clip.py: error: {platform.system()} is not a supported operating system\n')
|
||||
@@ -176,8 +173,7 @@ def wait_for_frames(procs: list[Popen]):
|
||||
while no_frames_drawn:
|
||||
sm.update()
|
||||
no_frames_drawn = sm['uiDebug'].drawTimeMillis == 0.
|
||||
for proc in procs:
|
||||
check_for_failure(proc)
|
||||
check_for_failure(procs)
|
||||
|
||||
|
||||
def clip(
|
||||
@@ -253,35 +249,22 @@ def clip(
|
||||
|
||||
with OpenpilotPrefix(prefix, shared_download_cache=True):
|
||||
populate_car_params(lr)
|
||||
|
||||
env = os.environ.copy()
|
||||
env['DISPLAY'] = display
|
||||
|
||||
xvfb_proc = start_proc(xvfb_cmd, env)
|
||||
atexit.register(lambda: xvfb_proc.terminate())
|
||||
ui_proc = start_proc(ui_cmd, env)
|
||||
atexit.register(lambda: ui_proc.terminate())
|
||||
replay_proc = start_proc(replay_cmd, env)
|
||||
atexit.register(lambda: replay_proc.terminate())
|
||||
procs = [replay_proc, ui_proc, xvfb_proc]
|
||||
|
||||
logger.info('waiting for replay to begin (loading segments, may take a while)...')
|
||||
wait_for_frames(procs)
|
||||
|
||||
logger.debug(f'letting UI warm up ({SECONDS_TO_WARM}s)...')
|
||||
time.sleep(SECONDS_TO_WARM)
|
||||
for proc in procs:
|
||||
check_for_failure(proc)
|
||||
|
||||
ffmpeg_proc = start_proc(ffmpeg_cmd, env)
|
||||
procs.append(ffmpeg_proc)
|
||||
atexit.register(lambda: ffmpeg_proc.terminate())
|
||||
|
||||
logger.info(f'recording in progress ({duration}s)...')
|
||||
ffmpeg_proc.wait(duration + PROC_WAIT_SECONDS)
|
||||
for proc in procs:
|
||||
check_for_failure(proc)
|
||||
logger.info(f'recording complete: {Path(out).resolve()}')
|
||||
with managed_proc(xvfb_cmd, env) as xvfb_proc, managed_proc(ui_cmd, env) as ui_proc, managed_proc(replay_cmd, env) as replay_proc:
|
||||
procs = [xvfb_proc, ui_proc, replay_proc]
|
||||
logger.info('waiting for replay to begin (loading segments, may take a while)...')
|
||||
wait_for_frames(procs)
|
||||
logger.debug(f'letting UI warm up ({SECONDS_TO_WARM}s)...')
|
||||
time.sleep(SECONDS_TO_WARM)
|
||||
check_for_failure(procs)
|
||||
with managed_proc(ffmpeg_cmd, env) as ffmpeg_proc:
|
||||
procs.append(ffmpeg_proc)
|
||||
logger.info(f'recording in progress ({duration}s)...')
|
||||
ffmpeg_proc.wait(duration + PROC_WAIT_SECONDS)
|
||||
check_for_failure(procs)
|
||||
logger.info(f'recording complete: {Path(out).resolve()}')
|
||||
|
||||
|
||||
def main():
|
||||
@@ -319,9 +302,7 @@ def main():
|
||||
logger.exception('interrupted by user', exc_info=e)
|
||||
except Exception as e:
|
||||
logger.exception('encountered error', exc_info=e)
|
||||
finally:
|
||||
atexit._run_exitfuncs()
|
||||
sys.exit(exit_code)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
from collections.abc import Callable
|
||||
|
||||
from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url
|
||||
from openpilot.tools.lib.openpilotci import get_url
|
||||
from openpilot.tools.lib.filereader import DATA_ENDPOINT, file_exists, internal_source_available
|
||||
from openpilot.tools.lib.route import Route, SegmentRange, FileName
|
||||
|
||||
# When passed a tuple of file names, each source will return the first that exists (rlog.zst, rlog.bz2)
|
||||
FileNames = tuple[str, ...]
|
||||
Source = Callable[[SegmentRange, list[int], FileNames], dict[int, str]]
|
||||
|
||||
InternalUnavailableException = Exception("Internal source not available")
|
||||
|
||||
|
||||
def comma_api_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> dict[int, str]:
|
||||
route = Route(sr.route_name)
|
||||
|
||||
# comma api will have already checked if the file exists
|
||||
if fns == FileName.RLOG:
|
||||
return {seg: route.log_paths()[seg] for seg in seg_idxs if route.log_paths()[seg] is not None}
|
||||
else:
|
||||
return {seg: route.qlog_paths()[seg] for seg in seg_idxs if route.qlog_paths()[seg] is not None}
|
||||
|
||||
|
||||
def internal_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames, endpoint_url: str = DATA_ENDPOINT) -> dict[int, str]:
|
||||
if not internal_source_available(endpoint_url):
|
||||
raise InternalUnavailableException
|
||||
|
||||
def get_internal_url(sr: SegmentRange, seg, file):
|
||||
return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}"
|
||||
|
||||
return eval_source({seg: [get_internal_url(sr, seg, fn) for fn in fns] for seg in seg_idxs})
|
||||
|
||||
|
||||
def openpilotci_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> dict[int, str]:
|
||||
return eval_source({seg: [get_url(sr.route_name, seg, fn) for fn in fns] for seg in seg_idxs})
|
||||
|
||||
|
||||
def comma_car_segments_source(sr: SegmentRange, seg_idxs: list[int], fns: FileNames) -> dict[int, str]:
|
||||
return eval_source({seg: get_comma_segments_url(sr.route_name, seg) for seg in seg_idxs})
|
||||
|
||||
|
||||
def eval_source(files: dict[int, list[str] | str]) -> dict[int, str]:
|
||||
# Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst)
|
||||
valid_files: dict[int, str] = {}
|
||||
|
||||
for seg_idx, urls in files.items():
|
||||
if isinstance(urls, str):
|
||||
urls = [urls]
|
||||
|
||||
# Add first valid file URL
|
||||
for url in urls:
|
||||
if file_exists(url):
|
||||
valid_files[seg_idx] = url
|
||||
break
|
||||
|
||||
return valid_files
|
||||
+22
-74
@@ -12,16 +12,15 @@ import urllib.parse
|
||||
import warnings
|
||||
import zstandard as zstd
|
||||
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from collections.abc import Iterable, Iterator
|
||||
from typing import cast
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from cereal import log as capnp_log
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url
|
||||
from openpilot.tools.lib.openpilotci import get_url
|
||||
from openpilot.tools.lib.filereader import DATA_ENDPOINT, FileReader, file_exists, internal_source_available
|
||||
from openpilot.tools.lib.route import Route, SegmentRange, FileName
|
||||
from openpilot.tools.lib.filereader import FileReader
|
||||
from openpilot.tools.lib.file_sources import comma_api_source, internal_source, openpilotci_source, comma_car_segments_source, Source
|
||||
from openpilot.tools.lib.route import SegmentRange, FileName
|
||||
from openpilot.tools.lib.log_time_series import msgs_to_time_series
|
||||
|
||||
LogMessage = type[capnp._DynamicStructReader]
|
||||
@@ -40,6 +39,7 @@ def save_log(dest, log_msgs, compress=True):
|
||||
with open(dest, "wb") as f:
|
||||
f.write(dat)
|
||||
|
||||
|
||||
def decompress_stream(data: bytes):
|
||||
dctx = zstd.ZstdDecompressor()
|
||||
decompressed_data = b""
|
||||
@@ -139,73 +139,22 @@ class ReadMode(enum.StrEnum):
|
||||
AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user
|
||||
|
||||
|
||||
LogPath = str | None
|
||||
LogFileName = tuple[str, ...]
|
||||
Source = Callable[[SegmentRange, LogFileName], list[LogPath]]
|
||||
|
||||
InternalUnavailableException = Exception("Internal source not available")
|
||||
|
||||
|
||||
class LogsUnavailable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def comma_api_source(sr: SegmentRange, fns: LogFileName) -> list[LogPath]:
|
||||
route = Route(sr.route_name)
|
||||
|
||||
# comma api will have already checked if the file exists
|
||||
if fns == FileName.RLOG:
|
||||
return [route.log_paths()[seg] for seg in sr.seg_idxs]
|
||||
else:
|
||||
return [route.qlog_paths()[seg] for seg in sr.seg_idxs]
|
||||
|
||||
|
||||
def internal_source(sr: SegmentRange, fns: LogFileName, endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]:
|
||||
if not internal_source_available(endpoint_url):
|
||||
raise InternalUnavailableException
|
||||
|
||||
def get_internal_url(sr: SegmentRange, seg, file):
|
||||
return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}"
|
||||
|
||||
return eval_source([[get_internal_url(sr, seg, fn) for fn in fns] for seg in sr.seg_idxs])
|
||||
|
||||
|
||||
def openpilotci_source(sr: SegmentRange, fns: LogFileName) -> list[LogPath]:
|
||||
return eval_source([[get_url(sr.route_name, seg, fn) for fn in fns] for seg in sr.seg_idxs])
|
||||
|
||||
|
||||
def comma_car_segments_source(sr: SegmentRange, fns: LogFileName) -> list[LogPath]:
|
||||
return eval_source([get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs])
|
||||
|
||||
|
||||
def direct_source(file_or_url: str) -> list[str]:
|
||||
return [file_or_url]
|
||||
|
||||
|
||||
def eval_source(files: list[list[str] | str]) -> list[LogPath]:
|
||||
# Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst)
|
||||
valid_files: list[LogPath] = []
|
||||
|
||||
for urls in files:
|
||||
if isinstance(urls, str):
|
||||
urls = [urls]
|
||||
|
||||
for url in urls:
|
||||
if file_exists(url):
|
||||
valid_files.append(url)
|
||||
break
|
||||
else:
|
||||
valid_files.append(None)
|
||||
|
||||
return valid_files
|
||||
|
||||
|
||||
# TODO this should apply to camera files as well
|
||||
def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) -> list[str]:
|
||||
exceptions = {}
|
||||
|
||||
sr = SegmentRange(identifier)
|
||||
mode = default_mode if sr.selector is None else ReadMode(sr.selector)
|
||||
needed_seg_idxs = sr.seg_idxs
|
||||
|
||||
mode = default_mode if sr.selector is None else ReadMode(sr.selector)
|
||||
if mode == ReadMode.QLOG:
|
||||
try_fns = [FileName.QLOG]
|
||||
else:
|
||||
@@ -217,37 +166,35 @@ def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode)
|
||||
|
||||
# Build a dict of valid files as we evaluate each source. May contain mix of rlogs, qlogs, and None.
|
||||
# This function only returns when we've sourced all files, or throws an exception
|
||||
valid_files: dict[int, LogPath] = {}
|
||||
valid_files: dict[int, str] = {}
|
||||
for fn in try_fns:
|
||||
for source in sources:
|
||||
try:
|
||||
files = source(sr, fn)
|
||||
|
||||
# Check every source returns an expected number of files
|
||||
assert len(files) == len(valid_files) or len(valid_files) == 0, f"Source {source.__name__} returned unexpected number of files"
|
||||
files = source(sr, needed_seg_idxs, fn)
|
||||
|
||||
# Build a dict of valid files
|
||||
for idx, f in enumerate(files):
|
||||
if valid_files.get(idx) is None:
|
||||
valid_files[idx] = f
|
||||
valid_files |= files
|
||||
|
||||
# Don't check for segment files that have already been found
|
||||
needed_seg_idxs = [idx for idx in needed_seg_idxs if idx not in valid_files]
|
||||
|
||||
# We've found all files, return them
|
||||
if all(f is not None for f in valid_files.values()):
|
||||
if len(needed_seg_idxs) == 0:
|
||||
return cast(list[str], list(valid_files.values()))
|
||||
|
||||
except Exception as e:
|
||||
exceptions[source.__name__] = e
|
||||
|
||||
if fn == try_fns[0]:
|
||||
missing_logs = list(valid_files.values()).count(None)
|
||||
missing_logs = len(needed_seg_idxs)
|
||||
if mode == ReadMode.AUTO:
|
||||
cloudlog.warning(f"{missing_logs}/{len(valid_files)} rlogs were not found, falling back to qlogs for those segments...")
|
||||
cloudlog.warning(f"{missing_logs}/{len(sr.seg_idxs)} rlogs were not found, falling back to qlogs for those segments...")
|
||||
elif mode == ReadMode.AUTO_INTERACTIVE:
|
||||
if input(f"{missing_logs}/{len(valid_files)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y":
|
||||
if input(f"{missing_logs}/{len(sr.seg_idxs)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y":
|
||||
break
|
||||
|
||||
missing_logs = list(valid_files.values()).count(None)
|
||||
raise LogsUnavailable(f"{missing_logs}/{len(valid_files)} logs were not found, please ensure all logs " +
|
||||
missing_logs = len(needed_seg_idxs)
|
||||
raise LogsUnavailable(f"{missing_logs}/{len(sr.seg_idxs)} logs were not found, please ensure all logs " +
|
||||
"are uploaded. You can fall back to qlogs with '/a' selector at the end of the route name.\n\n" +
|
||||
"Exceptions for sources:\n - " + "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()]))
|
||||
|
||||
@@ -298,7 +245,7 @@ class LogReader:
|
||||
def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG,
|
||||
sources: list[Source] = None, sort_by_time=False, only_union_types=False):
|
||||
if sources is None:
|
||||
sources = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source]
|
||||
sources = [internal_source, comma_api_source, openpilotci_source, comma_car_segments_source]
|
||||
|
||||
self.default_mode = default_mode
|
||||
self.sources = sources
|
||||
@@ -351,6 +298,7 @@ class LogReader:
|
||||
def time_series(self):
|
||||
return msgs_to_time_series(self)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import codecs
|
||||
|
||||
|
||||
+3
-4
@@ -231,7 +231,6 @@ class RouteName:
|
||||
def __str__(self) -> str: return self._canonical_name
|
||||
|
||||
|
||||
|
||||
class SegmentName:
|
||||
# TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
|
||||
# of this class instead of manually constructing a segment name (use canonical_name prop instead)
|
||||
@@ -252,7 +251,7 @@ class SegmentName:
|
||||
@property
|
||||
def canonical_name(self) -> str: return self._canonical_name
|
||||
|
||||
#TODO should only use one name
|
||||
# TODO should only use one name
|
||||
@property
|
||||
def data_name(self) -> str: return f"{self._route_name.canonical_name}/{self._num}"
|
||||
|
||||
@@ -283,7 +282,7 @@ class SegmentName:
|
||||
@staticmethod
|
||||
def from_file_name(file_name):
|
||||
# ??????/xxxxxxxxxxxxxxxx|1111-11-11-11--11-11-11/1/rlog.bz2
|
||||
dongle_id, route_name, segment_num = file_name.replace('|','/').split('/')[-4:-1]
|
||||
dongle_id, route_name, segment_num = file_name.replace('|', '/').split('/')[-4:-1]
|
||||
return SegmentName(dongle_id + "|" + route_name + "--" + segment_num)
|
||||
|
||||
@staticmethod
|
||||
@@ -304,6 +303,7 @@ class SegmentName:
|
||||
dongle_id, route_name, segment_num = prefix.split("/")
|
||||
return SegmentName(dongle_id + "|" + route_name + "--" + segment_num)
|
||||
|
||||
|
||||
@cache
|
||||
def get_max_seg_number_cached(sr: 'SegmentRange') -> int:
|
||||
try:
|
||||
@@ -365,4 +365,3 @@ class SegmentRange:
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
|
||||
|
||||
@@ -10,7 +10,8 @@ import requests
|
||||
from parameterized import parameterized
|
||||
|
||||
from cereal import log as capnp_log
|
||||
from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException
|
||||
from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, parse_indirect, ReadMode
|
||||
from openpilot.tools.lib.file_sources import comma_api_source, InternalUnavailableException
|
||||
from openpilot.tools.lib.route import SegmentRange
|
||||
from openpilot.tools.lib.url_file import URLFileException
|
||||
|
||||
@@ -36,12 +37,12 @@ def setup_source_scenario(mocker, is_internal=False):
|
||||
comma_api_source_mock.__name__ = comma_api_source_mock._mock_name
|
||||
|
||||
if is_internal:
|
||||
internal_source_mock.return_value = [QLOG_FILE]
|
||||
internal_source_mock.return_value = {3: QLOG_FILE}
|
||||
else:
|
||||
internal_source_mock.side_effect = InternalUnavailableException
|
||||
|
||||
openpilotci_source_mock.return_value = [None]
|
||||
comma_api_source_mock.return_value = [QLOG_FILE]
|
||||
openpilotci_source_mock.return_value = {}
|
||||
comma_api_source_mock.return_value = {3: QLOG_FILE}
|
||||
|
||||
yield
|
||||
|
||||
@@ -90,7 +91,7 @@ class TestLogReader:
|
||||
|
||||
@pytest.mark.parametrize("cache_enabled", [True, False])
|
||||
def test_direct_parsing(self, mocker, cache_enabled):
|
||||
file_exists_mock = mocker.patch("openpilot.tools.lib.logreader.file_exists")
|
||||
file_exists_mock = mocker.patch("openpilot.tools.lib.filereader.file_exists")
|
||||
os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
|
||||
qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False)
|
||||
|
||||
@@ -208,13 +209,12 @@ class TestLogReader:
|
||||
assert qlog_len == log_len
|
||||
|
||||
@pytest.mark.parametrize("is_internal", [True, False])
|
||||
@pytest.mark.slow
|
||||
def test_auto_source_scenarios(self, mocker, is_internal):
|
||||
lr = LogReader(QLOG_FILE)
|
||||
qlog_len = len(list(lr))
|
||||
|
||||
with setup_source_scenario(mocker, is_internal=is_internal):
|
||||
lr = LogReader(f"{TEST_ROUTE}/0/q")
|
||||
lr = LogReader(f"{TEST_ROUTE}/3/q")
|
||||
log_len = len(list(lr))
|
||||
assert qlog_len == log_len
|
||||
|
||||
|
||||
+10
-8
@@ -9,12 +9,14 @@ from urllib3.util import Timeout
|
||||
|
||||
from openpilot.common.file_helpers import atomic_write_in_dir
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
# Cache chunk size
|
||||
K = 1000
|
||||
CHUNK_SIZE = 1000 * K
|
||||
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def hash_256(link: str) -> str:
|
||||
return sha256((link.split("?")[0]).encode('utf-8')).hexdigest()
|
||||
|
||||
@@ -24,7 +26,7 @@ class URLFileException(Exception):
|
||||
|
||||
|
||||
class URLFile:
|
||||
_pool_manager: PoolManager|None = None
|
||||
_pool_manager: PoolManager | None = None
|
||||
|
||||
@staticmethod
|
||||
def reset() -> None:
|
||||
@@ -33,16 +35,16 @@ class URLFile:
|
||||
@staticmethod
|
||||
def pool_manager() -> PoolManager:
|
||||
if URLFile._pool_manager is None:
|
||||
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),]
|
||||
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
|
||||
retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[409, 429, 503, 504])
|
||||
URLFile._pool_manager = PoolManager(num_pools=10, maxsize=100, socket_options=socket_options, retries=retries)
|
||||
return URLFile._pool_manager
|
||||
|
||||
def __init__(self, url: str, timeout: int=10, debug: bool=False, cache: bool|None=None):
|
||||
def __init__(self, url: str, timeout: int = 10, debug: bool = False, cache: bool | None = None):
|
||||
self._url = url
|
||||
self._timeout = Timeout(connect=timeout, read=timeout)
|
||||
self._pos = 0
|
||||
self._length: int|None = None
|
||||
self._length: int | None = None
|
||||
self._debug = debug
|
||||
# True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input
|
||||
self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0"))
|
||||
@@ -58,7 +60,7 @@ class URLFile:
|
||||
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
||||
pass
|
||||
|
||||
def _request(self, method: str, url: str, headers: dict[str, str]|None=None) -> BaseHTTPResponse:
|
||||
def _request(self, method: str, url: str, headers: dict[str, str] | None = None) -> BaseHTTPResponse:
|
||||
return URLFile.pool_manager().request(method, url, timeout=self._timeout, headers=headers)
|
||||
|
||||
def get_length_online(self) -> int:
|
||||
@@ -85,7 +87,7 @@ class URLFile:
|
||||
file_length.write(str(self._length))
|
||||
return self._length
|
||||
|
||||
def read(self, ll: int|None=None) -> bytes:
|
||||
def read(self, ll: int | None = None) -> bytes:
|
||||
if self._force_download:
|
||||
return self.read_aux(ll=ll)
|
||||
|
||||
@@ -117,7 +119,7 @@ class URLFile:
|
||||
self._pos = file_end
|
||||
return response
|
||||
|
||||
def read_aux(self, ll: int|None=None) -> bytes:
|
||||
def read_aux(self, ll: int | None = None) -> bytes:
|
||||
download_range = False
|
||||
headers = {}
|
||||
if self._pos != 0 or ll is not None:
|
||||
@@ -152,7 +154,7 @@ class URLFile:
|
||||
self._pos += len(ret)
|
||||
return ret
|
||||
|
||||
def seek(self, pos:int) -> None:
|
||||
def seek(self, pos: int) -> None:
|
||||
self._pos = pos
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user