azure_container: pass the filename explicitely when using get_url (#33407)

* Pass filename explicitely when using azure container

* Do not update segments in test_processes

* openpilotci_source_zst

* Fix static analysisc
old-commit-hash: b5958ebb600f503f585a1080236a0f26b4f5c6a0
This commit is contained in:
Kacper Rączy
2024-08-29 21:51:55 -07:00
committed by GitHub
parent 6bb9e72c64
commit 786aff27da
6 changed files with 22 additions and 18 deletions
+2 -3
View File
@@ -53,9 +53,8 @@ class AzureContainer:
key = get_container_sas(self.ACCOUNT, self.CONTAINER)
return client, key
def get_url(self, route_name: str, segment_num, log_type="rlog") -> str:
ext = "hevc" if log_type.endswith('camera') else "bz2"
return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}"
def get_url(self, route_name: str, segment_num: str, filename: str) -> str:
return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}"
def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str:
from azure.storage.blob import BlobClient
+9 -4
View File
@@ -163,13 +163,17 @@ def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst")
return internal_source(sr, mode, file_ext)
def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs]
def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths:
rlog_paths = [get_url(sr.route_name, seg, f"rlog.{file_ext}") for seg in sr.seg_idxs]
qlog_paths = [get_url(sr.route_name, seg, f"qlog.{file_ext}") for seg in sr.seg_idxs]
return apply_strategy(mode, rlog_paths, qlog_paths)
def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> LogPaths:
return openpilotci_source(sr, mode, "zst")
def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs]
@@ -201,7 +205,8 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
if mode == ReadMode.SANITIZED:
return comma_car_segments_source(sr, mode)
SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source, testing_closet_source,]
SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst,
comma_api_source, comma_car_segments_source, testing_closet_source,]
exceptions = {}
# for automatic fallback modes, auto_source needs to first check if rlogs exist for any source
+2 -2
View File
@@ -24,8 +24,8 @@ def input_ready():
def replay(route, segment, loop):
route = route.replace('|', '/')
lr = LogReader(get_url(route, segment))
fr = FrameReader(get_url(route, segment, "fcamera"), readahead=True)
lr = LogReader(get_url(route, segment, "rlog.bz2"))
fr = FrameReader(get_url(route, segment, "fcamera.hevc"), readahead=True)
# Build mapping from frameId to segmentId from roadEncodeIdx, type == fullHEVC
msgs = [m for m in lr if m.which() not in IGNORE]