mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-08-21 00:03:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

regen: Refactor log migrate functions to avoid needing azure keys (#26049)

Browse Source 
* refactor migrate fns to avoid needing to use azure keys on import

* move azure key init behind a function

* resolve comments
old-commit-hash: 5f7d9a519e98b446b14866a8920dd0493b4dce26
This commit is contained in:
Vivek Aithal
2022-10-13 13:21:35 -07:00
committed by GitHub
parent b4f8289972
commit 7a826259c4
2 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selfdrive/test/process_replay/regen.py
+2 -1
View File
@@ -224,13 +224,14 @@ def migrate_sensorEvents(lr, old_logtime=False):
m_dat.sensor = evt.sensor
m_dat.type = evt.type
m_dat.source = evt.source
if old_logtime:
m_dat.timestamp = evt.timestamp
setattr(m_dat, evt.which(), getattr(evt, evt.which()))
all_msgs.append(m.as_reader())
return all_msgs
def regen_segment(lr, frs=None, outdir=FAKEDATA, disable_tqdm=False):
lr = migrate_carparams(list(lr))
lr = migrate_sensorEvents(list(lr))
selfdrive/test/update_ci_routes.py
+13 -7
View File
@@ -14,11 +14,16 @@ SOURCES = [
(_DATA_ACCOUNT_CI, "commadataci"),
]
DEST_KEY = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci")
SOURCE_KEYS = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES]
SERVICE = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=DEST_KEY)
def get_azure_keys():
dest_key = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci")
source_keys = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES]
service = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=dest_key)
return dest_key, source_keys, service
def upload_route(path, exclude_patterns=None):
dest_key, _, _ = get_azure_keys()
if exclude_patterns is None:
exclude_patterns = ['*/dcamera.hevc']
@@ -29,27 +34,28 @@ def upload_route(path, exclude_patterns=None):
"azcopy",
"copy",
f"{path}/*",
f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{DEST_KEY}",
f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{dest_key}",
"--recursive=false",
"--overwrite=false",
] + [f"--exclude-pattern={p}" for p in exclude_patterns]
subprocess.check_call(cmd)
def sync_to_ci_public(route):
dest_key, source_keys, service = get_azure_keys()
key_prefix = route.replace('|', '/')
dongle_id = key_prefix.split('/')[0]
if next(azureutil.list_all_blobs(SERVICE, "openpilotci", prefix=key_prefix), None) is not None:
if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None:
return True
print(f"Uploading {route}")
for (source_account, source_bucket), source_key in zip(SOURCES, SOURCE_KEYS):
for (source_account, source_bucket), source_key in zip(SOURCES, source_keys):
print(f"Trying {source_account}/{source_bucket}")
cmd = [
"azcopy",
"copy",
f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}",
f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{DEST_KEY}",
f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{dest_key}",
"--recursive=true",
"--overwrite=false",
"--exclude-pattern=*/dcamera.hevc",