From f562fdb05956038d7409b746c60193a038e38674 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 8 Dec 2020 22:01:34 -0800 Subject: [PATCH] add modelV2 to model replay (#2725) * add modelV2 to model replay * update refs * ignore fields for modelv2 * execution time * fix compare logs Co-authored-by: Comma Device old-commit-hash: 8992639754b3643896b289e37673f1fff5172309 --- selfdrive/test/openpilotci.py | 2 +- selfdrive/test/process_replay/camera_replay.py | 10 ++++++++-- selfdrive/test/process_replay/compare_logs.py | 2 +- selfdrive/test/process_replay/model_replay_ref_commit | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) mode change 100644 => 100755 selfdrive/test/openpilotci.py diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py old mode 100644 new mode 100755 index 6cda6e04e..1939677b1 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -16,7 +16,7 @@ def upload_file(path, name): sas_token = None if os.path.isfile(TOKEN_PATH): - sas_token = open(TOKEN_PATH).read().strip + sas_token = open(TOKEN_PATH).read().strip() if sas_token is None: sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \ diff --git a/selfdrive/test/process_replay/camera_replay.py b/selfdrive/test/process_replay/camera_replay.py index 853470c97..f64d615b9 100755 --- a/selfdrive/test/process_replay/camera_replay.py +++ b/selfdrive/test/process_replay/camera_replay.py @@ -35,9 +35,10 @@ def replace_calib(msg, calib): def camera_replay(lr, fr, desire=None, calib=None): spinner = Spinner() + spinner.update("starting model replay") pm = messaging.PubMaster(['frame', 'liveCalibration', 'pathPlan']) - sm = messaging.SubMaster(['model']) + sm = messaging.SubMaster(['model', 'modelV2']) # TODO: add dmonitoringmodeld print("preparing procs") @@ -48,6 +49,7 @@ def camera_replay(lr, fr, desire=None, calib=None): manager.start_managed_process("camerad") manager.start_managed_process("modeld") time.sleep(5) + sm.update(1000) print("procs started") desires_by_index = {v:k for k,v in log.PathPlan.Desire.schema.enumerants.items()} @@ -76,6 +78,7 @@ def camera_replay(lr, fr, desire=None, calib=None): pm.send(msg.which(), f) with Timeout(seconds=15): log_msgs.append(messaging.recv_one(sm.sock['model'])) + log_msgs.append(messaging.recv_one(sm.sock['modelV2'])) spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count)) @@ -108,8 +111,11 @@ if __name__ == "__main__": ref_commit = open(ref_commit_fn).read().strip() log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit) cmp_log = LogReader(BASE_URL + log_fn) + + ignore = ['logMonoTime', 'valid', 'model.frameDropPerc', 'model.modelExecutionTime', + 'modelV2.frameDropPerc', 'modelV2.modelExecutionTime'] results: Any = {TEST_ROUTE: {}} - results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid', 'model.frameDropPerc', 'model.modelExecutionTime']) + results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=ignore) diff1, diff2, failed = format_diff(results, ref_commit) print(diff1) diff --git a/selfdrive/test/process_replay/compare_logs.py b/selfdrive/test/process_replay/compare_logs.py index 397e6de56..f1b9fdea1 100755 --- a/selfdrive/test/process_replay/compare_logs.py +++ b/selfdrive/test/process_replay/compare_logs.py @@ -31,7 +31,7 @@ def remove_ignored_fields(msg, ignore): for key in ignore: attr = msg keys = key.split(".") - if msg.which() not in key and len(keys) > 1: + if msg.which() != keys[0] and len(keys) > 1: continue for k in keys[:-1]: diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit index 19f83517b..0ee29e4fb 100644 --- a/selfdrive/test/process_replay/model_replay_ref_commit +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -1 +1 @@ -f040d87da71d8d9b29000b0eeea43d4f932137dd \ No newline at end of file +857a00fc1093422907d49fc8647655de498f195b \ No newline at end of file