Model + camerad test (#1622)

* kind of works

* move that

* hack to get camerad to reliably terminate

* not sure why SIGTERM wasn't working before

* compare bytes

* clean up some hacks

* gitignore

* fix that

* WIP

* no reboot

* comparison works

* pretty print

* fix build

* run in jenkins

* python path

* space

* raise timeout

* new eon

* skip the copy

* spinner

* spin less

* update model ref commit

* reenable that

* clean up

* fix jenkinsfile

* parallel

* wrap it in a stage

* fix linter

* better progress

* lower timeout

Co-authored-by: Comma Device <device@comma.ai>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 63c8e8439b

selfdrive/test/.gitignore

@@ -1,2 +1,8 @@
 out/
 docker_out/
+
+process_replay/diff.txt
+process_replay/model_diff.txt
+
+*.bz2
+*.hevc

selfdrive/test/openpilotci.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+
+BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
+
+def get_url(route_name, segment_num, log_type="rlog"):
+  ext = "hevc" if log_type in ["fcamera", "dcamera"] else "bz2"
+  return BASE_URL + "%s/%s/%s.%s" % (route_name.replace("|", "/"), segment_num, log_type, ext)
+
+def upload_file(path, name):
+  from azure.storage.blob import BlockBlobService
+  sas_token = os.getenv("TOKEN", None)
+  if sas_token is None:
+    sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \
+                                         --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n")
+  service = BlockBlobService(account_name="commadataci", sas_token=sas_token)
+  service.create_blob_from_path("openpilotci", name, path)
+  return "https://commadataci.blob.core.windows.net/openpilotci/" + name
+
+if __name__ == "__main__":
+  for f in sys.argv[1:]:
+    name = os.path.basename(f)
+    url = upload_file(f, name)
+    print(url)

selfdrive/test/openpilotci_upload.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-import subprocess
-
-
-def upload_file(path, name):
-  from azure.storage.blob import BlockBlobService
-  sas_token = os.getenv("TOKEN", None)
-  if sas_token is None:
-    cmd = "az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions \
-           lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv"
-    sas_token = subprocess.check_output(cmd, shell=True).decode().strip("\n")
-  service = BlockBlobService(account_name="commadataci", sas_token=sas_token)
-  service.create_blob_from_path("openpilotci", name, path)
-  return "https://commadataci.blob.core.windows.net/openpilotci/" + name
-
-if __name__ == "__main__":
-  for f in sys.argv[1:]:
-    name = os.path.basename(f)
-    url = upload_file(f, name)
-    print(url)

selfdrive/test/phone_ci.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+import paramiko  # pylint: disable=import-error
+import os
+import sys
+import re
+import time
+import socket
+
+TEST_DIR = "/data/openpilotci"
+
+def run_test(name, test_func):
+  ssh = paramiko.SSHClient()
+  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+  key_file = open(os.path.join(os.path.dirname(__file__), "../../release/id_rsa_public"))
+  key = paramiko.RSAKey.from_private_key(key_file)
+
+  print("SSH to phone {}".format(name))
+
+  # Try connecting for one minute
+  t_start = time.time()
+  while True:
+    try:
+      ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
+    except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
+      print("Connection failed")
+      if time.time() - t_start > 60:
+        raise
+    else:
+      break
+    time.sleep(1)
+
+  conn = ssh.invoke_shell()
+  branch = os.environ['GIT_BRANCH']
+  commit = os.environ.get('GIT_COMMIT', branch)
+
+  conn.send("uname -a\n")
+
+  conn.send(f"cd {TEST_DIR}\n")
+  conn.send("git reset --hard\n")
+  conn.send("git fetch origin\n")
+  conn.send("git checkout %s\n" % commit)
+  conn.send("git clean -xdf\n")
+  conn.send("git submodule update --init\n")
+  conn.send("git submodule foreach --recursive git reset --hard\n")
+  conn.send("git submodule foreach --recursive git clean -xdf\n")
+  conn.send("echo \"git took $SECONDS seconds\"\n")
+
+  test_func(conn)
+
+  conn.send('echo "RESULT:" $?\n')
+  conn.send("exit\n")
+  return conn
+
+def test_modeld(conn):
+  conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n")
+
+if __name__ == "__main__":
+  eon_name = os.environ.get('eon_name', None)
+
+  conn = run_test(eon_name, test_modeld)
+
+  dat = b""
+
+  while True:
+    recvd = conn.recv(4096)
+    if len(recvd) == 0:
+      break
+
+    dat += recvd
+    sys.stdout.buffer.write(recvd)
+    sys.stdout.flush()
+
+  returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
+  sys.exit(int(returns[0]))

selfdrive/test/process_replay/.gitignore

@@ -1,2 +0,0 @@
-*.bz2
-diff.txt

selfdrive/test/process_replay/camera_replay.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+import os
+import sys
+import time
+from typing import Any
+from tqdm import tqdm
+
+from common.android import ANDROID
+if ANDROID:
+  os.environ['QCOM_REPLAY'] = "1"
+import selfdrive.manager as manager
+
+from common.spinner import Spinner
+import cereal.messaging as messaging
+from tools.lib.framereader import FrameReader
+from tools.lib.logreader import LogReader
+from selfdrive.test.openpilotci import BASE_URL, get_url
+from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
+from selfdrive.test.process_replay.test_processes import format_diff
+from selfdrive.version import get_git_commit
+
+TEST_ROUTE = "5b7c365c50084530|2020-04-15--16-13-24"
+
+def camera_replay(lr, fr):
+
+  spinner = Spinner()
+
+  pm = messaging.PubMaster(['frame', 'liveCalibration'])
+  sm = messaging.SubMaster(['model'])
+
+  # TODO: add dmonitoringmodeld
+  print("preparing procs")
+  manager.prepare_managed_process("camerad")
+  manager.prepare_managed_process("modeld")
+  try:
+    print("starting procs")
+    manager.start_managed_process("camerad")
+    manager.start_managed_process("modeld")
+    time.sleep(5)
+    print("procs started")
+
+    cal = [msg for msg in lr if msg.which() == "liveCalibration"]
+    for msg in cal[:5]:
+      pm.send(msg.which(), msg.as_builder())
+
+    log_msgs = []
+    frame_idx = 0
+    for msg in tqdm(lr):
+      if msg.which() == "liveCalibrationd":
+        pm.send(msg.which(), msg.as_builder())
+      elif msg.which() == "frame":
+        f = msg.as_builder()
+        img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1]
+        f.frame.image = img.flatten().tobytes()
+        frame_idx += 1
+
+        pm.send(msg.which(), f)
+        log_msgs.append(messaging.recv_one(sm.sock['model']))
+
+        spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
+
+        if frame_idx >= fr.frame_count:
+          break
+  except KeyboardInterrupt:
+    pass
+
+  print("replay done")
+  spinner.close()
+  manager.kill_managed_process('modeld')
+  time.sleep(2)
+  manager.kill_managed_process('camerad')
+  return log_msgs
+
+
+if __name__ == "__main__":
+
+  update = "--update" in sys.argv
+
+  lr = LogReader(get_url(TEST_ROUTE, 0))
+  fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera"))
+
+  log_msgs = camera_replay(list(lr), fr)
+
+  if update:
+    ref_commit = get_git_commit()
+    log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
+    save_log(log_fn, log_msgs)
+    with open("model_replay_ref_commit", "w") as f:
+      f.write(ref_commit)
+  else:
+    ref_commit = open("model_replay_ref_commit").read().strip()
+    log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
+    cmp_log = LogReader(BASE_URL + log_fn)
+    results: Any = {TEST_ROUTE: {}}
+    results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid'])
+    diff1, diff2, failed = format_diff(results, ref_commit)
+
+    print(diff1)
+    with open("model_diff.txt", "w") as f:
+      f.write(diff2)
+
+    sys.exit(int(failed))
+

selfdrive/test/process_replay/model_replay_ref_commit

@@ -0,0 +1 @@
+18e43945403c4022b1de72237d89736e1a8ab4c7

selfdrive/test/process_replay/update_model.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-from selfdrive.test.openpilotci_upload import upload_file
+from selfdrive.test.openpilotci import upload_file
 from selfdrive.test.process_replay.compare_logs import save_log
 from selfdrive.test.process_replay.test_processes import segments, get_segment
 from selfdrive.version import get_git_commit

selfdrive/test/process_replay/update_refs.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-from selfdrive.test.openpilotci_upload import upload_file
+from selfdrive.test.openpilotci import upload_file
 from selfdrive.test.process_replay.compare_logs import save_log
 from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS
 from selfdrive.test.process_replay.test_processes import segments, get_segment