openpilot v0.7.8 release

selfdrive/test/helpers.py

@@ -4,8 +4,19 @@ from nose.tools import nottest
 
 from common.android import ANDROID
 from common.apk import update_apks, start_offroad, pm_apply_packages, android_packages
+from common.params import Params
+from selfdrive.version import training_version, terms_version
 from selfdrive.manager import start_managed_process, kill_managed_process, get_running
 
+def set_params_enabled():
+  params = Params()
+  params.put("HasAcceptedTerms", terms_version)
+  params.put("HasCompletedSetup", "1")
+  params.put("OpenpilotEnabledToggle", "1")
+  params.put("CommunityFeaturesToggle", "1")
+  params.put("Passive", "0")
+  params.put("CompletedTrainingVersion", training_version)
+
 def phone_only(x):
   if ANDROID:
     return x

selfdrive/test/id_rsa

@@ -1,28 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N
-Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB
-t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt
-Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ
-acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6
-1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ
-rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q
-x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC
-gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix
-Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1
-LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL
-s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb
-Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj
-6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO
-uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh
-0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q
-FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo
-ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi
-4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA
-nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it
-VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl
-s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l
-jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7
-b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I
-RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY
-6co17RFaAZHwGfCFFjO76Q==
------END RSA PRIVATE KEY-----

selfdrive/test/phone_ci.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-import paramiko  # pylint: disable=import-error
-import os
-import sys
-import re
-import time
-import socket
-
-
-SOURCE_DIR = "/data/openpilot_source/"
-TEST_DIR = "/data/openpilot/"
-
-def run_on_phone(test_cmd):
-
-  eon_ip = os.environ.get('eon_ip', None)
-  if eon_ip is None:
-    raise Exception("'eon_ip' not set")
-
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  key_file = open(os.path.join(os.path.dirname(__file__), "id_rsa"))
-  key = paramiko.RSAKey.from_private_key(key_file)
-
-  print("SSH to phone at {}".format(eon_ip))
-
-  # try connecting for one minute
-  t_start = time.time()
-  while True:
-    try:
-      ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
-    except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
-      print("Connection failed")
-      if time.time() - t_start > 60:
-        raise
-    else:
-      break
-    time.sleep(1)
-
-  branch = os.environ['GIT_BRANCH']
-  commit = os.environ.get('GIT_COMMIT', branch)
-
-  conn = ssh.invoke_shell()
-
-  # pass in all environment variables prefixed with 'CI_'
-  for k, v in os.environ.items():
-    if k.startswith("CI_") or k in ["GIT_BRANCH", "GIT_COMMIT"]:
-      conn.send(f"export {k}='{v}'\n")
-  conn.send("export CI=1\n")
-
-  # clear scons cache dirs that haven't been written to in one day
-  conn.send("cd /tmp && find -name 'scons_cache_*' -type d -maxdepth 1 -mtime 1 -exec rm -rf '{}' \\;\n")
-
-  # set up environment
-  conn.send(f"cd {SOURCE_DIR}\n")
-  conn.send("git reset --hard\n")
-  conn.send("git fetch origin\n")
-  conn.send("find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \\;\n")
-  conn.send(f"git reset --hard {commit}\n")
-  conn.send(f"git checkout {commit}\n")
-  conn.send("git clean -xdf\n")
-  conn.send("git submodule update --init\n")
-  conn.send("git submodule foreach --recursive git reset --hard\n")
-  conn.send("git submodule foreach --recursive git clean -xdf\n")
-  conn.send('echo "git took $SECONDS seconds"\n')
-
-  conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")
-
-  # run the test
-  conn.send(test_cmd + "\n")
-
-  # get the result and print it back out
-  conn.send('echo "RESULT:" $?\n')
-  conn.send("exit\n")
-
-  dat = b""
-  conn.settimeout(240)
-
-  while True:
-    try:
-      recvd = conn.recv(4096)
-    except socket.timeout:
-      print("connection to phone timed out")
-      sys.exit(1)
-
-    if len(recvd) == 0:
-      break
-
-    dat += recvd
-    sys.stdout.buffer.write(recvd)
-    sys.stdout.flush()
-
-  return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
-  sys.exit(return_code)
-
-
-if __name__ == "__main__":
-  run_on_phone(sys.argv[1])

selfdrive/test/setup_device_ci.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/bash -e
+
+export SOURCE_DIR="/data/openpilot_source/"
+
+if [ -z "$GIT_COMMIT" ]; then
+  echo "GIT_COMMIT must be set"
+  exit 1
+fi
+
+if [ -z "$TEST_DIR" ]; then
+
+  echo "TEST_DIR must be set"
+  exit 1
+fi
+
+# TODO: never clear qcom_replay cache
+# clear scons cache dirs that haven't been written to in one day
+cd /tmp && find -name 'scons_cache_*' -type d -maxdepth 1 -mtime 1 -exec rm -rf '{}' \;
+
+# set up environment
+cd $SOURCE_DIR
+git reset --hard
+git fetch origin
+find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
+git reset --hard $GIT_COMMIT
+git checkout $GIT_COMMIT
+git clean -xdf
+git submodule update --init
+git submodule foreach --recursive git reset --hard
+git submodule foreach --recursive git clean -xdf
+echo "git checkout took $SECONDS seconds"
+
+rsync -a --delete $SOURCE_DIR $TEST_DIR
+
+echo "$TEST_DIR synced with $GIT_COMMIT, took $SECONDS seconds"

selfdrive/test/test_cpu_usage.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
+import os
 import time
-import threading
-import _thread
-import signal
 import sys
+import subprocess
 
 import cereal.messaging as messaging
-import selfdrive.manager as manager
-
+from common.basedir import BASEDIR
+from common.params import Params
+from selfdrive.test.helpers import set_params_enabled
 
 def cputime_total(ct):
   return ct.cpuUser + ct.cpuSystem + ct.cpuChildrenUser + ct.cpuChildrenSystem
@@ -15,7 +15,7 @@ def cputime_total(ct):
 
 def print_cpu_usage(first_proc, last_proc):
   procs = [
-    ("selfdrive.controls.controlsd", 59.46),
+    ("selfdrive.controls.controlsd", 66.15),
     ("selfdrive.locationd.locationd", 34.38),
     ("./loggerd", 33.90),
     ("selfdrive.controls.plannerd", 19.77),
@@ -39,7 +39,7 @@ def print_cpu_usage(first_proc, last_proc):
     ("./logcatd", 0),
   ]
 
-  r = 0
+  r = True
   dt = (last_proc.logMonoTime - first_proc.logMonoTime) / 1e9
   result = "------------------------------------------------\n"
   for proc_name, normal_cpu_usage in procs:
@@ -50,47 +50,54 @@ def print_cpu_usage(first_proc, last_proc):
       cpu_usage = cpu_time / dt * 100.
       if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0):
         result += f"Warning {proc_name} using more CPU than normal\n"
-        r = 1
+        r = False
       elif cpu_usage < min(normal_cpu_usage * 0.3, max(normal_cpu_usage - 1.0, 0.0)):
         result += f"Warning {proc_name} using less CPU than normal\n"
-        r = 1
+        r = False
       result += f"{proc_name.ljust(35)}  {cpu_usage:.2f}%\n"
     except IndexError:
       result += f"{proc_name.ljust(35)}  NO METRICS FOUND\n"
-      r = 1
+      r = False
   result += "------------------------------------------------\n"
   print(result)
   return r
 
-return_code = 1
-def test_thread():
-  global return_code
-  proc_sock = messaging.sub_sock('procLog', conflate=True)
+def test_cpu_usage():
+  cpu_ok = False
 
-  # wait until everything's started and get first sample
-  time.sleep(30)
-  first_proc = messaging.recv_sock(proc_sock, wait=True)
+  # start manager
+  manager_path = os.path.join(BASEDIR, "selfdrive/manager.py")
+  manager_proc = subprocess.Popen(["python", manager_path])
+  try:
+    proc_sock = messaging.sub_sock('procLog', conflate=True, timeout=2000)
 
-  # run for a minute and get last sample
-  time.sleep(60)
-  last_proc = messaging.recv_sock(proc_sock, wait=True)
+    # wait until everything's started and get first sample
+    start_time = time.monotonic()
+    while time.monotonic() - start_time < 120:
+      if Params().get("CarParams") is not None:
+        break
+      time.sleep(2)
+    first_proc = messaging.recv_sock(proc_sock, wait=True)
+    if first_proc is None:
+      raise Exception("\n\nTEST FAILED: progLog recv timed out\n\n")
 
-  running = manager.get_running()
-  all_running = all(p in running and running[p].is_alive() for p in manager.car_started_processes)
-  return_code = print_cpu_usage(first_proc, last_proc)
-  if not all_running:
-    return_code = 1
-  _thread.interrupt_main()
+    # run for a minute and get last sample
+    time.sleep(60)
+    last_proc = messaging.recv_sock(proc_sock, wait=True)
+    cpu_ok = print_cpu_usage(first_proc, last_proc)
+  finally:
+    manager_proc.terminate()
+    ret = manager_proc.wait(20)
+    if ret is None:
+      manager_proc.kill()
+  return cpu_ok
 
 if __name__ == "__main__":
+  set_params_enabled()
+  Params().delete("CarParams")
 
-  # setup signal handler to exit with test status
-  def handle_exit(sig, frame):
-    sys.exit(return_code)
-  signal.signal(signal.SIGINT, handle_exit)
-
-  # start manager and test thread
-  t = threading.Thread(target=test_thread)
-  t.daemon = True
-  t.start()
-  manager.main()
+  passed = False
+  try:
+    passed = test_cpu_usage()
+  finally:
+    sys.exit(int(not passed))

selfdrive/test/test_openpilot.py

@@ -5,7 +5,7 @@ os.environ['FAKEUPLOAD'] = "1"
 from common.params import Params
 from common.realtime import sec_since_boot
 from selfdrive.manager import manager_init, manager_prepare, start_daemon_process
-from selfdrive.test.helpers import phone_only, with_processes
+from selfdrive.test.helpers import phone_only, with_processes, set_params_enabled
 import json
 import requests
 import signal
@@ -16,6 +16,7 @@ import time
 # must run first
 @phone_only
 def test_manager_prepare():
+  set_params_enabled()
   manager_init()
   manager_prepare()