On-device CI framework (#1784)

* let's see if this works

* fix build_release actions job

* does jenkins like this config

* separate jenkinsfile for release build

* fix devel build

* devel build should work

* always pass that for now

* run modeld replay

* release2 build will be a separate PR

* pass env to phone shell

* force checkout

* run on real jenkins eons

* add timeout

* rsync

* more timeout

* trailing slash

* fix branch detection

* debug

* not sure why paramiko doesn't pass it through

* newline

* CI_PUSH

* still not passing it

* test branch

* should be good now

selfdrive/test/phone_ci.py

@@ -6,22 +6,29 @@ import re
 import time
 import socket
 
-TEST_DIR = "/data/openpilotci"
 
-def run_test(name, test_func):
+SOURCE_DIR = "/data/openpilot_source/"
+TEST_DIR = "/data/openpilot/"
+
+def run_on_phone(test_cmd):
+
+  eon_ip = os.environ.get('eon_ip', None)
+  if eon_ip is None:
+    raise Exception("'eon_ip' not set")
+
   ssh = paramiko.SSHClient()
   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
   key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa"))
   key = paramiko.RSAKey.from_private_key(key_file)
 
-  print("SSH to phone {}".format(name))
+  print("SSH to phone at {}".format(eon_ip))
 
-  # Try connecting for one minute
+  # try connecting for one minute
   t_start = time.time()
   while True:
     try:
-      ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
+      ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
     except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
       print("Connection failed")
       if time.time() - t_start > 60:
@@ -30,40 +37,47 @@ def run_test(name, test_func):
       break
     time.sleep(1)
 
-  conn = ssh.invoke_shell()
   branch = os.environ['GIT_BRANCH']
   commit = os.environ.get('GIT_COMMIT', branch)
 
-  conn.send("uname -a\n")
+  conn = ssh.invoke_shell()
 
-  conn.send(f"cd {TEST_DIR}\n")
+  # pass in all environment variables prefixed with 'CI_'
+  for k, v in os.environ.items():
+    if k.startswith("CI_"):
+      conn.send(f"export {k}='{v}'\n")
+  conn.send("export CI=1\n")
+
+  # set up environment
+  conn.send(f"cd {SOURCE_DIR}\n")
   conn.send("git reset --hard\n")
   conn.send("git fetch origin\n")
-  conn.send("git checkout %s\n" % commit)
+  conn.send(f"git checkout {commit}\n")
   conn.send("git clean -xdf\n")
   conn.send("git submodule update --init\n")
   conn.send("git submodule foreach --recursive git reset --hard\n")
   conn.send("git submodule foreach --recursive git clean -xdf\n")
-  conn.send("echo \"git took $SECONDS seconds\"\n")
+  conn.send('echo "git took $SECONDS seconds"\n')
 
-  test_func(conn)
+  conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")
 
+  # run the test
+  conn.send(test_cmd + "\n")
+
+  # get the result and print it back out
   conn.send('echo "RESULT:" $?\n')
   conn.send("exit\n")
-  return conn
-
-def test_modeld(conn):
-  conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n")
-
-if __name__ == "__main__":
-  eon_name = os.environ.get('eon_name', None)
-
-  conn = run_test(eon_name, test_modeld)
 
   dat = b""
+  conn.settimeout(120)
 
   while True:
-    recvd = conn.recv(4096)
+    try:
+      recvd = conn.recv(4096)
+    except socket.timeout:
+      print("connection to phone timed out")
+      sys.exit(1)
+
     if len(recvd) == 0:
       break
 
@@ -71,5 +85,9 @@ if __name__ == "__main__":
     sys.stdout.buffer.write(recvd)
     sys.stdout.flush()
 
-  returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
-  sys.exit(int(returns[0]))
+  return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
+  sys.exit(return_code)
+
+
+if __name__ == "__main__":
+  run_on_phone(sys.argv[1])