openpilot v0.6.3 release

old-commit-hash: d5f9caa82d

selfdrive/test/openpilotci_upload.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python2
+import os
+import sys
+import subprocess
+from azure.storage.blob import BlockBlobService
+
+def upload_file(path, name):
+  sas_token = os.getenv("TOKEN", None)
+  if sas_token is not None:
+    service = BlockBlobService(account_name="commadataci", sas_token=sas_token)
+  else:
+    account_key = subprocess.check_output("az storage account keys list --account-name commadataci --output tsv --query '[0].value'", shell=True)
+    service = BlockBlobService(account_name="commadataci", account_key=account_key)
+  service.create_blob_from_path("openpilotci", name, path)
+  return "https://commadataci.blob.core.windows.net/openpilotci/" + name
+
+if __name__ == "__main__":
+  for f in sys.argv[1:]:
+    name = os.path.basename(f)
+    url = upload_file(f, name)
+    print url
+

selfdrive/test/plant/plant.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import os
 import struct
+import time
 from collections import namedtuple
 import numpy as np
 
@@ -133,6 +134,11 @@ class Plant(object):
     self.ts = 1./rate
 
     self.cp = get_car_can_parser()
+    self.response_seen = False
+
+    time.sleep(1)
+    messaging.drain_sock(Plant.sendcan)
+    messaging.drain_sock(Plant.controls_state)
 
   def close(self):
     Plant.logcan.close()
@@ -158,13 +164,18 @@ class Plant(object):
 
     # ******** get messages sent to the car ********
     can_msgs = []
-    for a in messaging.drain_sock(Plant.sendcan):
+    for a in messaging.drain_sock(Plant.sendcan, wait_for_one=self.response_seen):
       can_msgs.extend(can_capnp_to_can_list(a.sendcan, [0,2]))
+
+    # After the first response the car is done fingerprinting, so we can run in lockstep with controlsd
+    if can_msgs:
+      self.response_seen = True
+
     self.cp.update_can(can_msgs)
 
     # ******** get controlsState messages for plotting ***
     controls_state_msgs = []
-    for a in messaging.drain_sock(Plant.controls_state):
+    for a in messaging.drain_sock(Plant.controls_state, wait_for_one=self.response_seen):
       controls_state_msgs.append(a.controlsState)
 
     fcw = None
@@ -217,7 +228,7 @@ class Plant(object):
     vls_tuple = namedtuple('vls', [
            'XMISSION_SPEED',
            'WHEEL_SPEED_FL', 'WHEEL_SPEED_FR', 'WHEEL_SPEED_RL', 'WHEEL_SPEED_RR',
-           'STEER_ANGLE', 'STEER_ANGLE_RATE', 'STEER_TORQUE_SENSOR',
+           'STEER_ANGLE', 'STEER_ANGLE_RATE', 'STEER_TORQUE_SENSOR', 'STEER_TORQUE_MOTOR',
            'LEFT_BLINKER', 'RIGHT_BLINKER',
            'GEAR',
            'WHEELS_MOVING',
@@ -250,7 +261,7 @@ class Plant(object):
     vls = vls_tuple(
            self.speed_sensor(speed),
            self.speed_sensor(speed), self.speed_sensor(speed), self.speed_sensor(speed), self.speed_sensor(speed),
-           self.angle_steer, self.angle_steer_rate, 0, #Steer torque sensor
+           self.angle_steer, self.angle_steer_rate, 0, 0,#Steer torque sensor
            0, 0,  # Blinkers
            self.gear_choice,
            speed != 0,
@@ -325,7 +336,6 @@ class Plant(object):
     msg_data = fix(msg_data, 0xe4)
     can_msgs.append([0xe4, 0, msg_data, 2])
 
-    Plant.logcan.send(can_list_to_can_capnp(can_msgs))
 
     # Fake sockets that controlsd subscribes to
     live_parameters = messaging.new_message()
@@ -388,10 +398,13 @@ class Plant(object):
 
       cal.liveCalibration.calStatus = 1
       cal.liveCalibration.calPerc = 100
+      cal.liveCalibration.rpyCalib = [0.] * 3
       # fake values?
       Plant.model.send(md.to_bytes())
       Plant.cal.send(cal.to_bytes())
 
+    Plant.logcan.send(can_list_to_can_capnp(can_msgs))
+
     # ******** update prevs ********
     self.speed = speed
     self.distance = distance
@@ -401,7 +414,11 @@ class Plant(object):
     self.distance_prev = distance
     self.distance_lead_prev = distance_lead
 
-    self.rk.keep_time()
+    if self.response_seen:
+      self.rk.monitor_time()
+    else:
+      self.rk.keep_time()
+
     return {
       "distance": distance,
       "speed": speed,

selfdrive/test/test_car_models_openpilot.py

@@ -0,0 +1,494 @@
+#!/usr/bin/env python2
+import shutil
+import time
+import zmq
+import os
+import sys
+import signal
+import subprocess
+import requests
+from cereal import car
+
+import selfdrive.manager as manager
+from selfdrive.services import service_list
+import selfdrive.messaging as messaging
+from common.params import Params
+from common.basedir import BASEDIR
+from selfdrive.car.honda.values import CAR as HONDA
+from selfdrive.car.toyota.values import CAR as TOYOTA
+from selfdrive.car.gm.values import CAR as GM
+from selfdrive.car.ford.values import CAR as FORD
+from selfdrive.car.hyundai.values import CAR as HYUNDAI
+from selfdrive.car.chrysler.values import CAR as CHRYSLER
+from selfdrive.car.subaru.values import CAR as SUBARU
+from selfdrive.car.mock.values import CAR as MOCK
+
+
+os.environ['NOCRASH'] = '1'
+
+
+def wait_for_socket(name, timeout=10.0):
+  socket = messaging.sub_sock(service_list[name].port)
+  cur_time = time.time()
+
+  r = None
+  while time.time() - cur_time < timeout:
+    print("waiting for %s" % name)
+    try:
+      r = socket.recv(zmq.NOBLOCK)
+      break
+    except zmq.error.Again:
+      pass
+    time.sleep(0.5)
+  return r
+
+def get_route_logs(route_name):
+  for log_f in ["rlog.bz2", "fcamera.hevc"]:
+    log_path = os.path.join("/tmp", "%s--0--%s" % (route_name.replace("|", "_"), log_f))
+
+    if not os.path.isfile(log_path):
+      log_url = "https://commadataci.blob.core.windows.net/openpilotci/%s/0/%s" % (route_name.replace("|", "/"), log_f)
+      r = requests.get(log_url)
+
+      if r.status_code == 200:
+        with open(log_path, "w") as f:
+          f.write(r.content)
+      else:
+        sys.exit(-1)
+
+routes = {
+
+  "975b26878285314d|2018-12-25--14-42-13": {
+    'carFingerprint': CHRYSLER.PACIFICA_2018_HYBRID,
+    'enableCamera': True,
+  },
+  "b0c9d2329ad1606b|2019-01-06--10-11-23": {
+    'carFingerprint': CHRYSLER.PACIFICA_2017_HYBRID,
+    'enableCamera': True,
+  },
+  "0607d2516fc2148f|2019-02-13--23-03-16": {
+    'carFingerprint': CHRYSLER.PACIFICA_2019_HYBRID,
+    'enableCamera': True,
+  },
+  # This pacifica was removed because the fingerprint seemed from a Volt
+  #"9f7a7e50a51fb9db|2019-01-03--14-05-01": {
+  #  'carFingerprint': CHRYSLER.PACIFICA_2018,
+  #  'enableCamera': True,
+  #},
+  "9f7a7e50a51fb9db|2019-01-17--18-34-21": {
+    'carFingerprint': CHRYSLER.JEEP_CHEROKEE,
+    'enableCamera': True,
+  },
+  "192a598e34926b1e|2019-04-04--13-27-39": {
+    'carFingerprint': CHRYSLER.JEEP_CHEROKEE_2019,
+    'enableCamera': True,
+  },
+  "f1b4c567731f4a1b|2018-04-18--11-29-37": {
+    'carFingerprint': FORD.FUSION,
+    'enableCamera': False,
+  },
+  "f1b4c567731f4a1b|2018-04-30--10-15-35": {
+    'carFingerprint': FORD.FUSION,
+    'enableCamera': True,
+  },
+  "7ed9cdf8d0c5f43e|2018-05-17--09-31-36": {
+    'carFingerprint': GM.CADILLAC_CT6,
+    'enableCamera': True,
+  },
+  "265007255e794bce|2018-11-24--22-08-31": {
+    'carFingerprint': GM.CADILLAC_ATS,
+    'enableCamera': True,
+  },
+  "c950e28c26b5b168|2018-05-30--22-03-41": {
+    'carFingerprint': GM.VOLT,
+    'enableCamera': True,
+  },
+  # TODO: use another route that has radar data at start
+  "aadda448b49c99ad|2018-10-25--17-16-22": {
+    'carFingerprint': GM.MALIBU,
+    'enableCamera': True,
+  },
+  "49c73650e65ff465|2018-11-19--16-58-04": {
+    'carFingerprint': GM.HOLDEN_ASTRA,
+    'enableCamera': True,
+  },
+  "7cc2a8365b4dd8a9|2018-12-02--12-10-44": {
+    'carFingerprint': GM.ACADIA,
+    'enableCamera': True,
+  },
+  "aa20e335f61ba898|2018-12-17--21-10-37": {
+    'carFingerprint': GM.BUICK_REGAL,
+    'enableCamera': False,
+  },
+  "aa20e335f61ba898|2019-02-05--16-59-04": {
+    'carFingerprint': GM.BUICK_REGAL,
+    'enableCamera': True,
+  },
+  "7d44af5b7a1b2c8e|2017-09-16--01-50-07": {
+    'carFingerprint': HONDA.CIVIC,
+    'enableCamera': True,
+  },
+  "c9d60e5e02c04c5c|2018-01-08--16-01-49": {
+    'carFingerprint': HONDA.CRV,
+    'enableCamera': True,
+  },
+  "1851183c395ef471|2018-05-31--18-07-21": {
+    'carFingerprint': HONDA.CRV_5G,
+    'enableCamera': True,
+  },
+  "232585b7784c1af4|2019-04-08--14-12-14": {
+    'carFingerprint': HONDA.CRV_HYBRID,
+    'enableCamera': True,
+  },
+  "2ac95059f70d76eb|2018-02-05--15-03-29": {
+    'carFingerprint': HONDA.ACURA_ILX,
+    'enableCamera': True,
+  },
+  "21aa231dee2a68bd|2018-01-30--04-54-41": {
+    'carFingerprint': HONDA.ODYSSEY,
+    'enableCamera': True,
+  },
+  "81722949a62ea724|2019-03-29--15-51-26": {
+    'carFingerprint': HONDA.ODYSSEY_CHN,
+    'enableCamera': False,
+  },
+  "81722949a62ea724|2019-04-06--15-19-25": {
+    'carFingerprint': HONDA.ODYSSEY_CHN,
+    'enableCamera': True,
+  },
+  "5a2cfe4bb362af9e|2018-02-02--23-41-07": {
+    'carFingerprint': HONDA.ACURA_RDX,
+    'enableCamera': True,
+  },
+  "3e9592a1c78a3d63|2018-02-08--20-28-24": {
+    'carFingerprint': HONDA.PILOT,
+    'enableCamera': True,
+  },
+  "34a84d2b765df688|2018-08-28--12-41-00": {
+    'carFingerprint': HONDA.PILOT_2019,
+    'enableCamera': True,
+  },
+  "900ad17e536c3dc7|2018-04-12--22-02-36": {
+    'carFingerprint': HONDA.RIDGELINE,
+    'enableCamera': True,
+  },
+  "f1b4c567731f4a1b|2018-06-06--14-43-46": {
+    'carFingerprint': HONDA.ACCORD,
+    'enableCamera': True,
+  },
+  "1582e1dc57175194|2018-08-15--07-46-07": {
+    'carFingerprint': HONDA.ACCORD_15,
+    'enableCamera': True,
+  },
+  # TODO: This doesnt fingerprint because the fingerprint overlaps with the Insight
+  # "690c4c9f9f2354c7|2018-09-15--17-36-05": {
+  #   'carFingerprint': HONDA.ACCORDH,
+  #   'enableCamera': True,
+  # },
+  "1632088eda5e6c4d|2018-06-07--08-03-18": {
+    'carFingerprint': HONDA.CIVIC_BOSCH,
+    'enableCamera': True,
+  },
+  #"18971a99f3f2b385|2018-11-14--19-09-31": {
+  #  'carFingerprint': HONDA.INSIGHT,
+  #  'enableCamera': True,
+  #},
+  "38bfd238edecbcd7|2018-08-22--09-45-44": {
+    'carFingerprint': HYUNDAI.SANTA_FE,
+    'enableCamera': False,
+  },
+  "38bfd238edecbcd7|2018-08-29--22-02-15": {
+    'carFingerprint': HYUNDAI.SANTA_FE,
+    'enableCamera': True,
+  },
+  "a893a80e5c5f72c8|2019-01-14--20-02-59": {
+    'carFingerprint': HYUNDAI.GENESIS,
+    'enableCamera': True,
+  },
+  "9d5fb4f0baa1b3e1|2019-01-14--17-45-59": {
+    'carFingerprint': HYUNDAI.KIA_SORENTO,
+    'enableCamera': True,
+  },
+  "215cd70e9c349266|2018-11-25--22-22-12": {
+    'carFingerprint': HYUNDAI.KIA_STINGER,
+    'enableCamera': True,
+  },
+  "31390e3eb6f7c29a|2019-01-23--08-56-00": {
+    'carFingerprint': HYUNDAI.KIA_OPTIMA,
+    'enableCamera': True,
+  },
+  "53ac3251e03f95d7|2019-01-10--13-43-32": {
+    'carFingerprint': HYUNDAI.ELANTRA,
+    'enableCamera': True,
+  },
+  "f7b6be73e3dfd36c|2019-05-12--18-07-16": {
+    'carFingerprint': TOYOTA.AVALON,
+    'enableCamera': False,
+    'enableDsu': False,
+  },
+  "f7b6be73e3dfd36c|2019-05-11--22-34-20": {
+    'carFingerprint': TOYOTA.AVALON,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  "b0f5a01cf604185c|2018-01-26--00-54-32": {
+    'carFingerprint': TOYOTA.COROLLA,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "b0f5a01cf604185c|2018-01-26--10-54-38": {
+    'carFingerprint': TOYOTA.COROLLA,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  "b0f5a01cf604185c|2018-01-26--10-59-31": {
+    'carFingerprint': TOYOTA.COROLLA,
+    'enableCamera': False,
+    'enableDsu': False,
+  },
+  "5f5afb36036506e4|2019-05-14--02-09-54": {
+    'carFingerprint': TOYOTA.COROLLA_TSS2,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "56fb1c86a9a86404|2017-11-10--10-18-43": {
+    'carFingerprint': TOYOTA.PRIUS,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "b0f5a01cf604185c|2017-12-18--20-32-32": {
+    'carFingerprint': TOYOTA.RAV4,
+    'enableCamera': True,
+    'enableDsu': True,
+    'enableGasInterceptor': False,
+  },
+  "b0c9d2329ad1606b|2019-04-02--13-24-43": {
+    'carFingerprint': TOYOTA.RAV4,
+    'enableCamera': True,
+    'enableDsu': True,
+    'enableGasInterceptor': True,
+  },
+  "cdf2f7de565d40ae|2019-04-25--03-53-41": {
+    'carFingerprint': TOYOTA.RAV4_TSS2,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "f49e8041283f2939|2019-05-29--13-48-33": {
+    'carFingerprint': TOYOTA.LEXUS_ESH_TSS2,
+    'enableCamera': False,
+    'enableDsu': False,
+  },
+  "f49e8041283f2939|2019-05-30--11-51-51": {
+    'carFingerprint': TOYOTA.LEXUS_ESH_TSS2,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "b0f5a01cf604185c|2018-02-01--21-12-28": {
+    'carFingerprint': TOYOTA.LEXUS_RXH,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  #FIXME: This works sometimes locally, but never in CI. Timing issue?
+  #"b0f5a01cf604185c|2018-01-31--20-11-39": {
+  #  'carFingerprint': TOYOTA.LEXUS_RXH,
+  #  'enableCamera': False,
+  #  'enableDsu': False,
+  #},
+  "8ae193ceb56a0efe|2018-06-18--20-07-32": {
+    'carFingerprint': TOYOTA.RAV4H,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "fd10b9a107bb2e49|2018-07-24--16-32-42": {
+    'carFingerprint': TOYOTA.CHR,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  "fd10b9a107bb2e49|2018-07-24--20-32-08": {
+    'carFingerprint': TOYOTA.CHR,
+    'enableCamera': False,
+    'enableDsu': False,
+  },
+  "b4c18bf13d5955da|2018-07-29--13-39-46": {
+    'carFingerprint': TOYOTA.CHRH,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  "d2d8152227f7cb82|2018-07-25--13-40-56": {
+    'carFingerprint': TOYOTA.CAMRY,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  "fbd011384db5e669|2018-07-26--20-51-48": {
+    'carFingerprint': TOYOTA.CAMRYH,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  # TODO: This replay has no good model/video
+  # "c9fa2dd0f76caf23|2018-02-10--13-40-28": {
+  #   'carFingerprint': TOYOTA.CAMRYH,
+  #   'enableCamera': False,
+  #   'enableDsu': False,
+  # },
+  # TODO: missingsome combos for highlander
+  "aa659debdd1a7b54|2018-08-31--11-12-01": {
+    'carFingerprint': TOYOTA.HIGHLANDER,
+    'enableCamera': False,
+    'enableDsu': False,
+  },
+  "362d23d4d5bea2fa|2018-09-02--17-03-55": {
+    'carFingerprint': TOYOTA.HIGHLANDERH,
+    'enableCamera': True,
+    'enableDsu': True,
+  },
+  "eb6acd681135480d|2019-06-20--20-00-00": {
+    'carFingerprint': TOYOTA.SIENNA,
+    'enableCamera': True,
+    'enableDsu': False,
+  },
+  "362d23d4d5bea2fa|2018-08-10--13-31-40": {
+    'carFingerprint': TOYOTA.HIGHLANDERH,
+    'enableCamera': False,
+    'enableDsu': False,
+  },
+  "791340bc01ed993d|2019-03-10--16-28-08": {
+    'carFingerprint': SUBARU.IMPREZA,
+    'enableCamera': True,
+  },
+  # Tesla route, should result in mock car
+  "07cb8a788c31f645|2018-06-17--18-50-29": {
+    'carFingerprint': MOCK.MOCK,
+  },
+  ## Route with no can data, should result in mock car. This is not supported anymore
+  #"bfa17080b080f3ec|2018-06-28--23-27-47": {
+  #  'carFingerprint': MOCK.MOCK,
+  #},
+}
+
+passive_routes = [
+  "07cb8a788c31f645|2018-06-17--18-50-29",
+  #"bfa17080b080f3ec|2018-06-28--23-27-47",
+]
+
+public_routes = [
+  "f1b4c567731f4a1b|2018-06-06--14-43-46",
+  "f1b4c567731f4a1b|2018-04-18--11-29-37",
+  "f1b4c567731f4a1b|2018-04-18--11-29-37",
+  "7ed9cdf8d0c5f43e|2018-05-17--09-31-36",
+  "38bfd238edecbcd7|2018-08-22--09-45-44",
+  "38bfd238edecbcd7|2018-08-29--22-02-15",
+  "b0f5a01cf604185c|2018-01-26--00-54-32",
+  "b0f5a01cf604185c|2018-01-26--10-54-38",
+  "b0f5a01cf604185c|2018-01-26--10-59-31",
+  "56fb1c86a9a86404|2017-11-10--10-18-43",
+  "b0f5a01cf604185c|2017-12-18--20-32-32",
+  "b0c9d2329ad1606b|2019-04-02--13-24-43",
+  "791340bc01ed993d|2019-03-10--16-28-08",
+]
+
+if __name__ == "__main__":
+
+  results = {}
+  for route, checks in routes.items():
+
+    if route not in public_routes:
+      print "route not public", route
+      continue
+
+    get_route_logs(route)
+
+    for _ in range(3):
+      shutil.rmtree('/data/params')
+      manager.gctx = {}
+      params = Params()
+      params.manager_start()
+      if route in passive_routes:
+        params.put("Passive", "1")
+      else:
+        params.put("Passive", "0")
+
+      print "testing ", route, " ", checks['carFingerprint']
+      print "Preparing processes"
+      manager.prepare_managed_process("radard")
+      manager.prepare_managed_process("controlsd")
+      manager.prepare_managed_process("plannerd")
+      print "Starting processes"
+      manager.start_managed_process("radard")
+      manager.start_managed_process("controlsd")
+      manager.start_managed_process("plannerd")
+      time.sleep(2)
+
+      # Start unlogger
+      print "Start unlogger"
+      unlogger_cmd = [os.path.join(BASEDIR, 'tools/replay/unlogger.py'), '%s' % route, '/tmp', '--disable', 'frame,plan,pathPlan,liveLongitudinalMpc,radarState,controlsState,liveTracks,liveMpc,sendcan,carState,carControl', '--no-interactive']
+      unlogger = subprocess.Popen(unlogger_cmd, preexec_fn=os.setsid)
+
+      print "Check sockets"
+      controls_state_result = wait_for_socket('controlsState', timeout=30)
+      radarstate_result = wait_for_socket('radarState', timeout=30)
+      plan_result = wait_for_socket('plan', timeout=30)
+
+      if route not in passive_routes:  # TODO The passive routes have very flaky models
+        path_plan_result = wait_for_socket('pathPlan', timeout=30)
+      else:
+        path_plan_result = True
+
+      carstate_result = wait_for_socket('carState', timeout=30)
+
+      print "Check if everything is running"
+      running = manager.get_running()
+      controlsd_running = running['controlsd'].is_alive()
+      radard_running = running['radard'].is_alive()
+      plannerd_running = running['plannerd'].is_alive()
+
+      manager.kill_managed_process("controlsd")
+      manager.kill_managed_process("radard")
+      manager.kill_managed_process("plannerd")
+      os.killpg(os.getpgid(unlogger.pid), signal.SIGTERM)
+
+      sockets_ok = all([
+        controls_state_result, radarstate_result, plan_result, path_plan_result, carstate_result,
+        controlsd_running, radard_running, plannerd_running
+      ])
+      params_ok = True
+      failures = []
+
+      if not controlsd_running:
+        failures.append('controlsd')
+      if not radard_running:
+        failures.append('radard')
+      if not radarstate_result:
+        failures.append('radarState')
+      if not controls_state_result:
+        failures.append('controlsState')
+      if not plan_result:
+        failures.append('plan')
+      if not path_plan_result:
+        failures.append('pathPlan')
+
+      try:
+        car_params = car.CarParams.from_bytes(params.get("CarParams"))
+        for k, v in checks.items():
+          if not v == getattr(car_params, k):
+            params_ok = False
+            failures.append(k)
+      except:
+        params_ok = False
+
+      if sockets_ok and params_ok:
+        print "Success"
+        results[route] = True, failures
+        break
+      else:
+        print "Failure"
+        results[route] = False, failures
+
+      time.sleep(2)
+
+  print results
+  params.put("Passive", "0")   # put back not passive to not leave the params in an unintended state
+  if not all(passed for passed, _ in results.values()):
+    print "TEST FAILED"
+    sys.exit(1)
+  else:
+    print "TEST SUCESSFUL"

selfdrive/test/tests/process_replay/.gitignore

@@ -0,0 +1,2 @@
+*.bz2
+diff.txt

selfdrive/test/tests/process_replay/README.md

@@ -0,0 +1,15 @@
+# process replay
+
+Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment.
+
+If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated.
+
+Use `test_processes.py` to run the test locally.
+
+Currently the following processes are tested:
+
+* controlsd
+* radard
+* plannerd
+* calibrationd
+

selfdrive/test/tests/process_replay/compare_logs.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python2
+import bz2
+import os
+import sys
+
+import dictdiffer
+if "CI" in os.environ:
+  tqdm = lambda x: x
+else:
+  from tqdm import tqdm
+
+from tools.lib.logreader import LogReader
+
+
+def save_log(dest, log_msgs):
+  dat = ""
+  for msg in log_msgs:
+    dat += msg.as_builder().to_bytes()
+  dat = bz2.compress(dat)
+
+  with open(dest, "w") as f:
+   f.write(dat)
+
+def compare_logs(log1, log2, ignore=[]):
+  assert len(log1) == len(log2), "logs are not same length"
+
+  diff = []
+  for msg1, msg2 in tqdm(zip(log1, log2)):
+    assert msg1.which() == msg2.which(), "msgs not aligned between logs"
+
+    msg1_bytes = msg1.as_builder().to_bytes()
+    msg2_bytes = msg2.as_builder().to_bytes()
+
+    if msg1_bytes != msg2_bytes:
+      msg1_dict = msg1.to_dict(verbose=True)
+      msg2_dict = msg2.to_dict(verbose=True)
+      dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore, tolerance=0)
+      diff.extend(dd)
+  return diff
+
+if __name__ == "__main__":
+  log1 = list(LogReader(sys.argv[1]))
+  log2 = list(LogReader(sys.argv[2]))
+
+  compare_logs(log1, log2, sys.argv[3:])

selfdrive/test/tests/process_replay/process_replay.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python2
+import gc
+import os
+import time
+
+if "CI" in os.environ:
+  tqdm = lambda x: x
+else:
+  from tqdm import tqdm
+
+from cereal import car
+from selfdrive.car.car_helpers import get_car
+import selfdrive.manager as manager
+import selfdrive.messaging as messaging
+from common.params import Params
+from selfdrive.services import service_list
+from collections import namedtuple
+
+ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback'])
+
+def fingerprint(msgs, pub_socks, sub_socks):
+  print "start fingerprinting"
+  manager.prepare_managed_process("logmessaged")
+  manager.start_managed_process("logmessaged")
+
+  can = pub_socks["can"]
+  logMessage = messaging.sub_sock(service_list["logMessage"].port)
+
+  time.sleep(1)
+  messaging.drain_sock(logMessage)
+
+  # controlsd waits for a health packet before fingerprinting
+  msg = messaging.new_message()
+  msg.init("health")
+  pub_socks["health"].send(msg.to_bytes())
+
+  canmsgs = filter(lambda msg: msg.which() == "can", msgs)
+  for msg in canmsgs[:200]:
+    can.send(msg.as_builder().to_bytes())
+
+    time.sleep(0.005)
+    log = messaging.recv_one_or_none(logMessage)
+    if log is not None and "fingerprinted" in log.logMessage:
+      break
+  manager.kill_managed_process("logmessaged")
+  print "finished fingerprinting"
+
+def get_car_params(msgs, pub_socks, sub_socks):
+  sendcan = pub_socks.get("sendcan", None)
+  if sendcan is None:
+    sendcan = messaging.pub_sock(service_list["sendcan"].port)
+  logcan = sub_socks.get("can", None)
+  if logcan is None:
+    logcan = messaging.sub_sock(service_list["can"].port)
+  can = pub_socks.get("can", None)
+  if can is None:
+    can = messaging.pub_sock(service_list["can"].port)
+
+  time.sleep(0.5)
+
+  canmsgs = filter(lambda msg: msg.which() == "can", msgs)
+  for m in canmsgs[:200]:
+    can.send(m.as_builder().to_bytes())
+  _, CP = get_car(logcan, sendcan)
+  Params().put("CarParams", CP.to_bytes())
+  time.sleep(0.5)
+  messaging.drain_sock(logcan)
+
+def radar_rcv_callback(msg, CP):
+  if msg.which() != "can":
+    return []
+
+  # hyundai and subaru don't have radar
+  radar_msgs = {"honda": [0x445], "toyota": [0x19f, 0x22f], "gm": [0x475],
+                "hyundai": [], "chrysler": [0x2d4], "subaru": []}.get(CP.carName, None)
+
+  if radar_msgs is None:
+    raise NotImplementedError
+
+  for m in msg.can:
+    if m.src == 1 and m.address in radar_msgs:
+      return ["radarState", "liveTracks"]
+
+  return []
+
+def plannerd_rcv_callback(msg, CP):
+  if msg.which() in ["model", "radarState"]:
+    time.sleep(0.005)
+  else:
+    time.sleep(0.002)
+  return {"model": ["pathPlan"], "radarState": ["plan"]}.get(msg.which(), [])
+
+CONFIGS = [
+  ProcessConfig(
+    proc_name="controlsd",
+    pub_sub={
+      "can": ["controlsState", "carState", "carControl", "sendcan"],
+      "thermal":  [], "health": [], "liveCalibration": [], "driverMonitoring": [], "plan": [], "pathPlan": []
+    },
+    ignore=["logMonoTime", "controlsState.startMonoTime", "controlsState.cumLagMs"],
+    init_callback=fingerprint,
+    should_recv_callback=None,
+  ),
+  ProcessConfig(
+    proc_name="radard",
+    pub_sub={
+      "can": ["radarState", "liveTracks"],
+      "liveParameters":  [], "controlsState":  [], "model":  [],
+    },
+    ignore=["logMonoTime", "radarState.cumLagMs"],
+    init_callback=get_car_params,
+    should_recv_callback=radar_rcv_callback,
+  ),
+  ProcessConfig(
+    proc_name="plannerd",
+    pub_sub={
+      "model": ["pathPlan"], "radarState": ["plan"],
+      "carState": [], "controlsState": [], "liveParameters": [],
+    },
+    ignore=["logMonoTime", "valid", "plan.processingDelay"],
+    init_callback=get_car_params,
+    should_recv_callback=plannerd_rcv_callback,
+  ),
+  ProcessConfig(
+    proc_name="calibrationd",
+    pub_sub={
+      "cameraOdometry": ["liveCalibration"]
+    },
+    ignore=["logMonoTime"],
+    init_callback=get_car_params,
+    should_recv_callback=None,
+  ),
+]
+
+def replay_process(cfg, lr):
+  gc.disable()  # gc can occasionally cause canparser to timeout
+
+  pub_socks, sub_socks = {}, {}
+  for pub, sub in cfg.pub_sub.iteritems():
+    pub_socks[pub] = messaging.pub_sock(service_list[pub].port)
+
+    for s in sub:
+      sub_socks[s] = messaging.sub_sock(service_list[s].port)
+
+  all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
+  pub_msgs = filter(lambda msg: msg.which() in pub_socks.keys(), all_msgs)
+
+  params = Params()
+  params.manager_start()
+  params.put("Passive", "0")
+
+  manager.gctx = {}
+  manager.prepare_managed_process(cfg.proc_name)
+  manager.start_managed_process(cfg.proc_name)
+  time.sleep(3)   # Wait for started process to be ready
+
+  if cfg.init_callback is not None:
+    cfg.init_callback(all_msgs, pub_socks, sub_socks)
+
+  CP = car.CarParams.from_bytes(params.get("CarParams", block=True))
+
+  log_msgs = []
+  for msg in tqdm(pub_msgs):
+    if cfg.should_recv_callback is not None:
+      recv_socks = cfg.should_recv_callback(msg, CP)
+    else:
+      recv_socks = cfg.pub_sub[msg.which()]
+
+    pub_socks[msg.which()].send(msg.as_builder().to_bytes())
+
+    if len(recv_socks):
+      # TODO: add timeout
+      for sock in recv_socks:
+        m = messaging.recv_one(sub_socks[sock])
+
+        # make these values fixed for faster comparison
+        m_builder = m.as_builder()
+        m_builder.logMonoTime = 0
+        m_builder.valid = True
+        log_msgs.append(m_builder.as_reader())
+
+  gc.enable()
+  manager.kill_managed_process(cfg.proc_name)
+  return log_msgs
+

selfdrive/test/tests/process_replay/ref_commit

@@ -0,0 +1 @@
+e3388c62ffb80f4b3ca8721da56a581a93c44e79

selfdrive/test/tests/process_replay/test_processes.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python2
+import os
+import requests
+import sys
+import tempfile
+
+from selfdrive.test.tests.process_replay.compare_logs import compare_logs
+from selfdrive.test.tests.process_replay.process_replay import replay_process, CONFIGS
+from tools.lib.logreader import LogReader
+
+segments = [
+  "0375fdf7b1ce594d|2019-06-13--08-32-25--3", # HONDA.ACCORD
+  "99c94dc769b5d96e|2019-08-03--14-19-59--2", # HONDA.CIVIC
+  "cce908f7eb8db67d|2019-08-02--15-09-51--3", # TOYOTA.COROLLA_TSS2
+  "7ad88f53d406b787|2019-07-09--10-18-56--8", # GM.VOLT
+  "704b2230eb5190d6|2019-07-06--19-29-10--0", # HYUNDAI.KIA_SORENTO
+  "b6e1317e1bfbefa6|2019-07-06--04-05-26--5", # CHRYSLER.JEEP_CHEROKEE
+  "7873afaf022d36e2|2019-07-03--18-46-44--0", # SUBARU.IMPREZA
+]
+
+def get_segment(segment_name):
+  route_name, segment_num = segment_name.rsplit("--", 1)
+  rlog_url = "https://commadataci.blob.core.windows.net/openpilotci/%s/%s/rlog.bz2" \
+                % (route_name.replace("|", "/"), segment_num)
+  r = requests.get(rlog_url)
+  if r.status_code != 200:
+    return None
+
+  with tempfile.NamedTemporaryFile(delete=False, suffix=".bz2") as f:
+    f.write(r.content)
+    return f.name
+
+if __name__ == "__main__":
+
+  process_replay_dir = os.path.dirname(os.path.abspath(__file__))
+  ref_commit_fn = os.path.join(process_replay_dir, "ref_commit")
+
+  if not os.path.isfile(ref_commit_fn):
+    print "couldn't find reference commit"
+    sys.exit(1)
+
+  ref_commit = open(ref_commit_fn).read().strip()
+  print "***** testing against commit %s *****" % ref_commit
+
+  results = {}
+  for segment in segments:
+    print "***** testing route segment %s *****\n" % segment
+
+    results[segment] = {}
+
+    rlog_fn = get_segment(segment)
+
+    if rlog_fn is None:
+      print "failed to get segment %s" % segment
+      sys.exit(1)
+
+    lr = LogReader(rlog_fn)
+
+    for cfg in CONFIGS:
+      log_msgs = replay_process(cfg, lr)
+
+      log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
+
+      if not os.path.isfile(log_fn):
+        url = "https://commadataci.blob.core.windows.net/openpilotci/"
+        req = requests.get(url + os.path.basename(log_fn))
+        if req.status_code != 200:
+          results[segment][cfg.proc_name] = "failed to download comparison log"
+          continue
+
+        with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
+          f.write(req.content)
+          f.flush()
+          f.seek(0)
+          cmp_log_msgs = list(LogReader(f.name))
+      else:
+        cmp_log_msgs = list(LogReader(log_fn))
+
+      diff = compare_logs(cmp_log_msgs, log_msgs, cfg.ignore)
+      results[segment][cfg.proc_name] = diff
+    os.remove(rlog_fn)
+
+  failed = False
+  with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f:
+    f.write("***** tested against commit %s *****\n" % ref_commit)
+
+    for segment, result in results.items():
+      f.write("***** differences for segment %s *****\n" % segment)
+      print "***** results for segment %s *****" % segment
+
+      for proc, diff in result.items():
+        f.write("*** process: %s ***\n" % proc)
+        print "\t%s" % proc
+
+        if isinstance(diff, str):
+          print "\t\t%s" % diff
+          failed = True
+        elif len(diff):
+          cnt = {}
+          for d in diff:
+            f.write("\t%s\n" % str(d))
+
+            k = str(d[1])
+            cnt[k] = 1 if k not in cnt else cnt[k] + 1
+
+          for k, v in sorted(cnt.items()):
+            print "\t\t%s: %s" % (k, v)
+          failed = True
+
+    if failed:
+      print "TEST FAILED"
+    else:
+      print "TEST SUCCEEDED"
+
+  print "\n\nTo update the reference logs for this test run:"
+  print "./update_refs.py"
+
+  sys.exit(int(failed))

selfdrive/test/tests/process_replay/update_refs.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python2
+import os
+import sys
+
+from selfdrive.test.openpilotci_upload import upload_file
+from selfdrive.test.tests.process_replay.compare_logs import save_log
+from selfdrive.test.tests.process_replay.process_replay import replay_process, CONFIGS
+from selfdrive.test.tests.process_replay.test_processes import segments, get_segment
+from selfdrive.version import get_git_commit
+from tools.lib.logreader import LogReader
+
+if __name__ == "__main__":
+
+  no_upload = "--no-upload" in sys.argv
+
+  process_replay_dir = os.path.dirname(os.path.abspath(__file__))
+  ref_commit_fn = os.path.join(process_replay_dir, "ref_commit")
+
+  ref_commit = get_git_commit()
+  with open(ref_commit_fn, "w") as f:
+    f.write(ref_commit)
+
+  for segment in segments:
+    rlog_fn = get_segment(segment)
+
+    if rlog_fn is None:
+      print "failed to get segment %s" % segment
+      sys.exit(1)
+
+    lr = LogReader(rlog_fn)
+
+    for cfg in CONFIGS:
+      log_msgs = replay_process(cfg, lr)
+      log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
+      save_log(log_fn, log_msgs)
+
+      if not no_upload:
+        upload_file(log_fn, os.path.basename(log_fn))
+        os.remove(log_fn)
+    os.remove(rlog_fn)
+
+  print "done"