dragonpilot beta3

date: 2023-10-09T10:55:55
commit: 91b6e3aecd7170f24bccacb10c515ec281c30295

selfdrive/test/helpers.py

@@ -1,12 +1,13 @@
 import os
 import time
+
 from functools import wraps
 
 import cereal.messaging as messaging
-from common.params import Params
-from selfdrive.manager.process_config import managed_processes
-from system.hardware import PC
-from system.version import training_version, terms_version
+from openpilot.common.params import Params
+from openpilot.selfdrive.manager.process_config import managed_processes
+from openpilot.system.hardware import PC
+from openpilot.system.version import training_version, terms_version
 
 
 def set_params_enabled():
@@ -67,3 +68,7 @@ def with_processes(processes, init_time=0, ignore_stopped=None):
 
     return wrap
   return wrapper
+
+
+def noop(*args, **kwargs):
+  pass

selfdrive/test/setup_device_ci.sh

@@ -56,31 +56,68 @@ sleep infinity
 EOF
 chmod +x $CONTINUE_PATH
 
+safe_checkout() {
+  # completely clean TEST_DIR
+
+  cd $SOURCE_DIR
+
+  # cleanup orphaned locks
+  find .git -type f -name "*.lock" -exec rm {} +
+
+  git reset --hard
+  git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin $GIT_COMMIT
+  find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
+  git reset --hard $GIT_COMMIT
+  git checkout $GIT_COMMIT
+  git clean -xdff
+  git submodule sync
+  git submodule update --init --recursive
+  git submodule foreach --recursive "git reset --hard && git clean -xdff"
+
+  git lfs pull
+  (ulimit -n 65535 && git lfs prune)
+
+  echo "git checkout done, t=$SECONDS"
+  du -hs $SOURCE_DIR $SOURCE_DIR/.git
+
+  if [ -z "SKIP_COPY" ]; then
+    rsync -a --delete $SOURCE_DIR $TEST_DIR
+  fi
+}
+
+unsafe_checkout() {
+  # checkout directly in test dir, leave old build products
+
+  cd $TEST_DIR
+
+  # cleanup orphaned locks
+  find .git -type f -name "*.lock" -exec rm {} +
+
+  git fetch --no-tags --no-recurse-submodules -j8 --verbose --depth 1 origin $GIT_COMMIT
+  git checkout --force --no-recurse-submodules $GIT_COMMIT
+  git reset --hard $GIT_COMMIT
+  git clean -df
+  git submodule sync
+  git submodule update --init --recursive
+  git submodule foreach --recursive "git reset --hard && git clean -df"
+
+  git lfs pull
+  (ulimit -n 65535 && git lfs prune)
+}
+
+export GIT_PACK_THREADS=8
+
 # set up environment
 if [ ! -d "$SOURCE_DIR" ]; then
   git clone https://github.com/commaai/openpilot.git $SOURCE_DIR
 fi
-cd $SOURCE_DIR
 
-# cleanup orphaned locks
-find .git -type f -name "*.lock" -exec rm {} +
-
-git reset --hard
-git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin $GIT_COMMIT
-find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
-git reset --hard $GIT_COMMIT
-git checkout $GIT_COMMIT
-git clean -xdff
-git submodule sync
-git submodule update --init --recursive
-git submodule foreach --recursive "git reset --hard && git clean -xdff"
-
-git lfs pull
-(ulimit -n 65535 && git lfs prune)
-
-echo "git checkout done, t=$SECONDS"
-du -hs $SOURCE_DIR $SOURCE_DIR/.git
-
-rsync -a --delete $SOURCE_DIR $TEST_DIR
+if [ ! -z "$UNSAFE" ]; then
+  echo "doing unsafe checkout"
+  unsafe_checkout
+else
+  echo "doing safe checkout"
+  safe_checkout
+fi
 
 echo "$TEST_DIR synced with $GIT_COMMIT, t=$SECONDS"

selfdrive/test/test_onroad.py

@@ -13,15 +13,15 @@ from pathlib import Path
 
 from cereal import car
 import cereal.messaging as messaging
-from cereal.services import service_list
-from common.basedir import BASEDIR
-from common.timeout import Timeout
-from common.params import Params
-from selfdrive.controls.lib.events import EVENTS, ET
-from system.hardware import HARDWARE
-from system.loggerd.config import ROOT
-from selfdrive.test.helpers import set_params_enabled, release_only
-from tools.lib.logreader import LogReader
+from cereal.services import SERVICE_LIST
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.timeout import Timeout
+from openpilot.common.params import Params
+from openpilot.selfdrive.controls.lib.events import EVENTS, ET
+from openpilot.system.hardware import HARDWARE
+from openpilot.selfdrive.test.helpers import set_params_enabled, release_only
+from openpilot.system.hardware.hw import Paths
+from openpilot.tools.lib.logreader import LogReader
 
 # Baseline CPU usage by process
 PROCS = {
@@ -34,15 +34,15 @@ PROCS = {
   "selfdrive.controls.plannerd": 16.5,
   "./_ui": 18.0,
   "selfdrive.locationd.paramsd": 9.0,
-  "./_sensord": 12.0,
+  "./_sensord": 7.0,
   "selfdrive.controls.radard": 4.5,
-  "./_modeld": 4.48,
-  "./_dmonitoringmodeld": 5.0,
-  "./_navmodeld": 1.0,
+  "selfdrive.modeld.modeld": 8.0,
+  "selfdrive.modeld.dmonitoringmodeld": 8.0,
+  "selfdrive.modeld.navmodeld": 1.0,
   "selfdrive.thermald.thermald": 3.87,
   "selfdrive.locationd.calibrationd": 2.0,
   "selfdrive.locationd.torqued": 5.0,
-  "./_soundd": (15.0, 65.0),
+  "./_soundd": (1.0, 65.0),
   "selfdrive.monitoring.dmonitoringd": 4.0,
   "./proclogd": 1.54,
   "system.logmessaged": 0.2,
@@ -102,7 +102,7 @@ class TestOnroad(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
     if "DEBUG" in os.environ:
-      segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog")), Path(ROOT).iterdir())
+      segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog")), Path(Paths.log_root()).iterdir())
       segs = sorted(segs, key=lambda x: x.stat().st_mtime)
       print(segs[-3])
       cls.lr = list(LogReader(os.path.join(segs[-3], "rlog")))
@@ -115,8 +115,8 @@ class TestOnroad(unittest.TestCase):
     params.remove("CurrentRoute")
     set_params_enabled()
     os.environ['TESTING_CLOSET'] = '1'
-    if os.path.exists(ROOT):
-      shutil.rmtree(ROOT)
+    if os.path.exists(Paths.log_root()):
+      shutil.rmtree(Paths.log_root())
     os.system("rm /dev/shm/*")
 
     # Make sure athena isn't running
@@ -143,8 +143,8 @@ class TestOnroad(unittest.TestCase):
 
         while len(cls.segments) < 3:
           segs = set()
-          if Path(ROOT).exists():
-            segs = set(Path(ROOT).glob(f"{route}--*"))
+          if Path(Paths.log_root()).exists():
+            segs = set(Path(Paths.log_root()).glob(f"{route}--*"))
           cls.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1]))
           time.sleep(2)
 
@@ -179,7 +179,7 @@ class TestOnroad(unittest.TestCase):
         continue
 
       with self.subTest(service=s):
-        assert len(msgs) >= math.floor(service_list[s].frequency*55)
+        assert len(msgs) >= math.floor(SERVICE_LIST[s].frequency*55)
 
   def test_cloudlog_size(self):
     msgs = [m for m in self.lr if m.which() == 'logMessage']
@@ -356,7 +356,7 @@ class TestOnroad(unittest.TestCase):
         raise Exception(f"missing {s}")
 
       ts = np.diff(msgs) / 1e9
-      dt = 1 / service_list[s].frequency
+      dt = 1 / SERVICE_LIST[s].frequency
 
       try:
         np.testing.assert_allclose(np.mean(ts), dt, rtol=0.03, err_msg=f"{s} - failed mean timing check")

selfdrive/test/test_time_to_onroad.py

@@ -4,9 +4,9 @@ import time
 import subprocess
 
 import cereal.messaging as messaging
-from common.basedir import BASEDIR
-from common.timeout import Timeout
-from selfdrive.test.helpers import set_params_enabled
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.timeout import Timeout
+from openpilot.selfdrive.test.helpers import set_params_enabled
 
 
 def test_time_to_onroad():
@@ -16,7 +16,7 @@ def test_time_to_onroad():
   proc = subprocess.Popen(["python", manager_path])
 
   start_time = time.monotonic()
-  sm = messaging.SubMaster(['controlsState', 'deviceState'])
+  sm = messaging.SubMaster(['controlsState', 'deviceState', 'carEvents'])
   try:
     # wait for onroad
     with Timeout(20, "timed out waiting to go onroad"):
@@ -38,7 +38,7 @@ def test_time_to_onroad():
     # once we're enageable, must be for the next few seconds
     for _ in range(500):
       sm.update(100)
-      assert sm['controlsState'].engageable
+      assert sm['controlsState'].engageable, f"events: {sm['carEvents']}"
   finally:
     proc.terminate()
     if proc.wait(60) is None: