jenkins speedups (#38328)

* power draw first

* speedup encoder

* speed up hardware test suites

* refresh IRQ actions after hardware setup

* skip unused big model in device checkout

* move LFS pruning off device checkout path

* wtf is that for?

* reverts

* locality

* rm that

openpilot/common/SConscript

@@ -13,7 +13,7 @@ Export('_common')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
-              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              ['tests/test_runner.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
               LIBS=[_common, 'json11', 'zmq', 'pthread'])
 
 # Cython bindings

openpilot/common/hardware/tici/tests/test_amplifier.py

@@ -1,6 +1,5 @@
 import pytest
 import time
-import random
 import subprocess
 
 from panda import Panda
@@ -57,7 +56,7 @@ class TestAmplifier:
       time.sleep(0.1)
 
       self.panda.set_siren(True)
-      time.sleep(random.randint(0, 5))
+      time.sleep(0.1)
 
       amp = Amplifier(debug=True)
       r = amp.initialize_configuration()

openpilot/selfdrive/pandad/tests/test_pandad_spi.py

@@ -6,6 +6,7 @@ import random
 
 import openpilot.cereal.messaging as messaging
 from openpilot.cereal.services import SERVICE_LIST
+from openpilot.common.timeout import Timeout
 from openpilot.selfdrive.test.helpers import with_processes
 from openpilot.selfdrive.pandad.tests.test_pandad_loopback import setup_pandad, send_random_can_messages
 
@@ -26,9 +27,14 @@ class TestBoarddSpi:
 
     sendcan = messaging.pub_sock('sendcan')
     socks = {s: messaging.sub_sock(s, conflate=False, timeout=100) for s in ('can', 'pandaStates', 'peripheralState')}
-    time.sleep(2)
-    for s in socks.values():
-      messaging.drain_sock_raw(s)
+    readiness_services = {'pandaStates', 'peripheralState'}
+    ready = set()
+    with Timeout(2, "pandad services didn't become ready"):
+      while not readiness_services <= ready:
+        for service, sock in socks.items():
+          if messaging.drain_sock_raw(sock):
+            ready.add(service)
+        time.sleep(0.01)
 
     total_recv_count = 0
     total_sent_count = 0

openpilot/selfdrive/test/setup_device_ci.sh

@@ -3,6 +3,7 @@
 set -e
 set -x
 
+
 if [ -z "$SOURCE_DIR" ]; then
   echo "SOURCE_DIR must be set"
   exit 1
@@ -55,6 +56,28 @@ sleep infinity
 EOF
 chmod +x $CONTINUE_PATH
 
+export GIT_LFS_SKIP_SMUDGE=1
+pull_lfs() {
+  # The big driving model is not used on these devices yet. Keep its pointer in
+  # the worktree, but don't download or copy the 1.8 GB LFS object.
+  LFS_EXCLUDE="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+
+  git config --local lfs.fetchexclude "$LFS_EXCLUDE"
+  git lfs pull --exclude="$LFS_EXCLUDE"
+  if git cat-file -e "HEAD:$LFS_EXCLUDE"; then
+    rm -f "$LFS_EXCLUDE"
+    git checkout -- "$LFS_EXCLUDE"
+
+    # `git lfs prune` retains objects referenced by HEAD, even when excluded.
+    # Remove this one explicitly so safe checkout doesn't rsync it either.
+    oid=$(git show "HEAD:$LFS_EXCLUDE" | sed -n 's/^oid sha256://p')
+    lfs_objects=$(git lfs env | sed -n 's/^LocalMediaDir=//p')
+    if [[ "$oid" =~ ^[0-9a-f]{64}$ && -n "$lfs_objects" ]]; then
+      rm -f "$lfs_objects/${oid:0:2}/${oid:2:2}/$oid"
+    fi
+  fi
+}
+
 safe_checkout() {
   # completely clean TEST_DIR
 
@@ -74,8 +97,7 @@ safe_checkout() {
   git submodule update --init --recursive
   git submodule foreach --recursive "git reset --hard && git clean -xdff"
 
-  git lfs pull
-  (ulimit -n 65535 && git lfs prune)
+  pull_lfs
 
   echo "git checkout done, t=$SECONDS"
   du -hs $SOURCE_DIR $SOURCE_DIR/.git
@@ -100,8 +122,7 @@ unsafe_checkout() {( set -e
   git submodule update --init --recursive
   git submodule foreach --recursive "git reset --hard && git clean -df"
 
-  git lfs pull
-  (ulimit -n 65535 && git lfs prune)
+  pull_lfs
 )}
 
 export GIT_PACK_THREADS=8

openpilot/selfdrive/test/test_power_draw.py

@@ -14,7 +14,7 @@ from openpilot.common.hardware.tici.power_monitor import get_power
 from openpilot.system.manager.process_config import managed_processes
 from openpilot.system.manager.manager import manager_cleanup
 
-SAMPLE_TIME = 8       # seconds to sample power
+SAMPLE_TIME = 2       # seconds to sample power
 MAX_WARMUP_TIME = 30  # seconds to wait for SAMPLE_TIME consecutive valid samples
 
 @dataclass
@@ -44,9 +44,6 @@ class TestPowerDraw:
   def setup_method(self):
     Params().put("CarParams", get_demo_car_params().to_bytes(), block=True)
 
-    # wait a bit for power save to disable
-    time.sleep(5)
-
   def teardown_method(self):
     manager_cleanup()
 

openpilot/system/loggerd/tests/test_encoder.py

@@ -1,13 +1,11 @@
 import math
 import os
 import pytest
-import random
 import shutil
 import subprocess
 import time
 from pathlib import Path
 
-from openpilot.common.parameterized import parameterized
 from tqdm import trange
 
 from openpilot.common.params import Params
@@ -51,9 +49,8 @@ class TestEncoder:
     return os.path.join(Paths.log_root(), last_route)
 
   # TODO: this should run faster than real time
-  @parameterized.expand([(True, ), (False, )])
-  def test_log_rotation(self, record_front):
-    Params().put_bool("RecordFront", record_front, block=True)
+  def test_log_rotation(self):
+    Params().put_bool("RecordFront", True, block=True)
 
     managed_processes['sensord'].start()
     managed_processes['loggerd'].start()
@@ -62,7 +59,7 @@ class TestEncoder:
     time.sleep(1.0)
     managed_processes['camerad'].start()
 
-    num_segments = int(os.getenv("SEGMENTS", random.randint(2, 8)))
+    num_segments = 3
 
     # wait for loggerd to make the dir for first segment
     route_prefix_path = None
@@ -78,9 +75,6 @@ class TestEncoder:
       counts = []
       first_frames = []
       for camera, fps, size_lambda, encode_idx_name in CAMERAS:
-        if not record_front and "dcamera" in camera:
-          continue
-
         file_path = f"{route_prefix_path}--{i}/{camera}"
 
         # check file exists

openpilot/system/manager/test/test_manager.py

@@ -27,10 +27,6 @@ class TestManager:
   def teardown_method(self):
     manager.manager_cleanup()
 
-  def test_manager_prepare(self):
-    os.environ['PREPAREONLY'] = '1'
-    manager.main()
-
   def test_duplicate_procs(self):
     assert len(procs) == len(managed_processes), "Duplicate process names"