Run device tests directly (#38397)

Jenkinsfile

@@ -206,35 +206,35 @@ node {
         deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
           step("build openpilot", "cd openpilot/system/manager && ./build.py"),
           step("check dirty", "tools/release/check-dirty.sh"),
-          step("onroad tests", "pytest openpilot/selfdrive/test/test_onroad.py -s", [timeout: 60]),
+          step("onroad tests", "./openpilot/selfdrive/test/test_onroad.py", [timeout: 60]),
         ])
       },
       'HW + Unit Tests': {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
           step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test power draw", "pytest -s openpilot/selfdrive/test//test_power_draw.py"),
-          step("test encoder", "pytest openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
-          step("test manager", "pytest openpilot/system/manager/test/test_manager.py"),
+          step("test power draw", "./openpilot/selfdrive/test/test_power_draw.py"),
+          step("test encoder", "./openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
+          step("test manager", "./openpilot/system/manager/test/test_manager.py"),
         ])
       },
       'camerad OX03C10': {
         deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
           step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test pandad", "pytest openpilot/selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'camerad OS04C10': {
         deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
           step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test pandad", "pytest openpilot/selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
+          step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
         ])
       },
       'sensord': {
         deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
           step("build", "cd openpilot/system/manager && ./build.py"),
-          step("test sensord", "pytest openpilot/system/sensord/tests/test_sensord.py"),
+          step("test sensord", "./openpilot/system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
@@ -246,9 +246,9 @@ node {
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           step("build openpilot", "cd openpilot/system/manager && ./build.py"),
-          step("test pandad loopback", "pytest openpilot/selfdrive/pandad/tests/test_pandad_loopback.py"),
-          step("test pandad spi", "pytest openpilot/selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test amp", "pytest openpilot/common/hardware/tici/tests/test_amplifier.py"),
+          step("test pandad loopback", "./openpilot/selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad spi", "./openpilot/selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test amp", "./openpilot/common/hardware/tici/tests/test_amplifier.py"),
         ])
       },
 

openpilot/common/hardware/tici/tests/test_amplifier.py

@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
+
 import time
 import subprocess
+import unittest
 
 from panda import Panda
 from openpilot.common.test import OpenpilotTestCase
@@ -62,3 +65,7 @@ class TestAmplifier(OpenpilotTestCase):
         break
     else:
       self.fail("didn't hit any i2c errors")
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/selfdrive/pandad/tests/test_pandad.py

@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
+
 import os
 import time
+import unittest
 
 from openpilot.common.test import OpenpilotTestCase
 import openpilot.cereal.messaging as messaging
@@ -79,3 +82,7 @@ class TestPandad(OpenpilotTestCase):
     assert not PandaDFU.list()
 
     self._run_test()
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/selfdrive/pandad/tests/test_pandad_loopback.py

@@ -1,7 +1,10 @@
+#!/usr/bin/env python3
+
 import os
 import copy
 import random
 import time
+import unittest
 from collections import defaultdict
 from pprint import pprint
 
@@ -114,3 +117,7 @@ class TestBoarddLoopback(OpenpilotTestCase):
       pprint(sm['pandaStates'])  # may drop messages due to RX buffer overflow
       for bus in sent_loopback.keys():
         assert not len(sent_loopback[bus]), f"loop {i}: bus {bus} missing {len(sent_loopback[bus])} out of {sent_total[bus]} messages"
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/selfdrive/pandad/tests/test_pandad_spi.py

@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
+
 import os
 import time
+import unittest
 import numpy as np
 import random
 
@@ -105,3 +108,7 @@ class TestBoarddSpi(OpenpilotTestCase):
     with subtests.test(msg="CAN traffic"):
       print(f"Sent {total_sent_count} CAN messages, got {total_recv_count} back. {total_recv_count/(total_sent_count+1e-4):.2%} received")
       assert total_recv_count > 20
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/selfdrive/test/test_onroad.py

@@ -1,9 +1,12 @@
+#!/usr/bin/env python3
+
 import math
 import json
 import os
 import shutil
 import subprocess
 import time
+import unittest
 import numpy as np
 from collections import Counter, defaultdict
 from pathlib import Path
@@ -442,3 +445,7 @@ class TestOnroad(OpenpilotTestCase):
     eng = [m.selfdriveState.engageable for m in self.msgs['selfdriveState'][offset:]]
     assert all(eng), \
            f"Not engageable for whole segment:\n- selfdriveState.engageable: {Counter(eng)}\n- No entry events: {no_entries}"
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/selfdrive/test/test_power_draw.py

@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
+
 from collections import defaultdict, deque
 import time
+import unittest
 import numpy as np
 from dataclasses import dataclass
 from openpilot.common.test import OpenpilotTestCase
@@ -123,3 +126,7 @@ class TestPowerDraw(OpenpilotTestCase):
         assert self.valid_power_draw(proc, cur), f"expected {expected:.2f}W, got {cur:.2f}W"
     print(tabulate(tab))
     print(f"Baseline {baseline:.2f}W\n")
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/system/camerad/test/test_camerad.py

@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
+
 import os
 import time
+import unittest
 import numpy as np
 
 from openpilot.common.parameterized import parameterized
@@ -158,3 +161,7 @@ class TestCamerad(OpenpilotTestCase):
     assert np.max([ np.max(np.diff(ts[c]['requestId'])) for c in CAMERAS ]) > 1
 
     self._sanity_checks(ts)
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/system/loggerd/tests/test_encoder.py

@@ -1,8 +1,11 @@
+#!/usr/bin/env python3
+
 import math
 import os
 import shutil
 import subprocess
 import time
+import unittest
 from pathlib import Path
 
 from tqdm import trange
@@ -144,3 +147,7 @@ class TestEncoder(OpenpilotTestCase):
       managed_processes['encoderd'].stop()
       managed_processes['camerad'].stop()
       managed_processes['sensord'].stop()
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/system/manager/test/test_manager.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 import unittest
 import signal
@@ -76,3 +78,7 @@ class TestManager(OpenpilotTestCase):
         if p.sigkill:
           exit_codes = [-signal.SIGKILL]
         assert exit_code in exit_codes, f"{p.name} died with {exit_code}"
+
+
+if __name__ == "__main__":
+  unittest.main()

openpilot/system/sensord/tests/test_sensord.py

@@ -1,6 +1,9 @@
+#!/usr/bin/env python3
+
 import os
 import subprocess
 import time
+import unittest
 import numpy as np
 from collections import namedtuple, defaultdict
 
@@ -183,3 +186,7 @@ class TestSensord(OpenpilotTestCase):
     time.sleep(1)
     state_two = get_irq_count(self.sensord_irq)
     assert state_one == state_two, "Interrupts received after sensord stop!"
+
+
+if __name__ == "__main__":
+  unittest.main()

tools/release/build_release.sh

@@ -85,7 +85,7 @@ git commit --amend -m "openpilot v$VERSION"
 
 # Run tests
 cd $BUILD_DIR
-RELEASE=1 pytest -n0 -s openpilot/selfdrive/test/test_onroad.py
+RELEASE=1 ./openpilot/selfdrive/test/test_onroad.py
 #pytest openpilot/selfdrive/car/tests/test_car_interfaces.py
 
 echo "[-] pushing release T=$SECONDS"