From 83000c571cb4fbc3ec2ef5ff06f0904e5a04c1a0 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 28 Jul 2026 16:42:38 -0700 Subject: [PATCH] no slow unittests (#38480) * rm test_regen * not slow * skip bridge * rm that one too * just op test * just test --- .github/workflows/tests.yaml | 3 +- openpilot/common/test.py | 11 +-- .../test/process_replay/test_regen.py | 40 -------- openpilot/selfdrive/test/test_onroad.py | 1 - openpilot/tools/lib/tests/test_logreader.py | 94 ++++++++++++------- openpilot/tools/sim/tests/test_sim_bridge.py | 2 +- tools/test_runner.py | 2 - 7 files changed, 64 insertions(+), 89 deletions(-) delete mode 100644 openpilot/selfdrive/test/process_replay/test_regen.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 85d66d03ec..85475189f8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -108,8 +108,7 @@ jobs: timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }} env: RAYLIB_BACKEND: headless - run: | - MAX_EXAMPLES=1 SKIP_SLOW=1 tools/test_runner.py --durations=0 + run: tools/op.sh test process_replay: name: process replay diff --git a/openpilot/common/test.py b/openpilot/common/test.py index e00fe0cfb3..6ccc76c5e3 100644 --- a/openpilot/common/test.py +++ b/openpilot/common/test.py @@ -22,12 +22,10 @@ def clean_env(): class OpenpilotTestCase(unittest.TestCase): - """TestCase with openpilot's per-test isolation and legacy hook support.""" + """TestCase with openpilot's per-test isolation.""" TICI_TEST = False - SKIP_TICI_SETUP = False SHARED_DOWNLOAD_CACHE = False - SLOW_TEST = False def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) @@ -63,8 +61,7 @@ class OpenpilotTestCase(unittest.TestCase): def run(self, result=None): # This boundary cannot live in setUp/tearDown: existing unittest classes # are allowed to override those hooks without calling super(). - if ((self.SLOW_TEST and os.environ.get("SKIP_SLOW")) or - (self.TICI_TEST and not TICI) or getattr(type(self), "__unittest_skip__", False)): + if (self.TICI_TEST and not TICI) or getattr(type(self), "__unittest_skip__", False): return super().run(result) test_env = clean_env() test_env.__enter__() @@ -83,8 +80,6 @@ class OpenpilotTestCase(unittest.TestCase): @classmethod def setUpClass(cls): super().setUpClass() - if cls.SLOW_TEST and os.environ.get("SKIP_SLOW"): - raise unittest.SkipTest("slow test") if cls.TICI_TEST and not TICI: raise unittest.SkipTest("Skipping tici test on PC") cls._class_env = clean_env() @@ -108,7 +103,7 @@ class OpenpilotTestCase(unittest.TestCase): if self.TICI_TEST and not TICI: self.skipTest("Skipping tici test on PC") - if self.TICI_TEST and not self.SKIP_TICI_SETUP: + if self.TICI_TEST: HARDWARE.initialize_hardware() HARDWARE.set_power_save(False) subprocess.run(["pkill", "-9", "-f", "athena"], check=False) diff --git a/openpilot/selfdrive/test/process_replay/test_regen.py b/openpilot/selfdrive/test/process_replay/test_regen.py deleted file mode 100644 index 1a3dc8442a..0000000000 --- a/openpilot/selfdrive/test/process_replay/test_regen.py +++ /dev/null @@ -1,40 +0,0 @@ -from openpilot.common.test import OpenpilotTestCase -from openpilot.common.parameterized import parameterized - -from openpilot.selfdrive.test.process_replay.regen import regen_segment -from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled -from openpilot.tools.lib.openpilotci import get_url -from openpilot.tools.lib.logreader import LogReader -from openpilot.tools.lib.framereader import FrameReader - -TESTED_SEGMENTS = [ - ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA.TOYOTA_PRIUS: NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED - # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation - # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"), # MAZDA.CX9_2021: TICI, incomplete managerState - # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"), # FORD.BRONCO_SPORT_MK1: TICI -] - - -def ci_setup_data_readers(route, sidx): - lr = LogReader(get_url(route, sidx, "rlog.bz2")) - frs = { - 'roadCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")), - 'driverCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")), - } - if next((True for m in lr if m.which() == "wideRoadCameraState"), False): - frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera.hevc")) - - return lr, frs - - -class TestRegen(OpenpilotTestCase): - SLOW_TEST = True - - @parameterized.expand(TESTED_SEGMENTS) - def test_engaged(self, case_name, segment): - route, sidx = segment.rsplit("--", 1) - lr, frs = ci_setup_data_readers(route, sidx) - output_logs = regen_segment(lr, frs, disable_tqdm=True) - - engaged = check_openpilot_enabled(output_logs) - assert engaged, f"openpilot not engaged in {case_name}" diff --git a/openpilot/selfdrive/test/test_onroad.py b/openpilot/selfdrive/test/test_onroad.py index 2504afb7cd..57bf48ee81 100755 --- a/openpilot/selfdrive/test/test_onroad.py +++ b/openpilot/selfdrive/test/test_onroad.py @@ -106,7 +106,6 @@ def cputime_total(ct): class TestOnroad(OpenpilotTestCase): TICI_TEST = True - SKIP_TICI_SETUP = True @classmethod def setup_class(cls): diff --git a/openpilot/tools/lib/tests/test_logreader.py b/openpilot/tools/lib/tests/test_logreader.py index edf2314ee1..9d5691abd8 100644 --- a/openpilot/tools/lib/tests/test_logreader.py +++ b/openpilot/tools/lib/tests/test_logreader.py @@ -11,9 +11,9 @@ from openpilot.common.test import OpenpilotTestCase from openpilot.common.parameterized import parameterized from openpilot.cereal import log as capnp_log -from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, parse_indirect, ReadMode -from openpilot.tools.lib.file_sources import comma_api_source, InternalUnavailableException -from openpilot.tools.lib.route import SegmentRange +from openpilot.tools.lib.logreader import _LogFileReader, LogsUnavailable, LogIterable, LogReader, parse_indirect, ReadMode +from openpilot.tools.lib.file_sources import InternalUnavailableException +from openpilot.tools.lib.route import FileName, SegmentRange from openpilot.tools.lib.url_file import URLFileException NUM_SEGS = 17 # number of segments in the test route @@ -49,6 +49,45 @@ def setup_source_scenario(mocker, is_internal=False): class TestLogReader(OpenpilotTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.tmpdir = tempfile.TemporaryDirectory() + cls.qlog_path = os.path.join(cls.tmpdir.name, "qlog") + cls.rlog_path = os.path.join(cls.tmpdir.name, "rlog") + cls._write_test_log(cls.qlog_path, 10, include_car_params=True) + cls._write_test_log(cls.rlog_path, 100) + + @classmethod + def tearDownClass(cls): + cls.tmpdir.cleanup() + super().tearDownClass() + + @staticmethod + def _write_test_log(path, count, include_car_params=False): + events = [] + for i in range(count): + event = capnp_log.Event.new_message() + event.logMonoTime = count - i # deliberately unsorted + if include_car_params and i == 0: + event.init("carParams") + event.carParams.carFingerprint = "SUBARU OUTBACK 6TH GEN" + else: + event.init("can", 0) + events.append(event.to_bytes()) + + with open(path, "wb") as f: + f.write(b"".join(events)) + + def local_source(self, sr, seg_idxs, fns): + path = self.qlog_path if fns == FileName.QLOG else self.rlog_path + return dict.fromkeys(seg_idxs, path) + + def local_auto_source(self, sr, seg_idxs, fns): + if fns == FileName.RLOG: + return {} + return dict.fromkeys(seg_idxs, self.qlog_path) + @parameterized.expand([ (f"{TEST_ROUTE}", ALL_SEGS), (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS), @@ -143,31 +182,27 @@ class TestLogReader(OpenpilotTestCase): _ = SegmentRange(segment_range).seg_idxs assert api_call == max_seg_mock.called - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_modes(self): - qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.QLOG))) - rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.RLOG))) + qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.QLOG, sources=[self.local_source]))) + rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.RLOG, sources=[self.local_source]))) assert qlog_len * 6 < rlog_len - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_modes_from_name(self): - qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q"))) - rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/r"))) + qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q", sources=[self.local_source]))) + rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/r", sources=[self.local_source]))) assert qlog_len * 6 < rlog_len - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_list(self): - qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q"))) - qlog_len_2 = len(list(LogReader([f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"]))) + qlog_len = len(list(LogReader(self.qlog_path))) + qlog_len_2 = len(list(LogReader([self.qlog_path, self.qlog_path]))) assert qlog_len * 2 == qlog_len_2 - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_multiple_iterations(self, mocker): - init_mock = mocker.patch("openpilot.tools.lib.logreader._LogFileReader") - lr = LogReader(f"{TEST_ROUTE}/0/q") + init_mock = mocker.patch("openpilot.tools.lib.logreader._LogFileReader", wraps=_LogFileReader) + lr = LogReader(self.qlog_path) qlog_len1 = len(list(lr)) qlog_len2 = len(list(lr)) @@ -176,43 +211,33 @@ class TestLogReader(OpenpilotTestCase): assert qlog_len1 == qlog_len2 - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_helpers(self): - lr = LogReader(f"{TEST_ROUTE}/0/q") + lr = LogReader(self.qlog_path) assert lr.first("carParams").carFingerprint == "SUBARU OUTBACK 6TH GEN" assert 0 < len(list(lr.filter("carParams"))) < len(list(lr)) - @parameterized.expand([(True,), (False,)]) - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") - def test_run_across_segments(self, cache_enabled): - if cache_enabled: - os.environ.pop("DISABLE_FILEREADER_CACHE", None) - else: - os.environ["DISABLE_FILEREADER_CACHE"] = "1" - lr = LogReader(f"{TEST_ROUTE}/0:4") + def test_run_across_segments(self): + lr = LogReader([self.qlog_path] * 4) assert len(lr.run_across_segments(4, noop)) == len(list(lr)) - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_auto_mode(self, subtests, mocker): - lr = LogReader(f"{TEST_ROUTE}/0/q") + lr = LogReader(self.qlog_path) qlog_len = len(list(lr)) - log_paths_mock = mocker.patch("openpilot.tools.lib.route.Route.log_paths") - log_paths_mock.return_value = [None] * NUM_SEGS # Should fall back to qlogs since rlogs are not available with subtests.test("interactive_yes"): mocker.patch("sys.stdin", new=io.StringIO("y\n")) - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source]) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[self.local_auto_source]) log_len = len(list(lr)) assert qlog_len == log_len with subtests.test("interactive_no"): mocker.patch("sys.stdin", new=io.StringIO("n\n")) with self.assertRaises(LogsUnavailable): - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source]) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[self.local_auto_source]) with subtests.test("non_interactive"): - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, sources=[comma_api_source]) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, sources=[self.local_auto_source]) log_len = len(list(lr)) assert qlog_len == log_len @@ -226,12 +251,11 @@ class TestLogReader(OpenpilotTestCase): log_len = len(list(lr)) assert qlog_len == log_len - @unittest.skipIf(os.environ.get("SKIP_SLOW"), "slow test") def test_sort_by_time(self): - msgs = list(LogReader(f"{TEST_ROUTE}/0/q")) + msgs = list(LogReader(self.qlog_path)) assert msgs != sorted(msgs, key=lambda m: m.logMonoTime) - msgs = list(LogReader(f"{TEST_ROUTE}/0/q", sort_by_time=True)) + msgs = list(LogReader(self.qlog_path, sort_by_time=True)) assert msgs == sorted(msgs, key=lambda m: m.logMonoTime) def test_only_union_types(self): diff --git a/openpilot/tools/sim/tests/test_sim_bridge.py b/openpilot/tools/sim/tests/test_sim_bridge.py index a2baddad60..a2c0611937 100644 --- a/openpilot/tools/sim/tests/test_sim_bridge.py +++ b/openpilot/tools/sim/tests/test_sim_bridge.py @@ -13,7 +13,6 @@ from openpilot.tools.sim.bridge.common import QueueMessageType SIM_DIR = os.path.join(BASEDIR, "openpilot/tools/sim") class TestSimBridgeBase(OpenpilotTestCase): - SLOW_TEST = True @classmethod def setup_class(cls): if cls is TestSimBridgeBase: @@ -22,6 +21,7 @@ class TestSimBridgeBase(OpenpilotTestCase): def setup_method(self): self.processes = [] + @unittest.skip("TODO: re-enable simulator bridge test") def test_driving(self): # Startup manager and bridge.py. Check processes are running, then engage and verify. p_manager = subprocess.Popen("./launch_openpilot.sh", cwd=SIM_DIR) diff --git a/tools/test_runner.py b/tools/test_runner.py index a31b348be2..eb2f109f97 100755 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -173,8 +173,6 @@ def collect(targets, keyword): continue if getattr(cls, "__unittest_skip_why__", "") == "parameterized base class": continue - if os.environ.get("SKIP_SLOW") and getattr(cls, "SLOW_TEST", False): - continue if not keyword or keyword.lower() in test.id().lower(): tests.append(test) return list({test.id(): test for test in tests}.values()), errors