diff --git a/prebuilt.json b/prebuilt.json index ec231714e5..2b27d66ce1 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "26921352e6a434a71d7b08e8636829f2c73434b726bed00630b600ea3341bd3a", "schema_version": 1} +{"protected_inventory_sha256": "3af36413ed707df65845fcffa7e0ee71832f0842e3a12de8544d7a8d38ed2273", "schema_version": 1} diff --git a/tools/ci/run_vehicle_tests.py b/tools/ci/run_vehicle_tests.py index 41f5833e3e..0b806832eb 100644 --- a/tools/ci/run_vehicle_tests.py +++ b/tools/ci/run_vehicle_tests.py @@ -382,6 +382,26 @@ def targets_for(suite): raise ValueError(f"Unknown vehicle test suite: {suite}") +def interface_class_names(path, class_name): + """Resolve concrete parameterized classes without accepting unrelated name prefixes.""" + concrete = set() + path = path.resolve() + for module in tuple(sys.modules.values()): + source = getattr(module, "__file__", None) + if not source or Path(source).resolve() != path: + continue + template = getattr(module, class_name, None) + if not isinstance(template, type) or template.__dict__.get("__test__") is not False: + continue + for name, candidate in vars(module).items(): + if (isinstance(candidate, type) and candidate.__bases__ == (template,) and + candidate.__module__ == module.__name__ and candidate.__name__ == name and + name.startswith(class_name + "_") and len(name) > len(class_name) + 1 and + candidate.__dict__.get("__test__") is True): + concrete.add(name) + return sorted(concrete) or [class_name] + + def audit_interface_modules(targets, test_ids): """Expose empty modules and test declarations unsupported by unittest discovery.""" modules, errors = [], [] @@ -395,11 +415,13 @@ def audit_interface_modules(targets, test_ids): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"): functions.append(node.name) elif isinstance(node, ast.ClassDef): + class_names = interface_class_names(path, node.name) for method in node.body: if isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)) and method.name.startswith("test_"): - expected = f"{module}.{node.name}.{method.name}" - if not any(test_id == expected or test_id.startswith((expected + "_", expected + "[")) for test_id in selected): - missing_methods.append(f"{node.name}.{method.name}") + for class_name in class_names: + expected = f"{module}.{class_name}.{method.name}" + if not any(test_id == expected or test_id.startswith((expected + "_", expected + "[")) for test_id in selected): + missing_methods.append(f"{class_name}.{method.name}") if not selected: errors.append(f"Interface test module collected zero tests: {target}") for function in functions: diff --git a/tools/ci/tests/test_vehicle_ci.py b/tools/ci/tests/test_vehicle_ci.py index 62400e7237..171dbf2851 100644 --- a/tools/ci/tests/test_vehicle_ci.py +++ b/tools/ci/tests/test_vehicle_ci.py @@ -1,4 +1,4 @@ -from contextlib import ExitStack, redirect_stdout +from contextlib import ExitStack, chdir, redirect_stdout import io import json from pathlib import Path @@ -254,6 +254,49 @@ class TestVehicleCollection(unittest.TestCase): self.assertEqual(modules[0]["collected_count"], 2) self.assertEqual(modules[0]["uncollected_test_methods"], []) + def test_module_audit_checks_every_real_parameterized_class_and_method(self): + from tools.ci import vehicle_pytest + + with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)): + path = Path(directory) / "test_parameterized_audit_fixture.py" + path.write_text("""import unittest +from opendbc.testing import parameterized_class +@parameterized_class('variant', [('A',), ('B',)]) +class TestCar(unittest.TestCase): + def test_first(self): pass + def test_second(self): pass +""") + try: + with chdir(directory), redirect_stdout(io.StringIO()): + nodeids, collection_errors = vehicle_pytest.collect([path.name]) + self.assertEqual(collection_errors, []) + ids = [vehicle_pytest.test_id(nodeid).split("test_parameterized_audit_fixture.", 1)[1] for nodeid in nodeids] + ids = ["test_parameterized_audit_fixture." + identity for identity in ids] + self.assertEqual(len(ids), 4) + # A similar name is not evidence that a class came from this template. + imported = sys.modules[path.stem] + imported.TestCar_Unrelated = type("TestCar_Unrelated", (unittest.TestCase,), + {"__module__": imported.__name__, "__test__": True}) + _, errors = audit_interface_modules([path.name], ids) + self.assertEqual(errors, []) + missing = "test_parameterized_audit_fixture.TestCar_B.test_second" + modules, errors = audit_interface_modules([path.name], [identity for identity in ids if identity != missing]) + self.assertEqual(modules[0]["uncollected_test_methods"], ["TestCar_B.test_second"]) + self.assertTrue(errors) + modules, errors = audit_interface_modules([path.name], [identity for identity in ids if ".TestCar_B." not in identity]) + self.assertEqual(modules[0]["uncollected_test_methods"], ["TestCar_B.test_first", "TestCar_B.test_second"]) + self.assertTrue(errors) + finally: + sys.modules.pop(path.stem, None) + + def test_module_audit_rejects_unrelated_class_prefix(self): + with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)): + path = Path(directory) / "test_fixture.py" + path.write_text("class TestCar:\n def test_method(self): pass\n") + modules, errors = audit_interface_modules([path.name], ["test_fixture.TestCar_A.test_method"]) + self.assertEqual(modules[0]["uncollected_test_methods"], ["TestCar.test_method"]) + self.assertTrue(errors) + def test_module_audit_rejects_uncollected_pytest_functions_and_methods(self): with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)): path = Path(directory) / "test_fixture.py"