Audit concrete parameterized vehicle test classes

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "26921352e6a434a71d7b08e8636829f2c73434b726bed00630b600ea3341bd3a", "schema_version": 1}
+{"protected_inventory_sha256": "3af36413ed707df65845fcffa7e0ee71832f0842e3a12de8544d7a8d38ed2273", "schema_version": 1}

tools/ci/run_vehicle_tests.py

@@ -382,6 +382,26 @@ def targets_for(suite):
   raise ValueError(f"Unknown vehicle test suite: {suite}")
 
 
+def interface_class_names(path, class_name):
+  """Resolve concrete parameterized classes without accepting unrelated name prefixes."""
+  concrete = set()
+  path = path.resolve()
+  for module in tuple(sys.modules.values()):
+    source = getattr(module, "__file__", None)
+    if not source or Path(source).resolve() != path:
+      continue
+    template = getattr(module, class_name, None)
+    if not isinstance(template, type) or template.__dict__.get("__test__") is not False:
+      continue
+    for name, candidate in vars(module).items():
+      if (isinstance(candidate, type) and candidate.__bases__ == (template,) and
+          candidate.__module__ == module.__name__ and candidate.__name__ == name and
+          name.startswith(class_name + "_") and len(name) > len(class_name) + 1 and
+          candidate.__dict__.get("__test__") is True):
+        concrete.add(name)
+  return sorted(concrete) or [class_name]
+
+
 def audit_interface_modules(targets, test_ids):
   """Expose empty modules and test declarations unsupported by unittest discovery."""
   modules, errors = [], []
@@ -395,11 +415,13 @@ def audit_interface_modules(targets, test_ids):
       if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"):
         functions.append(node.name)
       elif isinstance(node, ast.ClassDef):
+        class_names = interface_class_names(path, node.name)
         for method in node.body:
           if isinstance(method, (ast.FunctionDef, ast.AsyncFunctionDef)) and method.name.startswith("test_"):
-            expected = f"{module}.{node.name}.{method.name}"
-            if not any(test_id == expected or test_id.startswith((expected + "_", expected + "[")) for test_id in selected):
-              missing_methods.append(f"{node.name}.{method.name}")
+            for class_name in class_names:
+              expected = f"{module}.{class_name}.{method.name}"
+              if not any(test_id == expected or test_id.startswith((expected + "_", expected + "[")) for test_id in selected):
+                missing_methods.append(f"{class_name}.{method.name}")
     if not selected:
       errors.append(f"Interface test module collected zero tests: {target}")
     for function in functions:

tools/ci/tests/test_vehicle_ci.py

@@ -1,4 +1,4 @@
-from contextlib import ExitStack, redirect_stdout
+from contextlib import ExitStack, chdir, redirect_stdout
 import io
 import json
 from pathlib import Path
@@ -254,6 +254,49 @@ class TestVehicleCollection(unittest.TestCase):
     self.assertEqual(modules[0]["collected_count"], 2)
     self.assertEqual(modules[0]["uncollected_test_methods"], [])
 
+  def test_module_audit_checks_every_real_parameterized_class_and_method(self):
+    from tools.ci import vehicle_pytest
+
+    with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)):
+      path = Path(directory) / "test_parameterized_audit_fixture.py"
+      path.write_text("""import unittest
+from opendbc.testing import parameterized_class
+@parameterized_class('variant', [('A',), ('B',)])
+class TestCar(unittest.TestCase):
+  def test_first(self): pass
+  def test_second(self): pass
+""")
+      try:
+        with chdir(directory), redirect_stdout(io.StringIO()):
+          nodeids, collection_errors = vehicle_pytest.collect([path.name])
+        self.assertEqual(collection_errors, [])
+        ids = [vehicle_pytest.test_id(nodeid).split("test_parameterized_audit_fixture.", 1)[1] for nodeid in nodeids]
+        ids = ["test_parameterized_audit_fixture." + identity for identity in ids]
+        self.assertEqual(len(ids), 4)
+        # A similar name is not evidence that a class came from this template.
+        imported = sys.modules[path.stem]
+        imported.TestCar_Unrelated = type("TestCar_Unrelated", (unittest.TestCase,),
+                                         {"__module__": imported.__name__, "__test__": True})
+        _, errors = audit_interface_modules([path.name], ids)
+        self.assertEqual(errors, [])
+        missing = "test_parameterized_audit_fixture.TestCar_B.test_second"
+        modules, errors = audit_interface_modules([path.name], [identity for identity in ids if identity != missing])
+        self.assertEqual(modules[0]["uncollected_test_methods"], ["TestCar_B.test_second"])
+        self.assertTrue(errors)
+        modules, errors = audit_interface_modules([path.name], [identity for identity in ids if ".TestCar_B." not in identity])
+        self.assertEqual(modules[0]["uncollected_test_methods"], ["TestCar_B.test_first", "TestCar_B.test_second"])
+        self.assertTrue(errors)
+      finally:
+        sys.modules.pop(path.stem, None)
+
+  def test_module_audit_rejects_unrelated_class_prefix(self):
+    with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)):
+      path = Path(directory) / "test_fixture.py"
+      path.write_text("class TestCar:\n  def test_method(self): pass\n")
+      modules, errors = audit_interface_modules([path.name], ["test_fixture.TestCar_A.test_method"])
+      self.assertEqual(modules[0]["uncollected_test_methods"], ["TestCar.test_method"])
+      self.assertTrue(errors)
+
   def test_module_audit_rejects_uncollected_pytest_functions_and_methods(self):
     with tempfile.TemporaryDirectory() as directory, patch.object(run_vehicle_tests, "ROOT", Path(directory)):
       path = Path(directory) / "test_fixture.py"