Qualify native variants and current caller fixtures

opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py

@@ -23,7 +23,7 @@ def _build_libsafety(release: bool = False) -> str:
   ]
   # Coverage must exclude the branches inserted by UBSan.
   if os.environ.get("SAFETY_COVERAGE") == "1":
-    ldflags = ['-fprofile-arcs', '-ftest-coverage'] if not release else []
+    ldflags = ['-fprofile-arcs', '-ftest-coverage']
   else:
     ldflags = ['-fsanitize=undefined', '-fno-sanitize-recover=undefined']
   cflags += ldflags

openpilot/selfdrive/car/tests/test_conditional_manual_event.py

@@ -141,6 +141,7 @@ class CardManualEventTests(unittest.TestCase):
               rk=NS(remaining=0.0), v_cruise_helper=VCruiseHelper(self.cp), slc_receipts=[])
     initialize_publication_sources(fake)
     fake.radar_publish = MethodType(Car.radar_publish, fake)
+    fake._feedback_capture = MethodType(Car._feedback_capture, fake)
     fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event)
     Car.state_publish(cast(Car, fake), state(), None)
     records = [messaging.log_from_bytes(payload).slcCruiseEvent for service, payload in publisher.events
@@ -243,6 +244,7 @@ class CardManualEventTests(unittest.TestCase):
               rk=NS(remaining=0.0), v_cruise_helper=VCruiseHelper(self.cp), slc_receipts=[])
     initialize_publication_sources(fake)
     fake.radar_publish = MethodType(Car.radar_publish, fake)
+    fake._feedback_capture = MethodType(Car._feedback_capture, fake)
     fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event)
     Car.state_publish(cast(Car, fake), state(), None)
     published = [messaging.log_from_bytes(payload) for service, payload in publisher.events
@@ -271,6 +273,7 @@ class CardManualEventTests(unittest.TestCase):
               can_log_mono_time=self.now + 2_000_000)
     initialize_publication_sources(fake)
     fake.radar_publish = MethodType(Car.radar_publish, fake)
+    fake._feedback_capture = MethodType(Car._feedback_capture, fake)
     invalid = car.CarState(canValid=False)
     with patch('openpilot.selfdrive.car.card.REPLAY', True):
       fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event)
@@ -300,6 +303,7 @@ class CardManualEventTests(unittest.TestCase):
               v_cruise_helper=VCruiseHelper(self.cp), slc_receipts=[])
     initialize_publication_sources(fake)
     fake.radar_publish = MethodType(Car.radar_publish, fake)
+    fake._feedback_capture = MethodType(Car._feedback_capture, fake)
     fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event)
     Car.state_publish(cast(Car, fake), release, None)
     car_events = [messaging.log_from_bytes(payload).carState for service, payload in publisher.events if service == 'carState']

openpilot/starpilot/galaxy/tests/test_settings.py

@@ -827,7 +827,7 @@ class SettingsGatewayTest(unittest.TestCase):
     self.assertTrue(self.gateway.confirm(intent["intent"], self.session, self.generation))
     self.assertEqual(read_lane_change(self.params).policy.close_gap_seconds, 0.8)
     self.assertEqual(self.page("lane_change")["rows"][gap]["value"], "0.8")
-    self.assertEqual(self.params.get("LaneChangePreferences")["version"], 4)
+    self.assertEqual(self.params.get("LaneChangePreferences")["version"], 5)
 
     saved = self.page("lane_change")
     revoke = self.gateway.preview(saved["view"], close, -1, self.session, self.generation)

openpilot/starpilot/longitudinal/tests/test_cruise_delay.py

@@ -51,6 +51,10 @@ class NativeCruiseDelayTests(unittest.TestCase):
                                                    acceleration_max=.7, cruise_brake_magnitude=.6)
         if case in ('cem', 'experimental'):
           kwargs['conditional_handoff'] = KEY
+        # SubMaster supplies immutable payload readers; reading an absent struct
+        # on a fixture builder otherwise allocates wire storage.
+        for service, payload in sm.items():
+          sm[service] = payload.as_reader()
         before = message_bytes(envelopes)
         real_helper = planner_module.get_cruise_accel
         forwarded = []

openpilot/starpilot/ui/tests/test_feature_snapshot_reads.py

@@ -38,11 +38,11 @@ class FeatureSnapshotReadTests(unittest.TestCase):
 
   def test_duplicate_torque_sources_are_read_once_per_assembly(self):
     first = self.snapshot()
-    self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument"})
+    self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument", "LaneChangePreferences"})
     self.assertEqual(len(self.params.reads), len(set(self.params.reads)))
     self.params.reads.clear()
     self.assertEqual(self.snapshot(), first)
-    self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument"})
+    self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument", "LaneChangePreferences"})
     self.assertEqual(len(self.params.reads), len(set(self.params.reads)))
 
   def test_learning_row_reads_scoped_and_legacy_sources_once(self):

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "0f0b8ce8948aa864e6531f3e57ff69cabd2914856d8321ca65a5f13b119710ec", "schema_version": 1}
+{"protected_inventory_sha256": "b20354e2ad71a14cedd79907db48c8de6b79d4f828a0d627a33cdc72dc355e2a", "schema_version": 1}

tools/ci/safety_pytest_coverage.py

@@ -3,6 +3,7 @@ import argparse
 import ast
 import hashlib
 import json
+import os
 from pathlib import Path
 import sys
 from unittest import mock
@@ -37,10 +38,73 @@ def targets_for_coverage(root=ROOT):
   return sorted(selected - RELEASE_ONLY)
 
 
+
+class InstrumentedLibraries:
+  """Admit only the supplied library and authentic source-matched coverage builds."""
+  def __init__(self, native, module, root=ROOT):
+    self.native = Path(native).resolve()
+    self.module = module
+    self.root = root
+    self.original_build = module._build_libsafety
+    self.original_load = module.load
+    self.registered = {str(self.native): hashlib.sha256(self.native.read_bytes()).hexdigest()}
+    self.builds = []
+    self.source_sha256 = self.source_signature()
+
+  def source_signature(self):
+    files = {}
+    for relative in ('opendbc_repo/opendbc/safety', 'opendbc_repo/opendbc/bluepilot_lateral'):
+      for path in sorted((self.root / relative).rglob('*')):
+        if path.is_file() and path.suffix in ('.h', '.c', '.py'):
+          files[path.relative_to(self.root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest()
+    if not files:
+      raise RuntimeError('Missing native coverage source inputs')
+    return hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest()
+
+  def build(self, release=False):
+    if type(release) is not bool or os.environ.get('SAFETY_COVERAGE') != '1':
+      raise RuntimeError('Variant builds require genuine coverage instrumentation')
+    before = self.source_signature()
+    if before != self.source_sha256:
+      raise RuntimeError('Native coverage sources changed before a variant build')
+    path = Path(self.original_build(release=release)).resolve()
+    if self.source_signature() != before or not path.is_file():
+      raise RuntimeError('Native coverage sources changed during a variant build')
+    digest = hashlib.sha256(path.read_bytes()).hexdigest()
+    if str(path) in self.registered:
+      raise RuntimeError('Native coverage builder must return a fresh artifact')
+    self.registered[str(path)] = digest
+    self.builds.append({'path': str(path), 'sha256': digest, 'source_sha256': before,
+                        'variant': 'release' if release else 'debug', 'instrumented': True})
+    return str(path)
+
+  def load(self, path):
+    if self.source_signature() != self.source_sha256:
+      raise RuntimeError('Native coverage sources changed before loading')
+    path = Path(path).resolve()
+    expected = self.registered.get(str(path))
+    if expected is None or not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected:
+      raise RuntimeError('Native coverage cases must use a registered unchanged instrumented library')
+    return self.original_load(path)
+
+
 def report_complete(report, targets, native_sha256):
   if (not isinstance(report, dict) or type(report.get('schema_version')) is not int or
       report.get('schema_version') != 1 or report.get('full_catalog') is not True):
     return False
+  builds = report.get('native_builds')
+  source = report.get('native_source_sha256')
+  if (not isinstance(builds, list) or not isinstance(source, str) or len(source) != 64 or
+      any(char not in '0123456789abcdef' for char in source)):
+    return False
+  for row in builds:
+    if (not isinstance(row, dict) or row.get('instrumented') is not True or row.get('variant') not in ('debug', 'release') or
+        not isinstance(row.get('path'), str) or not Path(row['path']).is_absolute() or
+        not all(isinstance(row.get(key), str) and len(row[key]) == 64 and all(char in '0123456789abcdef' for char in row[key])
+                for key in ('sha256', 'source_sha256')) or row['source_sha256'] != report.get('native_source_sha256')):
+      return False
+  if len({row['path'] for row in builds}) != len(builds):
+    return False
   ids, records = report.get('nodeids'), report.get('records')
   if (report.get('targets') != targets or not targets or report.get('native_sha256') != native_sha256 or
       report.get('native_bound_before') is not True or report.get('native_bound_after') is not True or
@@ -64,23 +128,22 @@ def run(native, output, targets=None):
     raise ValueError('Coverage subset must name existing native catalog cases')
   digest = hashlib.sha256(native.read_bytes()).hexdigest()
   libsafety_py.load(native)
-  original_load = libsafety_py.load
-
-  def load_pinned(path):
-    if Path(path).resolve() != native:
-      raise RuntimeError('Native coverage cases must retain the same instrumented libsafety')
-    return original_load(native)
+  libraries = InstrumentedLibraries(native, libsafety_py)
 
   report = {'schema_version': 1, 'full_catalog': full_catalog, 'targets': targets, 'native_sha256': digest,
             'native_path': str(native), 'native_bound_before': Path(libsafety_py.libpath).resolve() == native,
             'nodeids': [], 'records': [], 'collection_errors': [], 'execution_errors': []}
-  with mock.patch.object(libsafety_py, 'load', side_effect=load_pinned):
+  with (mock.patch.object(libsafety_py, 'load', side_effect=libraries.load),
+        mock.patch.object(libsafety_py, '_build_libsafety', side_effect=libraries.build)):
     nodeids, errors = vehicle_pytest.collect(targets)
     report.update(nodeids=nodeids, collection_errors=errors)
     if not errors:
       records, errors = vehicle_pytest.run(nodeids)
       report.update(records=records, execution_errors=errors)
-  report['native_bound_after'] = Path(libsafety_py.libpath).resolve() == native and hashlib.sha256(native.read_bytes()).hexdigest() == digest
+  report['native_builds'] = libraries.builds
+  report['native_source_sha256'] = libraries.source_sha256
+  report['native_bound_after'] = (Path(libsafety_py.libpath).resolve() == native and hashlib.sha256(native.read_bytes()).hexdigest() == digest and
+                                    libraries.source_signature() == libraries.source_sha256)
   Path(output).write_text(json.dumps(report, indent=2) + '\n')
   # The root's bounded native subset is a diagnostic; production always requires the complete catalog.
   checked = dict(report, full_catalog=True)

tools/ci/tests/test_safety_qualification.py

@@ -10,7 +10,7 @@ import threading
 import time
 import tempfile
 from unittest import TestCase, skipUnless
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 from tools.ci import run_safety_qualification as qualification
 
@@ -124,7 +124,7 @@ class TestSafetyQualificationRunner(TestCase):
   def test_native_pytest_coverage_requires_complete_accounting_and_same_library(self):
     from tools.ci import safety_pytest_coverage as coverage
     targets = ['native.py']
-    report = {'schema_version': 1, 'full_catalog': True, 'targets': targets, 'native_sha256': 'pinned',
+    report = {'schema_version': 1, 'full_catalog': True, 'targets': targets, 'native_builds': [], 'native_source_sha256': 'b' * 64, 'native_sha256': 'pinned',
               'native_bound_before': True, 'native_bound_after': True, 'collection_errors': [], 'execution_errors': [],
               'nodeids': ['native.py::test_one'], 'records': [{'id': 'native.test_one', 'status': 'passed'}]}
     self.assertTrue(coverage.report_complete(report, targets, 'pinned'))
@@ -138,6 +138,20 @@ class TestSafetyQualificationRunner(TestCase):
     for changed in changes:
       with self.subTest(changed=changed):
         self.assertFalse(coverage.report_complete(dict(report, **changed), targets, 'pinned'))
+    build = {'path': '/actual-built.so', 'sha256': 'a' * 64, 'source_sha256': 'b' * 64, 'variant': 'release', 'instrumented': True}
+    built = dict(report, native_builds=[build], native_source_sha256='b' * 64)
+    self.assertTrue(coverage.report_complete(built, targets, 'pinned'))
+    for changed in ({'instrumented': False}, {'variant': 'unknown'}, {'path': 'relative.so'},
+                    {'sha256': 'unknown'}, {'source_sha256': 'c' * 64}):
+      self.assertFalse(coverage.report_complete(dict(built, native_builds=[dict(build, **changed)]), targets, 'pinned'))
+    self.assertFalse(coverage.report_complete(dict(built, native_builds=[build, build]), targets, 'pinned'))
+    self.assertFalse(coverage.report_complete(dict(built, native_builds={}), targets, 'pinned'))
+    for key in ('native_builds', 'native_source_sha256'):
+      stripped = dict(built)
+      stripped.pop(key)
+      self.assertFalse(coverage.report_complete(stripped, targets, 'pinned'))
+    self.assertFalse(coverage.report_complete(dict(built, native_source_sha256='invalid'), targets, 'pinned'))
+
 
   def test_coverage_runs_pytest_before_unchanged_full_coverage_requirement(self):
     from tools.ci import safety_pytest_coverage as coverage
@@ -161,7 +175,7 @@ class TestSafetyQualificationRunner(TestCase):
             self.assertIn(str(directory / 'libsafety-debug.so'), argv)
             if report_present:
               (directory / 'pytest-coverage-results.json').write_text(json.dumps(
-                {'schema_version': 1, 'full_catalog': True, 'targets': targets,
+                {'schema_version': 1, 'full_catalog': True, 'targets': targets, 'native_builds': [], 'native_source_sha256': 'b' * 64,
                  'native_sha256': qualification.hashlib.sha256(original.read_bytes()).hexdigest(),
                  'native_bound_before': True, 'native_bound_after': True, 'collection_errors': [], 'execution_errors': [],
                  'nodeids': ['native.py::test_one'], 'records': [{'id': 'native.test_one', 'status': 'passed'}]}))
@@ -178,6 +192,83 @@ class TestSafetyQualificationRunner(TestCase):
         assert isinstance(pytest_summary, dict)
         self.assertEqual(pytest_summary['complete'], report_present)
 
+  def test_coverage_admits_only_authentic_unchanged_instrumented_variant_builds(self):
+    from types import SimpleNamespace
+    from tools.ci import safety_pytest_coverage as coverage
+    with tempfile.TemporaryDirectory() as td:
+      root = Path(td)
+      source = root / 'opendbc_repo/opendbc/safety/declarations.h'
+      source.parent.mkdir(parents=True)
+      source.write_text('actual native source fixture')
+      native, variant = root / 'supplied.so', root / 'variant.so'
+      native.write_bytes(b'supplied instrumented fixture')
+      variant.write_bytes(b'authentic variant fixture')
+      module = SimpleNamespace(_build_libsafety=Mock(return_value=str(variant)), load=Mock())
+      libraries = coverage.InstrumentedLibraries(native, module, root)
+      libraries.load(native)
+      with self.assertRaisesRegex(RuntimeError, 'registered unchanged'):
+        libraries.load(variant)
+      with patch.dict(os.environ, {'SAFETY_COVERAGE': '0'}), self.assertRaisesRegex(RuntimeError, 'instrumentation'):
+        libraries.build()
+      module._build_libsafety.assert_not_called()
+      with patch.dict(os.environ, {'SAFETY_COVERAGE': '1'}):
+        self.assertEqual(libraries.build(release=True), str(variant.resolve()))
+      module._build_libsafety.assert_called_once_with(release=True)
+      self.assertEqual(libraries.builds[0]['variant'], 'release')
+      self.assertTrue(libraries.builds[0]['instrumented'])
+      self.assertEqual(len(libraries.builds[0]['source_sha256']), 64)
+      libraries.load(variant)
+      module.load.assert_called_with(variant.resolve())
+      params = root / 'opendbc_repo/opendbc/bluepilot_lateral/params/limits.h'
+      params.parent.mkdir(parents=True)
+      params.write_text('changed native include')
+      with self.assertRaisesRegex(RuntimeError, 'sources changed before loading'):
+        libraries.load(variant)
+      params.unlink()
+      variant.write_bytes(b'modified variant')
+      with self.assertRaisesRegex(RuntimeError, 'registered unchanged'):
+        libraries.load(variant)
+      native.write_bytes(b'modified supplied')
+      with self.assertRaisesRegex(RuntimeError, 'registered unchanged'):
+        libraries.load(native)
+      def changing_build(release=False):
+        source.write_text('changed during actual compiler call')
+        return str(root / 'other.so')
+      module._build_libsafety = changing_build
+      changing = coverage.InstrumentedLibraries(native, module, root)
+      with patch.dict(os.environ, {'SAFETY_COVERAGE': '1'}), self.assertRaisesRegex(RuntimeError, 'sources changed'):
+        changing.build()
+
+  def test_actual_native_builder_instruments_debug_and_release_without_changing_variant(self):
+    import ast
+    from types import SimpleNamespace
+    source = qualification.ROOT / 'opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py'
+    function = next(node for node in ast.parse(source.read_text()).body if isinstance(node, ast.FunctionDef) and node.name == '_build_libsafety')
+    code = compile(ast.Module(body=[function], type_ignores=[]), str(source), 'exec')
+    with tempfile.TemporaryDirectory() as td:
+      directory = Path(td)
+      library_dir = directory / 'one/two/three/libsafety'
+      library_dir.mkdir(parents=True)
+      def allocate(suffix, **kwargs):
+        return tempfile.mkstemp(suffix=suffix, dir=directory)
+      compiler = Mock()
+      namespace: dict[str, object] = {'os': os, 'Path': Path, 'libsafety_dir': str(library_dir),
+                   'tempfile': SimpleNamespace(mkstemp=allocate), 'subprocess': SimpleNamespace(check_call=compiler)}
+      exec(code, namespace)
+      builder = namespace['_build_libsafety']
+      assert callable(builder)
+      for instrumented in (False, True):
+        for release in (False, True):
+          with self.subTest(instrumented=instrumented, release=release), patch.dict(os.environ, {'SAFETY_COVERAGE': str(int(instrumented))}):
+            compiler.reset_mock()
+            builder(release=release)
+            compile_args, link_args = [call.args[0] for call in compiler.call_args_list]
+            self.assertEqual('-DALLOW_DEBUG' in compile_args, not release)
+            for flags in (compile_args, link_args):
+              self.assertEqual('-fprofile-arcs' in flags, instrumented)
+              self.assertEqual('-ftest-coverage' in flags, instrumented)
+              self.assertEqual('-fsanitize=undefined' in flags, not instrumented)
+
   def test_source_identity_includes_gate_and_workflow_inputs(self):
     identity = qualification.source_identity()
     self.assertFalse(any("/obj/" in path or "/gen/" in path for path in identity))