diff --git a/opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py b/opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py index 79f94afc0c..7c2c15f748 100644 --- a/opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py +++ b/opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py @@ -23,7 +23,7 @@ def _build_libsafety(release: bool = False) -> str: ] # Coverage must exclude the branches inserted by UBSan. if os.environ.get("SAFETY_COVERAGE") == "1": - ldflags = ['-fprofile-arcs', '-ftest-coverage'] if not release else [] + ldflags = ['-fprofile-arcs', '-ftest-coverage'] else: ldflags = ['-fsanitize=undefined', '-fno-sanitize-recover=undefined'] cflags += ldflags diff --git a/openpilot/selfdrive/car/tests/test_conditional_manual_event.py b/openpilot/selfdrive/car/tests/test_conditional_manual_event.py index 00fcfd0c2e..2b10f5fb80 100644 --- a/openpilot/selfdrive/car/tests/test_conditional_manual_event.py +++ b/openpilot/selfdrive/car/tests/test_conditional_manual_event.py @@ -141,6 +141,7 @@ class CardManualEventTests(unittest.TestCase): rk=NS(remaining=0.0), v_cruise_helper=VCruiseHelper(self.cp), slc_receipts=[]) initialize_publication_sources(fake) fake.radar_publish = MethodType(Car.radar_publish, fake) + fake._feedback_capture = MethodType(Car._feedback_capture, fake) fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event) Car.state_publish(cast(Car, fake), state(), None) records = [messaging.log_from_bytes(payload).slcCruiseEvent for service, payload in publisher.events @@ -243,6 +244,7 @@ class CardManualEventTests(unittest.TestCase): rk=NS(remaining=0.0), v_cruise_helper=VCruiseHelper(self.cp), slc_receipts=[]) initialize_publication_sources(fake) fake.radar_publish = MethodType(Car.radar_publish, fake) + fake._feedback_capture = MethodType(Car._feedback_capture, fake) fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event) Car.state_publish(cast(Car, fake), state(), None) published = [messaging.log_from_bytes(payload) for service, payload in publisher.events @@ -271,6 +273,7 @@ class CardManualEventTests(unittest.TestCase): can_log_mono_time=self.now + 2_000_000) initialize_publication_sources(fake) fake.radar_publish = MethodType(Car.radar_publish, fake) + fake._feedback_capture = MethodType(Car._feedback_capture, fake) invalid = car.CarState(canValid=False) with patch('openpilot.selfdrive.car.card.REPLAY', True): fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event) @@ -300,6 +303,7 @@ class CardManualEventTests(unittest.TestCase): v_cruise_helper=VCruiseHelper(self.cp), slc_receipts=[]) initialize_publication_sources(fake) fake.radar_publish = MethodType(Car.radar_publish, fake) + fake._feedback_capture = MethodType(Car._feedback_capture, fake) fake.publish_traffic_receipt = lambda event: Car.publish_traffic_receipt(cast(Car, fake), event) Car.state_publish(cast(Car, fake), release, None) car_events = [messaging.log_from_bytes(payload).carState for service, payload in publisher.events if service == 'carState'] diff --git a/openpilot/starpilot/galaxy/tests/test_settings.py b/openpilot/starpilot/galaxy/tests/test_settings.py index c967bb82c3..1ea8d3db59 100644 --- a/openpilot/starpilot/galaxy/tests/test_settings.py +++ b/openpilot/starpilot/galaxy/tests/test_settings.py @@ -827,7 +827,7 @@ class SettingsGatewayTest(unittest.TestCase): self.assertTrue(self.gateway.confirm(intent["intent"], self.session, self.generation)) self.assertEqual(read_lane_change(self.params).policy.close_gap_seconds, 0.8) self.assertEqual(self.page("lane_change")["rows"][gap]["value"], "0.8") - self.assertEqual(self.params.get("LaneChangePreferences")["version"], 4) + self.assertEqual(self.params.get("LaneChangePreferences")["version"], 5) saved = self.page("lane_change") revoke = self.gateway.preview(saved["view"], close, -1, self.session, self.generation) diff --git a/openpilot/starpilot/longitudinal/tests/test_cruise_delay.py b/openpilot/starpilot/longitudinal/tests/test_cruise_delay.py index 6589b259e9..4f5ab49d75 100644 --- a/openpilot/starpilot/longitudinal/tests/test_cruise_delay.py +++ b/openpilot/starpilot/longitudinal/tests/test_cruise_delay.py @@ -51,6 +51,10 @@ class NativeCruiseDelayTests(unittest.TestCase): acceleration_max=.7, cruise_brake_magnitude=.6) if case in ('cem', 'experimental'): kwargs['conditional_handoff'] = KEY + # SubMaster supplies immutable payload readers; reading an absent struct + # on a fixture builder otherwise allocates wire storage. + for service, payload in sm.items(): + sm[service] = payload.as_reader() before = message_bytes(envelopes) real_helper = planner_module.get_cruise_accel forwarded = [] diff --git a/openpilot/starpilot/ui/tests/test_feature_snapshot_reads.py b/openpilot/starpilot/ui/tests/test_feature_snapshot_reads.py index 7f50df228a..a1e4f545d0 100644 --- a/openpilot/starpilot/ui/tests/test_feature_snapshot_reads.py +++ b/openpilot/starpilot/ui/tests/test_feature_snapshot_reads.py @@ -38,11 +38,11 @@ class FeatureSnapshotReadTests(unittest.TestCase): def test_duplicate_torque_sources_are_read_once_per_assembly(self): first = self.snapshot() - self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument"}) + self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument", "LaneChangePreferences"}) self.assertEqual(len(self.params.reads), len(set(self.params.reads))) self.params.reads.clear() self.assertEqual(self.snapshot(), first) - self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument"}) + self.assertEqual(set(self.params.reads), {"TorqueOverrideDocument", "LaneChangePreferences"}) self.assertEqual(len(self.params.reads), len(set(self.params.reads))) def test_learning_row_reads_scoped_and_legacy_sources_once(self): diff --git a/prebuilt.json b/prebuilt.json index 693dd4fe4a..477ff2280f 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "0f0b8ce8948aa864e6531f3e57ff69cabd2914856d8321ca65a5f13b119710ec", "schema_version": 1} +{"protected_inventory_sha256": "b20354e2ad71a14cedd79907db48c8de6b79d4f828a0d627a33cdc72dc355e2a", "schema_version": 1} diff --git a/tools/ci/safety_pytest_coverage.py b/tools/ci/safety_pytest_coverage.py index f50d578852..60cf6914d7 100644 --- a/tools/ci/safety_pytest_coverage.py +++ b/tools/ci/safety_pytest_coverage.py @@ -3,6 +3,7 @@ import argparse import ast import hashlib import json +import os from pathlib import Path import sys from unittest import mock @@ -37,10 +38,73 @@ def targets_for_coverage(root=ROOT): return sorted(selected - RELEASE_ONLY) + +class InstrumentedLibraries: + """Admit only the supplied library and authentic source-matched coverage builds.""" + def __init__(self, native, module, root=ROOT): + self.native = Path(native).resolve() + self.module = module + self.root = root + self.original_build = module._build_libsafety + self.original_load = module.load + self.registered = {str(self.native): hashlib.sha256(self.native.read_bytes()).hexdigest()} + self.builds = [] + self.source_sha256 = self.source_signature() + + def source_signature(self): + files = {} + for relative in ('opendbc_repo/opendbc/safety', 'opendbc_repo/opendbc/bluepilot_lateral'): + for path in sorted((self.root / relative).rglob('*')): + if path.is_file() and path.suffix in ('.h', '.c', '.py'): + files[path.relative_to(self.root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + if not files: + raise RuntimeError('Missing native coverage source inputs') + return hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() + + def build(self, release=False): + if type(release) is not bool or os.environ.get('SAFETY_COVERAGE') != '1': + raise RuntimeError('Variant builds require genuine coverage instrumentation') + before = self.source_signature() + if before != self.source_sha256: + raise RuntimeError('Native coverage sources changed before a variant build') + path = Path(self.original_build(release=release)).resolve() + if self.source_signature() != before or not path.is_file(): + raise RuntimeError('Native coverage sources changed during a variant build') + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if str(path) in self.registered: + raise RuntimeError('Native coverage builder must return a fresh artifact') + self.registered[str(path)] = digest + self.builds.append({'path': str(path), 'sha256': digest, 'source_sha256': before, + 'variant': 'release' if release else 'debug', 'instrumented': True}) + return str(path) + + def load(self, path): + if self.source_signature() != self.source_sha256: + raise RuntimeError('Native coverage sources changed before loading') + path = Path(path).resolve() + expected = self.registered.get(str(path)) + if expected is None or not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + raise RuntimeError('Native coverage cases must use a registered unchanged instrumented library') + return self.original_load(path) + + def report_complete(report, targets, native_sha256): if (not isinstance(report, dict) or type(report.get('schema_version')) is not int or report.get('schema_version') != 1 or report.get('full_catalog') is not True): return False + builds = report.get('native_builds') + source = report.get('native_source_sha256') + if (not isinstance(builds, list) or not isinstance(source, str) or len(source) != 64 or + any(char not in '0123456789abcdef' for char in source)): + return False + for row in builds: + if (not isinstance(row, dict) or row.get('instrumented') is not True or row.get('variant') not in ('debug', 'release') or + not isinstance(row.get('path'), str) or not Path(row['path']).is_absolute() or + not all(isinstance(row.get(key), str) and len(row[key]) == 64 and all(char in '0123456789abcdef' for char in row[key]) + for key in ('sha256', 'source_sha256')) or row['source_sha256'] != report.get('native_source_sha256')): + return False + if len({row['path'] for row in builds}) != len(builds): + return False ids, records = report.get('nodeids'), report.get('records') if (report.get('targets') != targets or not targets or report.get('native_sha256') != native_sha256 or report.get('native_bound_before') is not True or report.get('native_bound_after') is not True or @@ -64,23 +128,22 @@ def run(native, output, targets=None): raise ValueError('Coverage subset must name existing native catalog cases') digest = hashlib.sha256(native.read_bytes()).hexdigest() libsafety_py.load(native) - original_load = libsafety_py.load - - def load_pinned(path): - if Path(path).resolve() != native: - raise RuntimeError('Native coverage cases must retain the same instrumented libsafety') - return original_load(native) + libraries = InstrumentedLibraries(native, libsafety_py) report = {'schema_version': 1, 'full_catalog': full_catalog, 'targets': targets, 'native_sha256': digest, 'native_path': str(native), 'native_bound_before': Path(libsafety_py.libpath).resolve() == native, 'nodeids': [], 'records': [], 'collection_errors': [], 'execution_errors': []} - with mock.patch.object(libsafety_py, 'load', side_effect=load_pinned): + with (mock.patch.object(libsafety_py, 'load', side_effect=libraries.load), + mock.patch.object(libsafety_py, '_build_libsafety', side_effect=libraries.build)): nodeids, errors = vehicle_pytest.collect(targets) report.update(nodeids=nodeids, collection_errors=errors) if not errors: records, errors = vehicle_pytest.run(nodeids) report.update(records=records, execution_errors=errors) - report['native_bound_after'] = Path(libsafety_py.libpath).resolve() == native and hashlib.sha256(native.read_bytes()).hexdigest() == digest + report['native_builds'] = libraries.builds + report['native_source_sha256'] = libraries.source_sha256 + report['native_bound_after'] = (Path(libsafety_py.libpath).resolve() == native and hashlib.sha256(native.read_bytes()).hexdigest() == digest and + libraries.source_signature() == libraries.source_sha256) Path(output).write_text(json.dumps(report, indent=2) + '\n') # The root's bounded native subset is a diagnostic; production always requires the complete catalog. checked = dict(report, full_catalog=True) diff --git a/tools/ci/tests/test_safety_qualification.py b/tools/ci/tests/test_safety_qualification.py index 05ad81cba3..8bd22caaf1 100644 --- a/tools/ci/tests/test_safety_qualification.py +++ b/tools/ci/tests/test_safety_qualification.py @@ -10,7 +10,7 @@ import threading import time import tempfile from unittest import TestCase, skipUnless -from unittest.mock import patch +from unittest.mock import Mock, patch from tools.ci import run_safety_qualification as qualification @@ -124,7 +124,7 @@ class TestSafetyQualificationRunner(TestCase): def test_native_pytest_coverage_requires_complete_accounting_and_same_library(self): from tools.ci import safety_pytest_coverage as coverage targets = ['native.py'] - report = {'schema_version': 1, 'full_catalog': True, 'targets': targets, 'native_sha256': 'pinned', + report = {'schema_version': 1, 'full_catalog': True, 'targets': targets, 'native_builds': [], 'native_source_sha256': 'b' * 64, 'native_sha256': 'pinned', 'native_bound_before': True, 'native_bound_after': True, 'collection_errors': [], 'execution_errors': [], 'nodeids': ['native.py::test_one'], 'records': [{'id': 'native.test_one', 'status': 'passed'}]} self.assertTrue(coverage.report_complete(report, targets, 'pinned')) @@ -138,6 +138,20 @@ class TestSafetyQualificationRunner(TestCase): for changed in changes: with self.subTest(changed=changed): self.assertFalse(coverage.report_complete(dict(report, **changed), targets, 'pinned')) + build = {'path': '/actual-built.so', 'sha256': 'a' * 64, 'source_sha256': 'b' * 64, 'variant': 'release', 'instrumented': True} + built = dict(report, native_builds=[build], native_source_sha256='b' * 64) + self.assertTrue(coverage.report_complete(built, targets, 'pinned')) + for changed in ({'instrumented': False}, {'variant': 'unknown'}, {'path': 'relative.so'}, + {'sha256': 'unknown'}, {'source_sha256': 'c' * 64}): + self.assertFalse(coverage.report_complete(dict(built, native_builds=[dict(build, **changed)]), targets, 'pinned')) + self.assertFalse(coverage.report_complete(dict(built, native_builds=[build, build]), targets, 'pinned')) + self.assertFalse(coverage.report_complete(dict(built, native_builds={}), targets, 'pinned')) + for key in ('native_builds', 'native_source_sha256'): + stripped = dict(built) + stripped.pop(key) + self.assertFalse(coverage.report_complete(stripped, targets, 'pinned')) + self.assertFalse(coverage.report_complete(dict(built, native_source_sha256='invalid'), targets, 'pinned')) + def test_coverage_runs_pytest_before_unchanged_full_coverage_requirement(self): from tools.ci import safety_pytest_coverage as coverage @@ -161,7 +175,7 @@ class TestSafetyQualificationRunner(TestCase): self.assertIn(str(directory / 'libsafety-debug.so'), argv) if report_present: (directory / 'pytest-coverage-results.json').write_text(json.dumps( - {'schema_version': 1, 'full_catalog': True, 'targets': targets, + {'schema_version': 1, 'full_catalog': True, 'targets': targets, 'native_builds': [], 'native_source_sha256': 'b' * 64, 'native_sha256': qualification.hashlib.sha256(original.read_bytes()).hexdigest(), 'native_bound_before': True, 'native_bound_after': True, 'collection_errors': [], 'execution_errors': [], 'nodeids': ['native.py::test_one'], 'records': [{'id': 'native.test_one', 'status': 'passed'}]})) @@ -178,6 +192,83 @@ class TestSafetyQualificationRunner(TestCase): assert isinstance(pytest_summary, dict) self.assertEqual(pytest_summary['complete'], report_present) + def test_coverage_admits_only_authentic_unchanged_instrumented_variant_builds(self): + from types import SimpleNamespace + from tools.ci import safety_pytest_coverage as coverage + with tempfile.TemporaryDirectory() as td: + root = Path(td) + source = root / 'opendbc_repo/opendbc/safety/declarations.h' + source.parent.mkdir(parents=True) + source.write_text('actual native source fixture') + native, variant = root / 'supplied.so', root / 'variant.so' + native.write_bytes(b'supplied instrumented fixture') + variant.write_bytes(b'authentic variant fixture') + module = SimpleNamespace(_build_libsafety=Mock(return_value=str(variant)), load=Mock()) + libraries = coverage.InstrumentedLibraries(native, module, root) + libraries.load(native) + with self.assertRaisesRegex(RuntimeError, 'registered unchanged'): + libraries.load(variant) + with patch.dict(os.environ, {'SAFETY_COVERAGE': '0'}), self.assertRaisesRegex(RuntimeError, 'instrumentation'): + libraries.build() + module._build_libsafety.assert_not_called() + with patch.dict(os.environ, {'SAFETY_COVERAGE': '1'}): + self.assertEqual(libraries.build(release=True), str(variant.resolve())) + module._build_libsafety.assert_called_once_with(release=True) + self.assertEqual(libraries.builds[0]['variant'], 'release') + self.assertTrue(libraries.builds[0]['instrumented']) + self.assertEqual(len(libraries.builds[0]['source_sha256']), 64) + libraries.load(variant) + module.load.assert_called_with(variant.resolve()) + params = root / 'opendbc_repo/opendbc/bluepilot_lateral/params/limits.h' + params.parent.mkdir(parents=True) + params.write_text('changed native include') + with self.assertRaisesRegex(RuntimeError, 'sources changed before loading'): + libraries.load(variant) + params.unlink() + variant.write_bytes(b'modified variant') + with self.assertRaisesRegex(RuntimeError, 'registered unchanged'): + libraries.load(variant) + native.write_bytes(b'modified supplied') + with self.assertRaisesRegex(RuntimeError, 'registered unchanged'): + libraries.load(native) + def changing_build(release=False): + source.write_text('changed during actual compiler call') + return str(root / 'other.so') + module._build_libsafety = changing_build + changing = coverage.InstrumentedLibraries(native, module, root) + with patch.dict(os.environ, {'SAFETY_COVERAGE': '1'}), self.assertRaisesRegex(RuntimeError, 'sources changed'): + changing.build() + + def test_actual_native_builder_instruments_debug_and_release_without_changing_variant(self): + import ast + from types import SimpleNamespace + source = qualification.ROOT / 'opendbc_repo/opendbc/safety/tests/libsafety/libsafety_py.py' + function = next(node for node in ast.parse(source.read_text()).body if isinstance(node, ast.FunctionDef) and node.name == '_build_libsafety') + code = compile(ast.Module(body=[function], type_ignores=[]), str(source), 'exec') + with tempfile.TemporaryDirectory() as td: + directory = Path(td) + library_dir = directory / 'one/two/three/libsafety' + library_dir.mkdir(parents=True) + def allocate(suffix, **kwargs): + return tempfile.mkstemp(suffix=suffix, dir=directory) + compiler = Mock() + namespace: dict[str, object] = {'os': os, 'Path': Path, 'libsafety_dir': str(library_dir), + 'tempfile': SimpleNamespace(mkstemp=allocate), 'subprocess': SimpleNamespace(check_call=compiler)} + exec(code, namespace) + builder = namespace['_build_libsafety'] + assert callable(builder) + for instrumented in (False, True): + for release in (False, True): + with self.subTest(instrumented=instrumented, release=release), patch.dict(os.environ, {'SAFETY_COVERAGE': str(int(instrumented))}): + compiler.reset_mock() + builder(release=release) + compile_args, link_args = [call.args[0] for call in compiler.call_args_list] + self.assertEqual('-DALLOW_DEBUG' in compile_args, not release) + for flags in (compile_args, link_args): + self.assertEqual('-fprofile-arcs' in flags, instrumented) + self.assertEqual('-ftest-coverage' in flags, instrumented) + self.assertEqual('-fsanitize=undefined' in flags, not instrumented) + def test_source_identity_includes_gate_and_workflow_inputs(self): identity = qualification.source_identity() self.assertFalse(any("/obj/" in path or "/gen/" in path for path in identity))