diff --git a/opendbc_repo/opendbc/car/toyota/eps8_sdsu.py b/opendbc_repo/opendbc/car/toyota/eps8_sdsu.py index 002f2439ed..32dc0e4474 100644 --- a/opendbc_repo/opendbc/car/toyota/eps8_sdsu.py +++ b/opendbc_repo/opendbc/car/toyota/eps8_sdsu.py @@ -39,7 +39,7 @@ def hardware_supported(candidate, fingerprint, car_fw): if ( candidate not in CARS or fingerprint.get(0, {}).get(0x2FF) != 8 - or any(0x2FF in messages for bus, messages in fingerprint.items() if bus != 0) + or any(0x2FF in messages and (bus != 2 or messages[0x2FF] != 8) for bus, messages in fingerprint.items() if bus != 0) or any(address in messages for messages in fingerprint.values() for address in (0x201, 0x2AA)) ): return False diff --git a/opendbc_repo/opendbc/car/toyota/rav4h_filter.py b/opendbc_repo/opendbc/car/toyota/rav4h_filter.py index 344b40fad2..8800715dd6 100644 --- a/opendbc_repo/opendbc/car/toyota/rav4h_filter.py +++ b/opendbc_repo/opendbc/car/toyota/rav4h_filter.py @@ -13,7 +13,7 @@ STOCK_WORD = 0x9249 def hardware_supported(candidate, fingerprint, car_fw): if (candidate not in EPS5_SDSU_CARS or fingerprint.get(0, {}).get(0x2FF) != 8 or - any(0x2FF in fingerprint.get(bus, {}) for bus in (1, 2)) or + any(0x2FF in messages and (bus != 2 or messages[0x2FF] != 8) for bus, messages in fingerprint.items() if bus != 0) or any(address in messages for messages in fingerprint.values() for address in (0x201, 0x2AA)) or any(address in fingerprint.get(2, {}) for address in (0x343, 0x4CB))): return False diff --git a/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu.py b/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu.py index fd59edfff9..068e094f38 100644 --- a/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu.py +++ b/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu.py @@ -35,7 +35,7 @@ def test_eps8_hardware_boundaries(candidate, bad): if bad == 'length': fp[0][0x2FF] = 4 if bad == 'mirror': - fp[2][0x2FF] = 8 + fp[2][0x2FF] = 7 if bad == 'pedal': fp[0][0x201] = 6 if bad == 'mixed': diff --git a/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu_callers.py b/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu_callers.py index 3f30f1b618..f34d46fe8c 100644 --- a/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu_callers.py +++ b/openpilot/starpilot/car/toyota/tests/test_eps8_sdsu_callers.py @@ -8,7 +8,7 @@ source_matched_native = helpers.source_matched_native daemon_gc_scope = helpers.daemon_gc_scope -def run(candidate, stock, bridge, scenario='nominal', aol=True, alpha=True, disable_setting=None, firmware=None, hybrid_release=False): +def run(candidate, stock, bridge, scenario='nominal', aol=True, alpha=True, disable_setting=None, firmware=None, hybrid_release=False, sdsu_mirror=False): angle = candidate == CAR.TOYOTA_RAV4_TSS2_2023 camera = bool(candidate.config.flags & ToyotaFlags.TSS2 and not candidate.config.flags & ToyotaFlags.RADAR_ACC) @@ -39,6 +39,7 @@ def run(candidate, stock, bridge, scenario='nominal', aol=True, alpha=True, disa brake_stop_check_start=355, firmware_override=firmware, assert_hybrid_release=hybrid_release, + sdsu_mirror=sdsu_mirror, ) diff --git a/openpilot/starpilot/car/toyota/tests/test_rav4h_caller_native.py b/openpilot/starpilot/car/toyota/tests/test_rav4h_caller_native.py index 71916a4de7..9d999559dc 100644 --- a/openpilot/starpilot/car/toyota/tests/test_rav4h_caller_native.py +++ b/openpilot/starpilot/car/toyota/tests/test_rav4h_caller_native.py @@ -226,7 +226,7 @@ def control_inputs(controls, published, state, sd_events, native, now): def run_case(stock, scenario='nominal', *, aol=True, candidate=CAR.TOYOTA_RAV4H, bridge=None, input_clock=False, brake_stop_check_start=350, sdsu_words=(0x9049, 0x9249), alpha_long=False, disable_setting=None, physical_frames=None, angle=False, - firmware_override=None, assert_hybrid_release=False): + firmware_override=None, assert_hybrid_release=False, sdsu_mirror=False): SYNTHETIC_LAST.clear() NATIVE_LAST.clear() safety = libsafety_py.libsafety @@ -247,6 +247,8 @@ def run_case(stock, scenario='nominal', *, aol=True, candidate=CAR.TOYOTA_RAV4H, assert exact and found == {candidate} fp = gen_empty_fingerprint() fp[0][0x2FF] = 8 + if sdsu_mirror: + fp[2][0x2FF] = 8 observed = (candidate, fp, '0' * 17, fw, structs.CarParams.FingerprintSource.fw, True) with ( patch.object(car_helpers, 'fingerprint', return_value=observed), @@ -341,12 +343,16 @@ def run_case(stock, scenario='nominal', *, aol=True, candidate=CAR.TOYOTA_RAV4H, elif scenario == 'off' and tick == 250: params.put_bool('DisableOpenpilotLongitudinal', not stock, block=True) frames = (physical if physical_frames is None else physical_frames)(packer, **fields) + if sdsu_mirror: + # Bus2 remains active even while the required bus0 source disappears. + # Its conflicting state7 must also never replace bus0 state0. + frames.append(packer.make_can_msg('SDSU', 2, {'STATE': 0 if scenario in ('faults', 'bad_dlc') else 7})) if scenario == 'bad_dlc' and 630 <= tick < 665: malformed = packer.make_can_msg('SDSU', 0, {'STATE': 0}) frames.append((malformed[0], malformed[1][:7], malformed[2])) for a, d, bus in frames: accepted = bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(a, bus, d))) - assert accepted == (not (a == 0x2FF and 0 < (d[2] >> 4) < 7)), (tick, hex(a), d.hex(), accepted) + assert accepted == (not (a == 0x2FF and bus == 0 and 0 < (d[2] >> 4) < 7)), (tick, hex(a), d.hex(), accepted) safety.safety_tick() native = native_events(host, safety, now, sd.aol_session_id) deliver(host.sm, [*native, *prev.values()]) diff --git a/openpilot/starpilot/car/toyota/tests/test_sdsu_forwarded_copy.py b/openpilot/starpilot/car/toyota/tests/test_sdsu_forwarded_copy.py new file mode 100644 index 0000000000..55518e1231 --- /dev/null +++ b/openpilot/starpilot/car/toyota/tests/test_sdsu_forwarded_copy.py @@ -0,0 +1,42 @@ +import pytest +from opendbc.car import gen_empty_fingerprint +from opendbc.car.toyota import eps8_sdsu, rav4h_filter +from opendbc.car.toyota.interface import CarInterface +from openpilot.starpilot.car.toyota.tests.test_pedal_caller_native import firmware + +CARS = sorted(eps8_sdsu.CARS | rav4h_filter.EPS5_SDSU_CARS) + + +def detector(candidate): + return eps8_sdsu.hardware_supported if candidate in eps8_sdsu.CARS else rav4h_filter.hardware_supported + + +@pytest.mark.parametrize('candidate', CARS) +@pytest.mark.parametrize('alpha', [False, True]) +def test_forwarded_copy_preserves_full_factory_configuration(candidate, alpha): + fp = gen_empty_fingerprint() + fp[0][0x2FF] = 8 + fw = firmware(candidate) + original = CarInterface.get_params(candidate, fp, fw, alpha, False, False) + fp[2][0x2FF] = 8 + mirrored = CarInterface.get_params(candidate, fp, fw, alpha, False, False) + assert detector(candidate)(candidate, fp, fw) + assert rav4h_filter.configured(mirrored) + assert mirrored.to_dict() == original.to_dict() + + +@pytest.mark.parametrize('candidate', CARS) +@pytest.mark.parametrize('bad', ['mirror_only', 'bus1', 'bus3', 'mirror0', 'mirror4', 'mirror7', 'source7']) +def test_copy_cannot_enroll_wrong_source_or_length(candidate, bad): + fp = gen_empty_fingerprint() + fp[0][0x2FF] = 8 + fp[2][0x2FF] = 8 + if bad == 'mirror_only': + del fp[0][0x2FF] + elif bad in ('bus1', 'bus3'): + fp[int(bad[-1])] = {0x2FF: 8} + elif bad.startswith('mirror'): + fp[2][0x2FF] = int(bad[-1]) + else: + fp[0][0x2FF] = 7 + assert not detector(candidate)(candidate, fp, firmware(candidate)) diff --git a/openpilot/starpilot/car/toyota/tests/test_sdsu_forwarded_copy_callers.py b/openpilot/starpilot/car/toyota/tests/test_sdsu_forwarded_copy_callers.py new file mode 100644 index 0000000000..fefe98af34 --- /dev/null +++ b/openpilot/starpilot/car/toyota/tests/test_sdsu_forwarded_copy_callers.py @@ -0,0 +1,49 @@ +import pytest +from opendbc.car.toyota import eps8_sdsu, rav4h_filter +from opendbc.car import gen_empty_fingerprint +from opendbc.car.toyota.interface import CarInterface +from opendbc.car.toyota.values import CAR +from opendbc.safety.tests.libsafety import libsafety_py +from openpilot.starpilot.car.toyota.tests import test_eps8_sdsu_callers as eps8 +from openpilot.starpilot.car.toyota.tests import test_rav4h_caller_native as helpers +from openpilot.starpilot.car.toyota.tests.test_toyota_cpp_transport import host_heartbeat_bridge as host_heartbeat_bridge + +source_matched_native = helpers.source_matched_native +daemon_gc_scope = helpers.daemon_gc_scope +CARS = sorted(eps8_sdsu.CARS | rav4h_filter.EPS5_SDSU_CARS) + + +def run(candidate, stock, bridge, scenario='nominal'): + if candidate in eps8_sdsu.CARS: + eps8.run(candidate, stock, bridge, scenario, sdsu_mirror=True) + else: + helpers.run_case(stock, scenario, candidate=candidate, bridge=bridge, input_clock=True, sdsu_mirror=True) + + +@pytest.mark.parametrize('candidate', CARS) +@pytest.mark.parametrize('stock', [False, True]) +def test_each_forwarded_sdsu_full_caller(candidate, stock, host_heartbeat_bridge): + run(candidate, stock, host_heartbeat_bridge) + + +@pytest.mark.parametrize('candidate', [CAR.TOYOTA_RAV4H, CAR.TOYOTA_CAMRY, CAR.TOYOTA_COROLLA_TSS2, CAR.TOYOTA_RAV4_TSS2_2023]) +@pytest.mark.parametrize('stock', [False, True]) +@pytest.mark.parametrize('scenario', ['faults', 'off']) +def test_copy_cannot_mask_bus0_source_loss_or_saved_off(candidate, stock, scenario, host_heartbeat_bridge): + run(candidate, stock, host_heartbeat_bridge, scenario) + + +@pytest.mark.parametrize('candidate', CARS) +@pytest.mark.parametrize('stock', [False, True]) +def test_native_forwarding_keeps_existing_owner(candidate, stock): + fp = gen_empty_fingerprint() + fp[0][0x2FF] = fp[2][0x2FF] = 8 + cp = CarInterface.get_params(candidate, fp, helpers.firmware(candidate), True, False, False) + if stock: + assert rav4h_filter.prepare_stock(cp) + safety = libsafety_py.libsafety + safety.init_tests() + assert safety.set_safety_hooks(cp.safetyConfigs[0].safetyModel.raw, cp.safetyConfigs[0].safetyParam) == 0 + for bus, target in ((0, 2), (2, 0)): + for address in (0x2FF, 0x343): + assert safety.safety_fwd_hook(bus, address) == target diff --git a/openpilot/starpilot/ui/layout_preview_renderer.py b/openpilot/starpilot/ui/layout_preview_renderer.py index 8cf7e5360f..52aa7ca68e 100644 --- a/openpilot/starpilot/ui/layout_preview_renderer.py +++ b/openpilot/starpilot/ui/layout_preview_renderer.py @@ -340,8 +340,9 @@ class _Canvas: for key in RUNTIME_PREVIEW_WIDGETS[str(self.profile)]: logical_width, logical_height = widget_size(state.customization, self.profile, key) # The restored compact US sign adds an offset row beyond its saved base size. - if (self.profile == Profile.COMPACT and key == 'speed_limit' and - not state.appearance.use_vienna_sign and resolve_unified_speed(state).offset_text): + compact_offset = (self.profile == Profile.COMPACT and key == 'speed_limit' and + not state.appearance.use_vienna_sign and bool(resolve_unified_speed(state).offset_text)) + if compact_offset: logical_height += 44 position = positions[key] widget = PROFILES[str(self.profile)]['widgets'][key] @@ -349,6 +350,8 @@ class _Canvas: crop_y = max(0, math.floor(position['y'] - widget.get('visualInsetTop', 0))) offset_x, offset_y = crop_x - position['x'], crop_y - position['y'] width, height = math.ceil(logical_width - offset_x), math.ceil(logical_height - offset_y) + if compact_offset and 0 <= crop_y < canvas_height: + height = min(height, canvas_height - crop_y) if x + width > 1024: x, y, row_height = 0, y + row_height, 0 sprites[key] = {'x': x, 'y': y, 'width': width, 'height': height, 'offsetX': offset_x, 'offsetY': offset_y} diff --git a/openpilot/starpilot/ui/tests/test_dinish_speed.py b/openpilot/starpilot/ui/tests/test_dinish_speed.py index a2e264d8c0..65e565e6c8 100644 --- a/openpilot/starpilot/ui/tests/test_dinish_speed.py +++ b/openpilot/starpilot/ui/tests/test_dinish_speed.py @@ -238,9 +238,10 @@ def test_compact_preview_sprite_keeps_the_sign_offset_arrow_and_override_cross() rl.init_window(128, 128, 'SLC preview crop fixture') canvas = _Canvas(Profile.COMPACT) try: - for y, scene in product((20, 0), ('slc_pending', 'slc_retained')): + for y, scene in product((20, 0, 64, 108), ('slc_pending', 'slc_retained')): document = default_document() document['layouts']['compact']['speed_limit']['y'] = y + assert validate_document(document)['layouts']['compact']['speed_limit']['y'] == y with patch.dict('openpilot.starpilot.ui.layout_preview_renderer.RUNTIME_PREVIEW_WIDGETS', compact=('speed_limit',)): png = canvas.render_widget_art(sample_state(scene, document), document['layouts']['compact']) with Image.open(io.BytesIO(png)) as sheet: @@ -256,7 +257,9 @@ def test_compact_preview_sprite_keeps_the_sign_offset_arrow_and_override_cross() rl.unload_image(image) assert np.array_equal(direct[..., 3], np.asarray(rebuilt)[..., 3]) assert slot['offsetX'] == slot['offsetY'] == 0 - assert (slot['width'], slot['height']) == (118, 176) + assert (slot['width'], slot['height']) == (118, min(176, Profile.COMPACT.size[1] - y)) + if scene == 'slc_retained': + assert np.any(direct[..., 3]), (y, scene) finally: canvas.close() if owns_context: diff --git a/openpilot/system/loggerd/loggerd b/openpilot/system/loggerd/loggerd index 4bc05fff42..31cd6e6ce2 100755 Binary files a/openpilot/system/loggerd/loggerd and b/openpilot/system/loggerd/loggerd differ diff --git a/openpilot/tools/replay/replay b/openpilot/tools/replay/replay index 63312b0d31..7df0b9c1de 100755 Binary files a/openpilot/tools/replay/replay and b/openpilot/tools/replay/replay differ diff --git a/prebuilt.json b/prebuilt.json index af67c998e4..dc54564c5a 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "25f884a9a6495f49cc4c52130811797f029a4f4ff31dd43626d0dc0c7fdae940", "schema_version": 1} +{"protected_inventory_sha256": "6101d9d0ca764768158bd18dcc22b09b80524729736f32b7021c377b4855877d", "schema_version": 1} diff --git a/tools/ci/run_safety_qualification.py b/tools/ci/run_safety_qualification.py index 0b05c8b3ea..1639b7f01f 100644 --- a/tools/ci/run_safety_qualification.py +++ b/tools/ci/run_safety_qualification.py @@ -238,6 +238,8 @@ def complete_mutation_artifact(path, summary, *, digest_out=None): def check_import_origins(python, env, output, *, full_catalog=False): + sidecar = output.with_suffix(".origins.json") + sidecar.unlink(missing_ok=True) script = """ import json from pathlib import Path @@ -278,14 +280,15 @@ print(json.dumps({'msgq_visionipc': str(Path(msgq.visionipc.visionipc_pyx.__file "cereal_messaging": ROOT / "openpilot/cereal", "msgq_ipc": ROOT / "msgq_repo"} if full_catalog: expected["msgq_visionipc"] = ROOT / "msgq_repo" - return all(isinstance(origins.get(name), str) and Path(origins[name]).is_relative_to(path) - for name, path in expected.items()) + valid = all(isinstance(origins.get(name), str) and Path(origins[name]).is_relative_to(path) + for name, path in expected.items()) + if valid: + sidecar.write_text(json.dumps(origins, sort_keys=True) + "\n") + return valid def imported_module_hashes(output): - origins = {} - for line in output.read_text().splitlines(): - origins.update(json.loads(line)) + origins = json.loads(output.with_suffix(".origins.json").read_text()) return {name: {"path": path, "sha256": hashlib.sha256(Path(path).read_bytes()).hexdigest()} for name, path in origins.items()} diff --git a/tools/ci/tests/test_safety_qualification.py b/tools/ci/tests/test_safety_qualification.py index c4e1821c48..13064f6999 100644 --- a/tools/ci/tests/test_safety_qualification.py +++ b/tools/ci/tests/test_safety_qualification.py @@ -56,8 +56,42 @@ class TestSafetyQualificationRunner(TestCase): Path(path).write_bytes(b"native") log = root / "imports.log" log.write_text("\n".join(json.dumps({name: path}) for name, path in modules.items()) + "\n") + log.with_suffix(".origins.json").write_text(json.dumps(modules)) self.assertEqual(set(qualification.imported_module_hashes(log)), set(modules)) + def test_validated_origins_sidecar_retains_multiline_stdout_and_stderr(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + prefixes = {"opendbc": "opendbc_repo", "panda": "panda", "msgq": "msgq_repo", + "openpilot": "openpilot", "libsafety_py": "opendbc_repo", + "can_parser": "opendbc_repo", "can_packer": "opendbc_repo", "can_dbc": "opendbc_repo", + "cereal_log": "openpilot/cereal", "cereal_messaging": "openpilot/cereal", "msgq_ipc": "msgq_repo", + "msgq_visionipc": "msgq_repo"} + origins = {name: str(root / prefix / name) for name, prefix in prefixes.items()} + for path in origins.values(): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_bytes(b"actual provenance fixture") + first = {name: path for name, path in origins.items() if name != "msgq_visionipc"} + stdout = json.dumps(first) + "\n" + json.dumps({"msgq_visionipc": origins["msgq_visionipc"]}) + "\n" + warning = "kj/filesystem-disk-unix.c++:1734: warning: PWD environment variable doesn't match current directory\n" + log = root / "imports.log" + sidecar = log.with_suffix(".origins.json") + with patch.object(qualification, "ROOT", root), patch.object(qualification.subprocess, "run", + return_value=subprocess.CompletedProcess([], 0, stdout, warning)): + self.assertTrue(qualification.check_import_origins(sys.executable, {}, log, full_catalog=True)) + self.assertEqual(log.read_text(), stdout + warning) + self.assertEqual(json.loads(sidecar.read_text()), origins) + self.assertEqual(set(qualification.imported_module_hashes(log)), set(origins)) + foreign = dict(origins, msgq_visionipc="/foreign/visionipc.so") + for code, invalid in ((0, json.dumps(foreign)), (0, "bad stdout"), (1, stdout)): + sidecar.write_text(json.dumps(origins)) # Never reuse earlier success after a failed probe. + with self.subTest(code=code, stdout=invalid), patch.object(qualification, "ROOT", root), \ + patch.object(qualification.subprocess, "run", return_value=subprocess.CompletedProcess([], code, invalid, warning)): + self.assertFalse(qualification.check_import_origins(sys.executable, {}, log, full_catalog=True)) + self.assertFalse(sidecar.exists()) + with self.assertRaises(FileNotFoundError): + qualification.imported_module_hashes(log) + def test_source_identity_includes_gate_and_workflow_inputs(self): identity = qualification.source_identity() self.assertFalse(any("/obj/" in path or "/gen/" in path for path in identity)) @@ -456,7 +490,25 @@ class TestSafetyQualificationRunner(TestCase): with tempfile.TemporaryDirectory() as td: output = Path(td) gate = qualification.Gate("coverage", output, "/usr/bin/python3") - code, text = gate.run("timeout", [sys.executable, "-c", "import time; print('partial', flush=True); time.sleep(10)"], timeout=0.05) + ready = output / "child-ready" + script = "import pathlib,sys,time; print('partial', flush=True); pathlib.Path(sys.argv[1]).touch(); time.sleep(10)" + spawn = subprocess.Popen + + def started_child(*args, **kwargs): + proc = spawn(*args, **kwargs) + deadline = time.monotonic() + 5 + while not ready.exists(): + if proc.poll() is not None or time.monotonic() >= deadline: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGKILL) + proc.communicate() + self.fail("Timeout fixture never became ready") + time.sleep(.005) + return proc + + # Exercise the real timeout after the child's partial output is actually flushed. + with patch.object(qualification.subprocess, "Popen", side_effect=started_child): + code, text = gate.run("timeout", [sys.executable, "-c", script, str(ready)], timeout=0.05) self.assertEqual(code, 124) self.assertIn("Timed out", text) self.assertIn("partial", (output / "timeout.log").read_text()) diff --git a/tools/laptop_device_build/tests/test_runtime_dependencies.py b/tools/laptop_device_build/tests/test_runtime_dependencies.py index a0d22dcc01..de33805441 100644 --- a/tools/laptop_device_build/tests/test_runtime_dependencies.py +++ b/tools/laptop_device_build/tests/test_runtime_dependencies.py @@ -1,6 +1,7 @@ """Catch device binaries linked to build-environment Cap'n Proto/KJ libraries.""" from pathlib import Path +import ast import subprocess import sys import unittest @@ -22,3 +23,26 @@ class DeviceRuntimeDependenciesTest(unittest.TestCase): dynamic = subprocess.check_output(['readelf', '--dynamic', '--wide', str(ROOT / binary)], text=True) self.assertNotRegex(dynamic, r'\(NEEDED\).*\[.*lib(?:capnp|kj)', f"{binary}: link Cap'n Proto/KJ statically for AGNOS") + + +class LiveSteeringServiceInventoryTest(unittest.TestCase): + def test_live_steering_tracking_is_registered_in_shipped_native_consumers(self): + service = 'starpilotSteeringTracking' + tree = ast.parse((ROOT / 'openpilot/cereal/services.py').read_text()) + declarations = [node.value for node in tree.body if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) and node.target.id == '_services'] + self.assertEqual(len(declarations), 1) + table = declarations[0] + if not isinstance(table, ast.Dict): + self.fail('Service registry must be a dictionary') + declared = [value for key, value in zip(table.keys, table.values, strict=True) + if isinstance(key, ast.Constant) and key.value == service] + self.assertEqual(len(declared), 1) + definition = declared[0] + if not isinstance(definition, ast.Tuple): + self.fail('Service declaration must be a tuple') + self.assertIs(ast.literal_eval(definition.elts[0]), True) + for binary in ('openpilot/system/loggerd/loggerd', 'openpilot/tools/replay/replay'): + with self.subTest(binary=binary): + self.assertIn(service.encode(), (ROOT / binary).read_bytes(), + f'{binary}: rebuild after changing the native service registry')