Merge qualified shared source

2026-10-28 00:53:45 +08:00 · 2026-10-22 19:59:30 -05:00
parent d747e5ca08 2c352b90b4
commit 08c0efb01d
15 changed files with 200 additions and 17 deletions
@@ -39,7 +39,7 @@ def hardware_supported(candidate, fingerprint, car_fw):
  if (
    candidate not in CARS
    or fingerprint.get(0, {}).get(0x2FF) != 8
-    or any(0x2FF in messages for bus, messages in fingerprint.items() if bus != 0)
+    or any(0x2FF in messages and (bus != 2 or messages[0x2FF] != 8) for bus, messages in fingerprint.items() if bus != 0)
    or any(address in messages for messages in fingerprint.values() for address in (0x201, 0x2AA))
  ):
    return False
@@ -13,7 +13,7 @@ STOCK_WORD = 0x9249

 def hardware_supported(candidate, fingerprint, car_fw):
  if (candidate not in EPS5_SDSU_CARS or fingerprint.get(0, {}).get(0x2FF) != 8 or
-      any(0x2FF in fingerprint.get(bus, {}) for bus in (1, 2)) or
+      any(0x2FF in messages and (bus != 2 or messages[0x2FF] != 8) for bus, messages in fingerprint.items() if bus != 0) or
      any(address in messages for messages in fingerprint.values() for address in (0x201, 0x2AA)) or
      any(address in fingerprint.get(2, {}) for address in (0x343, 0x4CB))):
    return False
@@ -35,7 +35,7 @@ def test_eps8_hardware_boundaries(candidate, bad):
  if bad == 'length':
    fp[0][0x2FF] = 4
  if bad == 'mirror':
-    fp[2][0x2FF] = 8
+    fp[2][0x2FF] = 7
  if bad == 'pedal':
    fp[0][0x201] = 6
  if bad == 'mixed':
@@ -8,7 +8,7 @@ source_matched_native = helpers.source_matched_native
 daemon_gc_scope = helpers.daemon_gc_scope


-def run(candidate, stock, bridge, scenario='nominal', aol=True, alpha=True, disable_setting=None, firmware=None, hybrid_release=False):
+def run(candidate, stock, bridge, scenario='nominal', aol=True, alpha=True, disable_setting=None, firmware=None, hybrid_release=False, sdsu_mirror=False):
  angle = candidate == CAR.TOYOTA_RAV4_TSS2_2023
  camera = bool(candidate.config.flags & ToyotaFlags.TSS2 and not candidate.config.flags & ToyotaFlags.RADAR_ACC)

@@ -39,6 +39,7 @@ def run(candidate, stock, bridge, scenario='nominal', aol=True, alpha=True, disa
    brake_stop_check_start=355,
    firmware_override=firmware,
    assert_hybrid_release=hybrid_release,
+    sdsu_mirror=sdsu_mirror,
  )


@@ -226,7 +226,7 @@ def control_inputs(controls, published, state, sd_events, native, now):

 def run_case(stock, scenario='nominal', *, aol=True, candidate=CAR.TOYOTA_RAV4H, bridge=None, input_clock=False, brake_stop_check_start=350,
             sdsu_words=(0x9049, 0x9249), alpha_long=False, disable_setting=None, physical_frames=None, angle=False,
-             firmware_override=None, assert_hybrid_release=False):
+             firmware_override=None, assert_hybrid_release=False, sdsu_mirror=False):
  SYNTHETIC_LAST.clear()
  NATIVE_LAST.clear()
  safety = libsafety_py.libsafety
@@ -247,6 +247,8 @@ def run_case(stock, scenario='nominal', *, aol=True, candidate=CAR.TOYOTA_RAV4H,
    assert exact and found == {candidate}
    fp = gen_empty_fingerprint()
    fp[0][0x2FF] = 8
+    if sdsu_mirror:
+      fp[2][0x2FF] = 8
    observed = (candidate, fp, '0' * 17, fw, structs.CarParams.FingerprintSource.fw, True)
    with (
      patch.object(car_helpers, 'fingerprint', return_value=observed),
@@ -341,12 +343,16 @@ def run_case(stock, scenario='nominal', *, aol=True, candidate=CAR.TOYOTA_RAV4H,
        elif scenario == 'off' and tick == 250:
          params.put_bool('DisableOpenpilotLongitudinal', not stock, block=True)
        frames = (physical if physical_frames is None else physical_frames)(packer, **fields)
+        if sdsu_mirror:
+          # Bus2 remains active even while the required bus0 source disappears.
+          # Its conflicting state7 must also never replace bus0 state0.
+          frames.append(packer.make_can_msg('SDSU', 2, {'STATE': 0 if scenario in ('faults', 'bad_dlc') else 7}))
        if scenario == 'bad_dlc' and 630 <= tick < 665:
          malformed = packer.make_can_msg('SDSU', 0, {'STATE': 0})
          frames.append((malformed[0], malformed[1][:7], malformed[2]))
        for a, d, bus in frames:
          accepted = bool(safety.safety_rx_hook(libsafety_py.make_CANPacket(a, bus, d)))
-          assert accepted == (not (a == 0x2FF and 0 < (d[2] >> 4) < 7)), (tick, hex(a), d.hex(), accepted)
+          assert accepted == (not (a == 0x2FF and bus == 0 and 0 < (d[2] >> 4) < 7)), (tick, hex(a), d.hex(), accepted)
        safety.safety_tick()
        native = native_events(host, safety, now, sd.aol_session_id)
        deliver(host.sm, [*native, *prev.values()])
@@ -0,0 +1,42 @@
+import pytest
+from opendbc.car import gen_empty_fingerprint
+from opendbc.car.toyota import eps8_sdsu, rav4h_filter
+from opendbc.car.toyota.interface import CarInterface
+from openpilot.starpilot.car.toyota.tests.test_pedal_caller_native import firmware
+
+CARS = sorted(eps8_sdsu.CARS | rav4h_filter.EPS5_SDSU_CARS)
+
+
+def detector(candidate):
+  return eps8_sdsu.hardware_supported if candidate in eps8_sdsu.CARS else rav4h_filter.hardware_supported
+
+
+@pytest.mark.parametrize('candidate', CARS)
+@pytest.mark.parametrize('alpha', [False, True])
+def test_forwarded_copy_preserves_full_factory_configuration(candidate, alpha):
+  fp = gen_empty_fingerprint()
+  fp[0][0x2FF] = 8
+  fw = firmware(candidate)
+  original = CarInterface.get_params(candidate, fp, fw, alpha, False, False)
+  fp[2][0x2FF] = 8
+  mirrored = CarInterface.get_params(candidate, fp, fw, alpha, False, False)
+  assert detector(candidate)(candidate, fp, fw)
+  assert rav4h_filter.configured(mirrored)
+  assert mirrored.to_dict() == original.to_dict()
+
+
+@pytest.mark.parametrize('candidate', CARS)
+@pytest.mark.parametrize('bad', ['mirror_only', 'bus1', 'bus3', 'mirror0', 'mirror4', 'mirror7', 'source7'])
+def test_copy_cannot_enroll_wrong_source_or_length(candidate, bad):
+  fp = gen_empty_fingerprint()
+  fp[0][0x2FF] = 8
+  fp[2][0x2FF] = 8
+  if bad == 'mirror_only':
+    del fp[0][0x2FF]
+  elif bad in ('bus1', 'bus3'):
+    fp[int(bad[-1])] = {0x2FF: 8}
+  elif bad.startswith('mirror'):
+    fp[2][0x2FF] = int(bad[-1])
+  else:
+    fp[0][0x2FF] = 7
+  assert not detector(candidate)(candidate, fp, firmware(candidate))
@@ -0,0 +1,49 @@
+import pytest
+from opendbc.car.toyota import eps8_sdsu, rav4h_filter
+from opendbc.car import gen_empty_fingerprint
+from opendbc.car.toyota.interface import CarInterface
+from opendbc.car.toyota.values import CAR
+from opendbc.safety.tests.libsafety import libsafety_py
+from openpilot.starpilot.car.toyota.tests import test_eps8_sdsu_callers as eps8
+from openpilot.starpilot.car.toyota.tests import test_rav4h_caller_native as helpers
+from openpilot.starpilot.car.toyota.tests.test_toyota_cpp_transport import host_heartbeat_bridge as host_heartbeat_bridge
+
+source_matched_native = helpers.source_matched_native
+daemon_gc_scope = helpers.daemon_gc_scope
+CARS = sorted(eps8_sdsu.CARS | rav4h_filter.EPS5_SDSU_CARS)
+
+
+def run(candidate, stock, bridge, scenario='nominal'):
+  if candidate in eps8_sdsu.CARS:
+    eps8.run(candidate, stock, bridge, scenario, sdsu_mirror=True)
+  else:
+    helpers.run_case(stock, scenario, candidate=candidate, bridge=bridge, input_clock=True, sdsu_mirror=True)
+
+
+@pytest.mark.parametrize('candidate', CARS)
+@pytest.mark.parametrize('stock', [False, True])
+def test_each_forwarded_sdsu_full_caller(candidate, stock, host_heartbeat_bridge):
+  run(candidate, stock, host_heartbeat_bridge)
+
+
+@pytest.mark.parametrize('candidate', [CAR.TOYOTA_RAV4H, CAR.TOYOTA_CAMRY, CAR.TOYOTA_COROLLA_TSS2, CAR.TOYOTA_RAV4_TSS2_2023])
+@pytest.mark.parametrize('stock', [False, True])
+@pytest.mark.parametrize('scenario', ['faults', 'off'])
+def test_copy_cannot_mask_bus0_source_loss_or_saved_off(candidate, stock, scenario, host_heartbeat_bridge):
+  run(candidate, stock, host_heartbeat_bridge, scenario)
+
+
+@pytest.mark.parametrize('candidate', CARS)
+@pytest.mark.parametrize('stock', [False, True])
+def test_native_forwarding_keeps_existing_owner(candidate, stock):
+  fp = gen_empty_fingerprint()
+  fp[0][0x2FF] = fp[2][0x2FF] = 8
+  cp = CarInterface.get_params(candidate, fp, helpers.firmware(candidate), True, False, False)
+  if stock:
+    assert rav4h_filter.prepare_stock(cp)
+  safety = libsafety_py.libsafety
+  safety.init_tests()
+  assert safety.set_safety_hooks(cp.safetyConfigs[0].safetyModel.raw, cp.safetyConfigs[0].safetyParam) == 0
+  for bus, target in ((0, 2), (2, 0)):
+    for address in (0x2FF, 0x343):
+      assert safety.safety_fwd_hook(bus, address) == target
@@ -340,8 +340,9 @@ class _Canvas:
    for key in RUNTIME_PREVIEW_WIDGETS[str(self.profile)]:
      logical_width, logical_height = widget_size(state.customization, self.profile, key)
      # The restored compact US sign adds an offset row beyond its saved base size.
-      if (self.profile == Profile.COMPACT and key == 'speed_limit' and
-          not state.appearance.use_vienna_sign and resolve_unified_speed(state).offset_text):
+      compact_offset = (self.profile == Profile.COMPACT and key == 'speed_limit' and
+                        not state.appearance.use_vienna_sign and bool(resolve_unified_speed(state).offset_text))
+      if compact_offset:
        logical_height += 44
      position = positions[key]
      widget = PROFILES[str(self.profile)]['widgets'][key]
@@ -349,6 +350,8 @@ class _Canvas:
      crop_y = max(0, math.floor(position['y'] - widget.get('visualInsetTop', 0)))
      offset_x, offset_y = crop_x - position['x'], crop_y - position['y']
      width, height = math.ceil(logical_width - offset_x), math.ceil(logical_height - offset_y)
+      if compact_offset and 0 <= crop_y < canvas_height:
+        height = min(height, canvas_height - crop_y)
      if x + width > 1024:
        x, y, row_height = 0, y + row_height, 0
      sprites[key] = {'x': x, 'y': y, 'width': width, 'height': height, 'offsetX': offset_x, 'offsetY': offset_y}
@@ -238,9 +238,10 @@ def test_compact_preview_sprite_keeps_the_sign_offset_arrow_and_override_cross()
    rl.init_window(128, 128, 'SLC preview crop fixture')
  canvas = _Canvas(Profile.COMPACT)
  try:
-    for y, scene in product((20, 0), ('slc_pending', 'slc_retained')):
+    for y, scene in product((20, 0, 64, 108), ('slc_pending', 'slc_retained')):
      document = default_document()
      document['layouts']['compact']['speed_limit']['y'] = y
+      assert validate_document(document)['layouts']['compact']['speed_limit']['y'] == y
      with patch.dict('openpilot.starpilot.ui.layout_preview_renderer.RUNTIME_PREVIEW_WIDGETS', compact=('speed_limit',)):
        png = canvas.render_widget_art(sample_state(scene, document), document['layouts']['compact'])
      with Image.open(io.BytesIO(png)) as sheet:
@@ -256,7 +257,9 @@ def test_compact_preview_sprite_keeps_the_sign_offset_arrow_and_override_cross()
        rl.unload_image(image)
      assert np.array_equal(direct[..., 3], np.asarray(rebuilt)[..., 3])
      assert slot['offsetX'] == slot['offsetY'] == 0
-      assert (slot['width'], slot['height']) == (118, 176)
+      assert (slot['width'], slot['height']) == (118, min(176, Profile.COMPACT.size[1] - y))
+      if scene == 'slc_retained':
+        assert np.any(direct[..., 3]), (y, scene)
  finally:
    canvas.close()
    if owns_context:
@@ -1 +1 @@
-{"protected_inventory_sha256": "25f884a9a6495f49cc4c52130811797f029a4f4ff31dd43626d0dc0c7fdae940", "schema_version": 1}
+{"protected_inventory_sha256": "6101d9d0ca764768158bd18dcc22b09b80524729736f32b7021c377b4855877d", "schema_version": 1}
@@ -238,6 +238,8 @@ def complete_mutation_artifact(path, summary, *, digest_out=None):


 def check_import_origins(python, env, output, *, full_catalog=False):
+  sidecar = output.with_suffix(".origins.json")
+  sidecar.unlink(missing_ok=True)
  script = """
 import json
 from pathlib import Path
@@ -278,14 +280,15 @@ print(json.dumps({'msgq_visionipc': str(Path(msgq.visionipc.visionipc_pyx.__file
              "cereal_messaging": ROOT / "openpilot/cereal", "msgq_ipc": ROOT / "msgq_repo"}
  if full_catalog:
    expected["msgq_visionipc"] = ROOT / "msgq_repo"
-  return all(isinstance(origins.get(name), str) and Path(origins[name]).is_relative_to(path)
-             for name, path in expected.items())
+  valid = all(isinstance(origins.get(name), str) and Path(origins[name]).is_relative_to(path)
+              for name, path in expected.items())
+  if valid:
+    sidecar.write_text(json.dumps(origins, sort_keys=True) + "\n")
+  return valid


 def imported_module_hashes(output):
-  origins = {}
-  for line in output.read_text().splitlines():
-    origins.update(json.loads(line))
+  origins = json.loads(output.with_suffix(".origins.json").read_text())
  return {name: {"path": path, "sha256": hashlib.sha256(Path(path).read_bytes()).hexdigest()}
          for name, path in origins.items()}

@@ -56,8 +56,42 @@ class TestSafetyQualificationRunner(TestCase):
        Path(path).write_bytes(b"native")
      log = root / "imports.log"
      log.write_text("\n".join(json.dumps({name: path}) for name, path in modules.items()) + "\n")
+      log.with_suffix(".origins.json").write_text(json.dumps(modules))
      self.assertEqual(set(qualification.imported_module_hashes(log)), set(modules))

+  def test_validated_origins_sidecar_retains_multiline_stdout_and_stderr(self):
+    with tempfile.TemporaryDirectory() as td:
+      root = Path(td)
+      prefixes = {"opendbc": "opendbc_repo", "panda": "panda", "msgq": "msgq_repo",
+                  "openpilot": "openpilot", "libsafety_py": "opendbc_repo",
+                  "can_parser": "opendbc_repo", "can_packer": "opendbc_repo", "can_dbc": "opendbc_repo",
+                  "cereal_log": "openpilot/cereal", "cereal_messaging": "openpilot/cereal", "msgq_ipc": "msgq_repo",
+                  "msgq_visionipc": "msgq_repo"}
+      origins = {name: str(root / prefix / name) for name, prefix in prefixes.items()}
+      for path in origins.values():
+        Path(path).parent.mkdir(parents=True, exist_ok=True)
+        Path(path).write_bytes(b"actual provenance fixture")
+      first = {name: path for name, path in origins.items() if name != "msgq_visionipc"}
+      stdout = json.dumps(first) + "\n" + json.dumps({"msgq_visionipc": origins["msgq_visionipc"]}) + "\n"
+      warning = "kj/filesystem-disk-unix.c++:1734: warning: PWD environment variable doesn't match current directory\n"
+      log = root / "imports.log"
+      sidecar = log.with_suffix(".origins.json")
+      with patch.object(qualification, "ROOT", root), patch.object(qualification.subprocess, "run",
+          return_value=subprocess.CompletedProcess([], 0, stdout, warning)):
+        self.assertTrue(qualification.check_import_origins(sys.executable, {}, log, full_catalog=True))
+      self.assertEqual(log.read_text(), stdout + warning)
+      self.assertEqual(json.loads(sidecar.read_text()), origins)
+      self.assertEqual(set(qualification.imported_module_hashes(log)), set(origins))
+      foreign = dict(origins, msgq_visionipc="/foreign/visionipc.so")
+      for code, invalid in ((0, json.dumps(foreign)), (0, "bad stdout"), (1, stdout)):
+        sidecar.write_text(json.dumps(origins))  # Never reuse earlier success after a failed probe.
+        with self.subTest(code=code, stdout=invalid), patch.object(qualification, "ROOT", root), \
+             patch.object(qualification.subprocess, "run", return_value=subprocess.CompletedProcess([], code, invalid, warning)):
+          self.assertFalse(qualification.check_import_origins(sys.executable, {}, log, full_catalog=True))
+        self.assertFalse(sidecar.exists())
+        with self.assertRaises(FileNotFoundError):
+          qualification.imported_module_hashes(log)
+
  def test_source_identity_includes_gate_and_workflow_inputs(self):
    identity = qualification.source_identity()
    self.assertFalse(any("/obj/" in path or "/gen/" in path for path in identity))
@@ -456,7 +490,25 @@ class TestSafetyQualificationRunner(TestCase):
    with tempfile.TemporaryDirectory() as td:
      output = Path(td)
      gate = qualification.Gate("coverage", output, "/usr/bin/python3")
-      code, text = gate.run("timeout", [sys.executable, "-c", "import time; print('partial', flush=True); time.sleep(10)"], timeout=0.05)
+      ready = output / "child-ready"
+      script = "import pathlib,sys,time; print('partial', flush=True); pathlib.Path(sys.argv[1]).touch(); time.sleep(10)"
+      spawn = subprocess.Popen
+
+      def started_child(*args, **kwargs):
+        proc = spawn(*args, **kwargs)
+        deadline = time.monotonic() + 5
+        while not ready.exists():
+          if proc.poll() is not None or time.monotonic() >= deadline:
+            if proc.poll() is None:
+              os.killpg(proc.pid, signal.SIGKILL)
+            proc.communicate()
+            self.fail("Timeout fixture never became ready")
+          time.sleep(.005)
+        return proc
+
+      # Exercise the real timeout after the child's partial output is actually flushed.
+      with patch.object(qualification.subprocess, "Popen", side_effect=started_child):
+        code, text = gate.run("timeout", [sys.executable, "-c", script, str(ready)], timeout=0.05)
      self.assertEqual(code, 124)
      self.assertIn("Timed out", text)
      self.assertIn("partial", (output / "timeout.log").read_text())
@@ -1,6 +1,7 @@
 """Catch device binaries linked to build-environment Cap'n Proto/KJ libraries."""

 from pathlib import Path
+import ast
 import subprocess
 import sys
 import unittest
@@ -22,3 +23,26 @@ class DeviceRuntimeDependenciesTest(unittest.TestCase):
        dynamic = subprocess.check_output(['readelf', '--dynamic', '--wide', str(ROOT / binary)], text=True)
        self.assertNotRegex(dynamic, r'\(NEEDED\).*\[.*lib(?:capnp|kj)',
                            f"{binary}: link Cap'n Proto/KJ statically for AGNOS")
+
+
+class LiveSteeringServiceInventoryTest(unittest.TestCase):
+  def test_live_steering_tracking_is_registered_in_shipped_native_consumers(self):
+    service = 'starpilotSteeringTracking'
+    tree = ast.parse((ROOT / 'openpilot/cereal/services.py').read_text())
+    declarations = [node.value for node in tree.body if isinstance(node, ast.AnnAssign)
+                    and isinstance(node.target, ast.Name) and node.target.id == '_services']
+    self.assertEqual(len(declarations), 1)
+    table = declarations[0]
+    if not isinstance(table, ast.Dict):
+      self.fail('Service registry must be a dictionary')
+    declared = [value for key, value in zip(table.keys, table.values, strict=True)
+                if isinstance(key, ast.Constant) and key.value == service]
+    self.assertEqual(len(declared), 1)
+    definition = declared[0]
+    if not isinstance(definition, ast.Tuple):
+      self.fail('Service declaration must be a tuple')
+    self.assertIs(ast.literal_eval(definition.elts[0]), True)
+    for binary in ('openpilot/system/loggerd/loggerd', 'openpilot/tools/replay/replay'):
+      with self.subTest(binary=binary):
+        self.assertIn(service.encode(), (ROOT / binary).read_bytes(),
+                      f'{binary}: rebuild after changing the native service registry')