mawei/opendbc
1
0
Fork 0
mirror of https://github.com/commaai/opendbc.git synced 2026-06-08 06:04:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

speed up safety tests with boundary-focused testing (#3200)

Browse Source
This commit is contained in:
Adeeb Shihadeh
2026-03-11 19:01:54 -07:00
committed by GitHub
parent 31e816bff7
commit 3a852b049f
5 changed files with 55 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lefthook.yml
View File

@@ -27,4 +27,4 @@ test:
# *** tests ***
unittest:
run: unittest-parallel -j8 -s opendbc -p 'test_*.py' -t .
run: unittest-parallel -j4

24
opendbc/safety/tests/common.py
View File

@@ -90,12 +90,25 @@ class SafetyTestBase(unittest.TestCase):
def _tx(self, msg):
return self.safety.safety_tx_hook(msg)
@staticmethod
def _boundary_values(boundaries, min_val, max_val, step=1, width=5, sparse_count=100):
"""Generate test values dense around boundaries and sparse across the full range."""
values = set()
for b in boundaries:
for offset in range(-width, width + 1):
v = round(b + offset * step, 2)
if min_val <= v < max_val:
values.add(v)
sparse_step = max(step, (max_val - min_val) / sparse_count)
for v in np.arange(min_val, max_val, sparse_step):
values.add(round(v, 2))
return sorted(values)
def _generic_limit_safety_check(self, msg_function: MessageFunction, min_allowed_value: float, max_allowed_value: float,
min_possible_value: float, max_possible_value: float, test_delta: float = 1, inactive_value: float = 0,
msg_allowed = True, additional_setup: Callable[[float], None] | None = None):
"""
Enforces that a signal within a message is only allowed to be sent within a specific range, min_allowed_value -> max_allowed_value.
Tests the range of min_possible_value -> max_possible_value with a delta of test_delta.
Message is also only allowed to be sent when controls_allowed is true, unless the value is equal to inactive_value.
Message is never allowed if msg_allowed is false, for example when stock longitudinal is enabled and you are sending acceleration requests.
additional_setup is used for extra setup before each _tx, ex: for setting the previous torque for rate limits
@@ -105,10 +118,11 @@ class SafetyTestBase(unittest.TestCase):
self.assertGreater(max_possible_value, max_allowed_value)
self.assertLessEqual(min_possible_value, min_allowed_value)
test_values = self._boundary_values([min_allowed_value, max_allowed_value, 0, inactive_value],
min_possible_value, max_possible_value, test_delta)
for controls_allowed in [False, True]:
# enforce we don't skip over 0 or inactive
for v in np.concatenate((np.arange(min_possible_value, max_possible_value, test_delta), np.array([0, inactive_value]))):
v = round(v, 2) # floats might not hit exact boundary conditions without rounding
for v in test_values:
self.safety.set_controls_allowed(controls_allowed)
if additional_setup is not None:
additional_setup(v)
@@ -448,7 +462,7 @@ class DriverTorqueSteeringSafetyTest(TorqueSteeringSafetyTestBase, abc.ABC):
# Cannot stay at MAX_TORQUE if above DRIVER_TORQUE_ALLOWANCE
for sign in [-1, 1]:
for driver_torque in np.arange(0, self.DRIVER_TORQUE_ALLOWANCE * 2, 1):
for driver_torque in self._boundary_values([self.DRIVER_TORQUE_ALLOWANCE], 0, self.DRIVER_TORQUE_ALLOWANCE * 2):
self._reset_torque_driver_measurement(-driver_torque * sign)
self._set_prev_torque(max_torque * sign)
should_tx = abs(driver_torque) <= self.DRIVER_TORQUE_ALLOWANCE

2
opendbc/safety/tests/test.sh
View File

@@ -10,7 +10,7 @@ source ../../../setup.sh
rm -f ./libsafety/*.gcda
# run safety tests and generate coverage data
python -m unittest discover -s . -p 'test_*.py' -t ../../../
python -m unittest discover -s .
# NOTE: we accept that these tools will have slight differences,
# and in return, we get to use the stock toolchain instead of

19
opendbc/safety/tests/test_ford.py
View File

@@ -241,16 +241,22 @@ class TestFordSafetyBase(common.CarSafetyTest):
def test_max_lateral_acceleration(self):
# Ford CAN FD can achieve a higher max lateral acceleration than CAN so we limit curvature based on speed
step = 1 / self.DEG_TO_CAN
for speed in np.arange(0, 40, 0.5):
# Clip so we test curvature limiting at low speed due to low max curvature
_, curvature_accel_limit_upper = self.get_canfd_curvature_limits(speed)
curvature_accel_limit_upper = np.clip(curvature_accel_limit_upper, -self.MAX_CURVATURE, self.MAX_CURVATURE)
# Test boundary curvature values around the limit, rounded to CAN precision
lower = curvature_accel_limit_upper * 0.8
upper = min(curvature_accel_limit_upper * 1.2, self.MAX_CURVATURE)
test_curvatures = {round(c * self.DEG_TO_CAN) / self.DEG_TO_CAN
for c in self._boundary_values([curvature_accel_limit_upper], lower, upper, step)
if 0 <= c <= self.MAX_CURVATURE}
for sign in (-1, 1):
# Test above and below the lateral by 20%, max is clipped since
# max curvature at low speed is higher than the signal max
for curvature in np.arange(curvature_accel_limit_upper * 0.8, min(curvature_accel_limit_upper * 1.2, self.MAX_CURVATURE), 1 / self.DEG_TO_CAN):
curvature = sign * round(curvature * self.DEG_TO_CAN) / self.DEG_TO_CAN # fix np rounding errors
for curvature in sorted(test_curvatures):
curvature = sign * curvature
self.safety.set_controls_allowed(True)
self._set_prev_desired_angle(curvature)
self._reset_curvature_measurement(curvature, speed)
@@ -439,11 +445,12 @@ class TestFordLongitudinalSafetyBase(TestFordSafetyBase):
self.assertEqual(should_tx, self._tx(self._acc_command_msg(gas, self.INACTIVE_ACCEL, controls_allowed)))
def test_brake_safety_check(self):
brake_values = self._boundary_values([self.MIN_ACCEL, self.MAX_ACCEL, self.INACTIVE_ACCEL],
self.MIN_ACCEL - 2, self.MAX_ACCEL + 2, 0.05)
for controls_allowed in (True, False):
self.safety.set_controls_allowed(controls_allowed)
for brake_actuation in (True, False):
for brake in np.arange(self.MIN_ACCEL - 2, self.MAX_ACCEL + 2, 0.05):
brake = round(brake, 2) # floats might not hit exact boundary conditions without rounding
for brake in brake_values:
should_tx = (controls_allowed and self.MIN_ACCEL <= brake <= self.MAX_ACCEL) or brake == self.INACTIVE_ACCEL
should_tx = should_tx and (controls_allowed or not brake_actuation)
self.assertEqual(should_tx, self._tx(self._acc_command_msg(self.INACTIVE_GAS, brake, brake_actuation)))

37
opendbc/safety/tests/test_toyota.py
View File

@@ -202,28 +202,33 @@ class TestToyotaSafetyAngle(TestToyotaSafetyBase, common.AngleSteeringSafetyTest
should_tx = req == req2 and (torque_wind_down in (0, 100)) and not mismatch
self.assertEqual(should_tx, self._tx(self._lta_msg(req, req2, angle, torque_wind_down)))
# Test max EPS torque and driver override thresholds
cases = itertools.product(
(0, self.MAX_MEAS_TORQUE - 1, self.MAX_MEAS_TORQUE, self.MAX_MEAS_TORQUE + 1, self.MAX_MEAS_TORQUE * 2),
(0, self.MAX_LTA_DRIVER_TORQUE - 1, self.MAX_LTA_DRIVER_TORQUE, self.MAX_LTA_DRIVER_TORQUE + 1, self.MAX_LTA_DRIVER_TORQUE * 2)
)
for eps_torque, driver_torque in cases:
for sign in (-1, 1):
for _ in range(6):
self._rx(self._torque_meas_msg(sign * eps_torque, sign * driver_torque))
# Toyota adds 1 to EPS torque since it is rounded after EPS factor
should_tx = (eps_torque - 1) <= self.MAX_MEAS_TORQUE and driver_torque <= self.MAX_LTA_DRIVER_TORQUE
self.assertEqual(should_tx, self._tx(self._lta_msg(1, 1, angle, 100)))
self.assertTrue(self._tx(self._lta_msg(1, 1, angle, 0))) # should tx if we wind down torque
else:
# Controls not allowed
for req, req2, torque_wind_down in itertools.product([0, 1], [0, 1], [0, 50, 100]):
should_tx = not (req or req2) and torque_wind_down == 0
self.assertEqual(should_tx, self._tx(self._lta_msg(req, req2, angle, torque_wind_down)))
# Test max EPS torque and driver override thresholds (independent of angle, test a few representative angles)
for angle in (-89, 0, 89):
self.safety.set_controls_allowed(True)
self._reset_angle_measurement(angle)
self._set_prev_desired_angle(angle)
cases = itertools.product(
(0, self.MAX_MEAS_TORQUE - 1, self.MAX_MEAS_TORQUE, self.MAX_MEAS_TORQUE + 1, self.MAX_MEAS_TORQUE * 2),
(0, self.MAX_LTA_DRIVER_TORQUE - 1, self.MAX_LTA_DRIVER_TORQUE, self.MAX_LTA_DRIVER_TORQUE + 1, self.MAX_LTA_DRIVER_TORQUE * 2)
)
for eps_torque, driver_torque in cases:
for sign in (-1, 1):
for _ in range(6):
self._rx(self._torque_meas_msg(sign * eps_torque, sign * driver_torque))
# Toyota adds 1 to EPS torque since it is rounded after EPS factor
should_tx = (eps_torque - 1) <= self.MAX_MEAS_TORQUE and driver_torque <= self.MAX_LTA_DRIVER_TORQUE
self.assertEqual(should_tx, self._tx(self._lta_msg(1, 1, angle, 100)))
self.assertTrue(self._tx(self._lta_msg(1, 1, angle, 0))) # should tx if we wind down torque
def test_angle_measurements(self):
"""
* Tests angle meas quality flag dictates whether angle measurement is parsed, and if rx is valid