diff --git a/openpilot/starpilot/flm/event_physical.py b/openpilot/starpilot/flm/event_physical.py index 62fb53212e..f12d0271a5 100644 --- a/openpilot/starpilot/flm/event_physical.py +++ b/openpilot/starpilot/flm/event_physical.py @@ -39,10 +39,11 @@ def fit_event_physical(identity, context, rows, *, lag_ns, now_ns, min_excitatio # never creates another independent observation or another heldout ID. for group in (train, heldout): for direction in ('left', 'right'): - supporting = {row.block_id for row in group if row.direction == direction} - if sum(row.direction == direction for row in group) < 80: + selected = tuple(row for row in group if (row.command > 0 if direction == 'left' else row.command < 0)) + supporting = {row.block_id for row in selected} + if len(selected) < 80: return None - if sum(sum(row.block_id == block and row.direction == direction for row in group) >= 20 for block in supporting) < 2: + if sum(sum(row.block_id == block for row in selected) >= 20 for block in supporting) < 2: return None mean_x = sum(row.command for row in train) / len(train) mean_y = sum(row.physical_response for row in train) / len(train) diff --git a/openpilot/starpilot/flm/physical_response.py b/openpilot/starpilot/flm/physical_response.py index 6bbc011cda..9edff90b6b 100644 --- a/openpilot/starpilot/flm/physical_response.py +++ b/openpilot/starpilot/flm/physical_response.py @@ -96,6 +96,7 @@ def fit_physical(identity, domain, context, rows, *, lag_ns, min_excitation, max for row in rows: if (not row.healthy or row.overridden or row.saturated or row.context != context or row.direction not in ('left', 'right') or + row.command > 0 and row.direction != 'left' or row.command < 0 and row.direction != 'right' or type(row.command_ns) is not int or type(row.response_ns) is not int or not 0 < row.command_ns <= row.response_ns <= now_ns or row.command_ns < now_ns - history_ns or not math.isfinite(row.command) or not math.isfinite(row.physical_response) or row.command_ns <= previous or @@ -105,7 +106,8 @@ def fit_physical(identity, domain, context, rows, *, lag_ns, min_excitation, max train = [row for row in rows if row.block_id in train_ids] heldout = [row for row in rows if row.block_id in heldout_ids] for group in (train, heldout): - if any(len({row.block_id for row in group if row.direction == direction}) < 2 for direction in ('left', 'right')): + if any(len({row.block_id for row in group if (row.command > 0 if direction == 'left' else row.command < 0)}) < 2 + for direction in ('left', 'right')): return None mean_x = sum(row.command for row in train) / len(train) mean_y = sum(row.physical_response for row in train) / len(train) diff --git a/openpilot/starpilot/flm/tests/test_bounded_stage_regression.py b/openpilot/starpilot/flm/tests/test_bounded_stage_regression.py index 7dbf2a8e14..83aeb93a4f 100644 --- a/openpilot/starpilot/flm/tests/test_bounded_stage_regression.py +++ b/openpilot/starpilot/flm/tests/test_bounded_stage_regression.py @@ -28,7 +28,7 @@ def known_fit(block_count=8): (-1 if block % 2 else 1) * (0.1 + j * 0.01), 2 * (-1 if block % 2 else 1) * (0.1 + j * 0.01) + 0.01, 'calibration-v1', - 'left' if block % 2 else 'right', + 'right' if block % 2 else 'left', True, False, False, diff --git a/openpilot/starpilot/flm/tests/test_event_physical.py b/openpilot/starpilot/flm/tests/test_event_physical.py index d0374b3d7c..6f854fc873 100644 --- a/openpilot/starpilot/flm/tests/test_event_physical.py +++ b/openpilot/starpilot/flm/tests/test_event_physical.py @@ -56,3 +56,30 @@ def test_local_model_does_not_extrapolate_or_accept_quiet_unidentified_input(): unsupported = tuple(replace(row, command=row.command * 2, physical_response=row.physical_response * 2) if row.block_id in ('4', '5', '6', '7') else row for row in rows) assert fit_event_physical('a' * 64, 'physical-context', unsupported, lag_ns=50_000_000, now_ns=unsupported[-1].response_ns) is None + + +def test_zero_and_negative_zero_do_not_supply_negative_direction_support(): + for zero in (0.0, -0.0): + for change_train in (True, False): + rows = tuple(replace(row, command=zero, physical_response=.002, direction='right') + if row.command < 0 and (change_train or row.block_id in ('4', '5', '6', '7')) else row + for row in episodes()) + assert fit_event_physical('a' * 64, 'physical-context', rows, + lag_ns=50_000_000, now_ns=rows[-1].response_ns) is None + + +def test_tiny_positive_has_no_negative_direction_support_and_no_epsilon_is_used(): + for tiny, expected in ((1e-15, False), (-1e-15, True)): + rows = tuple(replace(row, command=tiny, physical_response=1.98 * tiny + .002, + direction='left' if tiny > 0 else 'right') if row.command < 0 else row + for row in episodes()) + model = fit_event_physical('a' * 64, 'physical-context', rows, + lag_ns=50_000_000, now_ns=rows[-1].response_ns) + assert (model is not None) is expected + + +def test_true_bilateral_original_control_still_qualifies(): + rows = episodes() + assert any(row.command > 0 for row in rows) and any(row.command < 0 for row in rows) + assert fit_event_physical('a' * 64, 'physical-context', rows, + lag_ns=50_000_000, now_ns=rows[-1].response_ns) is not None diff --git a/openpilot/starpilot/flm/tests/test_physical_direction_support.py b/openpilot/starpilot/flm/tests/test_physical_direction_support.py new file mode 100644 index 0000000000..7dac07255b --- /dev/null +++ b/openpilot/starpilot/flm/tests/test_physical_direction_support.py @@ -0,0 +1,77 @@ +"""Neutral commands and direction labels cannot manufacture bilateral evidence.""" +from dataclasses import replace +import math + +import pytest + +from openpilot.starpilot.flm.event_physical import fit_event_physical +from openpilot.starpilot.flm.physical_response import PhysicalRow, fit_physical + + +IDENTITY = 'a' * 64 +CONTEXT = 'exact-independent-sign-support' +LAG = 50_000_000 + + +def cohort(signs): + rows = [] + for block, sign in enumerate(signs): + for sample in range(80): + stamp = 1_000_000_000 + (block * 80 + sample) * 50_000_000 + command = sign * (.10 + sample * .004) + rows.append(PhysicalRow(str(block), stamp, stamp + LAG, command, + 1.98 * command + .002, CONTEXT, + 'left' if command > 0 else 'right', True, False, False)) + return tuple(rows) + + +def event_fit(rows): + return fit_event_physical(IDENTITY, CONTEXT, rows, lag_ns=LAG, now_ns=rows[-1].response_ns) + + +@pytest.mark.parametrize('neutral', [0.0, -0.0]) +def test_two_positive_two_neutral_episodes_per_chronological_half_cannot_fit(neutral): + rows = cohort((1., 1., neutral, neutral, 1., 1., neutral, neutral)) + assert len(rows) == 640 and len({row.block_id for row in rows}) == 8 + for half in (rows[:320], rows[320:]): + assert sum(row.command > 0 for row in half) == 160 + assert sum(row.command == 0 for row in half) == 160 + assert not any(row.command < 0 for row in half) + assert sum(row.direction == 'right' for row in half) == 160 + # Independent measured-response rule has ample positive-vs-zero variance + # and exact heldout OLS; the sole missing qualification is negative support. + train = rows[:320] + mean = sum(row.command for row in train) / len(train) + assert sum((row.command - mean)**2 for row in train) / len(train) > .05**2 + assert all(math.isclose(row.physical_response, 1.98 * row.command + .002, abs_tol=1e-15) for row in rows) + assert event_fit(rows) is None + + +@pytest.mark.parametrize('neutral', [0.0, -0.0]) +def test_neutral_observations_remain_valid_when_real_bilateral_support_exists(neutral): + data = cohort((1., -1., 1., -1., 1., -1., 1., -1.)) + # Keep60 substantive sign rows perblock, hence120/sign/half and2independent + # blocks, while preserving all neutral points inOLS/heldout/error evaluation. + rows = tuple(replace(row, command=neutral, physical_response=.002, direction='right') + if index % 80 < 20 else row for index, row in enumerate(data)) + model = event_fit(rows) + assert model is not None + assert math.isclose(model.slope, 1.98, abs_tol=1e-12) + assert model.train_blocks == ('0', '1', '2', '3') + assert model.heldout_blocks == ('4', '5', '6', '7') + assert model.heldout_rmse < 1e-12 + + +def test_real_negative_rows_only_in_one_episode_do_not_become_two_with_zero_episode(): + rows = cohort((1., -1., 1., 0., 1., -1., 1., 0.)) + assert event_fit(rows) is None + + +def test_legacy_wrong_sign_labels_cannot_create_bilateral_support(): + # Unlike event rows, legacyblocks must each have one direction. Preserve + # chronology, nonzero variance and exactphysicalOLS, but forge half labels. + data = cohort((1., 1., 1., 1., 1., 1., 1., 1.)) + rows = tuple(replace(row, direction='left' if int(row.block_id) % 2 == 0 else 'right') for row in data) + assert all(row.command > 0 for row in rows) + assert fit_physical(IDENTITY, 'normalized_torque', CONTEXT, rows, lag_ns=LAG, + min_excitation=.05, max_rmse=.05, now_ns=rows[-1].response_ns) is None diff --git a/openpilot/starpilot/flm/tests/test_physical_response.py b/openpilot/starpilot/flm/tests/test_physical_response.py index ff2983225c..3487b06791 100644 --- a/openpilot/starpilot/flm/tests/test_physical_response.py +++ b/openpilot/starpilot/flm/tests/test_physical_response.py @@ -10,7 +10,7 @@ def rows(): 1_110_000_000 + (block * 40 + index) * 50_000_000, (-1 if block % 2 else 1) * (.1 + index * .005), 2 * (-1 if block % 2 else 1) * (.1 + index * .005) + .01, - 'stable_vm_roll_lag', 'left' if block % 2 else 'right', True, False, False) + 'stable_vm_roll_lag', 'right' if block % 2 else 'left', True, False, False) for block in range(8) for index in range(40))