mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-11-08 05:23:41 +08:00
firestar5683
311 lines
16 KiB
Python
311 lines
16 KiB
Python
from pathlib import Path
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
from openpilot.cereal import messaging
|
|
import zstandard as zstd
|
|
|
|
from openpilot.starpilot.models.catalog import BY_ID
|
|
from openpilot.starpilot.galaxy import drive_analysis
|
|
from openpilot.starpilot.galaxy.drive_analysis import analyze_route
|
|
|
|
|
|
ROUTE = '2026-09-25--12-00-00'
|
|
BASE = 1_000_000_000
|
|
|
|
|
|
def event(kind, stamp, **fields):
|
|
alerts = fields.pop('onroadEvents', None)
|
|
message = messaging.new_message(kind, size=len(alerts), valid=True) if alerts is not None else messaging.new_message(kind, valid=True)
|
|
message.logMonoTime = stamp
|
|
payload = getattr(message, kind)
|
|
if alerts is not None:
|
|
for item, alert in zip(payload, alerts, strict=True):
|
|
item.name = alert['name']
|
|
for key, value in fields.items():
|
|
setattr(payload, key, value)
|
|
return message
|
|
|
|
|
|
def write(root, number, events, *, lock=False, kind='rlog'):
|
|
name = f'{ROUTE}--{number}'
|
|
segment = root / name
|
|
segment.mkdir()
|
|
(segment / f'{kind}.zst').write_bytes(zstd.compress(b''.join(item.as_reader().as_builder().to_bytes() for item in events)))
|
|
if lock:
|
|
(segment / 'rlog.lock').touch()
|
|
return {'number': number, 'segmentName': name, 'files': {kind: True}}
|
|
|
|
|
|
def selection(*segments):
|
|
return {'routeId': ROUTE, 'segmentCount': len(segments), 'segments': list(segments)}
|
|
|
|
|
|
class DriveAnalysisTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name).resolve()
|
|
|
|
def sample(self, stamp, speed, enabled, alerts=()):
|
|
return [event('carState', stamp, vEgo=speed), event('selfdriveState', stamp, enabled=enabled),
|
|
event('onroadEvents', stamp, onroadEvents=[{'name': name} for name in alerts])]
|
|
|
|
def test_aol_counts_control_activity_separately_across_segments(self):
|
|
samples = []
|
|
for index, (enabled, active) in enumerate(((False, True), (False, True), (True, True), (False, False), (False, False))):
|
|
stamp = BASE + index * 500_000_000
|
|
samples.append([*self.sample(stamp, 10, enabled),
|
|
event('carControl', stamp, enabled=enabled, latActive=active),
|
|
# An admitted axis is not proof of active control.
|
|
event('aolAxisState', stamp, lateralActive=True, desiredLateral=True)])
|
|
first = [*samples[0], *samples[1], event('sentinel', BASE + 750_000_000, type='endOfSegment')]
|
|
second = [*samples[2], *samples[3], *samples[4], event('sentinel', BASE + 2_000_000_000, type='endOfRoute')]
|
|
result = analyze_route(self.root, selection(write(self.root, 0, first, kind='qlog'),
|
|
write(self.root, 1, second, kind='qlog')), permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual((result['aolSeconds'], result['aolPercent']), (1.0, 50.0))
|
|
self.assertEqual((result['engagedSeconds'], result['engagedPercent']), (.5, 25.0))
|
|
self.assertEqual(result['distanceMeters'], 20)
|
|
|
|
def test_gapped_control_does_not_invalidate_existing_statistics(self):
|
|
events = [item for index in range(5) for item in self.sample(BASE + index * 500_000_000, 10, True)]
|
|
events += [event('carControl', BASE, enabled=False, latActive=True),
|
|
event('carControl', BASE + 2_000_000_000, enabled=False, latActive=True),
|
|
event('sentinel', BASE + 2_000_000_000, type='endOfRoute')]
|
|
result = analyze_route(self.root, selection(write(self.root, 0, events)), permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['engagedSeconds'], 2)
|
|
self.assertIsNone(result['aolSeconds'])
|
|
self.assertIsNone(result['aolPercent'])
|
|
|
|
def test_complete_contiguous_route_uses_observed_values_and_event_edges(self):
|
|
first = [event('clocks', BASE, wallTimeNanos=1_790_334_400_000_000_000),
|
|
*self.sample(BASE, 0, True),
|
|
*self.sample(BASE + 500_000_000, 10, True, ('driverDistracted2',)),
|
|
event('sentinel', BASE + 900_000_000, type='endOfSegment')]
|
|
second = [*self.sample(BASE + 1_000_000_000, 10, False, ('driverDistracted2', 'driverUnresponsive3')),
|
|
*self.sample(BASE + 1_500_000_000, 10, False),
|
|
event('sentinel', BASE + 1_500_000_000, type='endOfRoute')]
|
|
route = selection(write(self.root, 0, first), write(self.root, 1, second))
|
|
result = analyze_route(self.root, route, permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['distanceMeters'], 12.5)
|
|
self.assertEqual(result['durationSeconds'], 1.5)
|
|
self.assertEqual(result['engagedSeconds'], 1)
|
|
self.assertAlmostEqual(result['engagedPercent'], 66.7)
|
|
self.assertEqual(result['distractedMoments'], 1)
|
|
self.assertEqual(result['unresponsiveMoments'], 1)
|
|
self.assertEqual(result['startTime'], 1_790_334_400)
|
|
self.assertIsNone(result['model'])
|
|
self.assertIsNone(result['aolSeconds'])
|
|
self.assertIsNone(result['aolPercent'])
|
|
|
|
def test_missing_sentinel_gap_and_noncontiguous_segments_are_incomplete(self):
|
|
events = [*self.sample(BASE, 5, True), *self.sample(BASE + 2_000_000_000, 5, True)]
|
|
route = selection(write(self.root, 0, events))
|
|
result = analyze_route(self.root, route, permitted=lambda: True)
|
|
self.assertFalse(result['complete'])
|
|
self.assertIsNone(result['distanceMeters'])
|
|
self.assertIn('End of route', result['reason'])
|
|
route['segments'][0]['number'] = 1
|
|
self.assertIn('missing segments', analyze_route(self.root, route, permitted=lambda: True)['reason'])
|
|
|
|
def test_lock_and_added_segment_invalidate_inventory(self):
|
|
first = write(self.root, 0, [*self.sample(BASE, 5, False), *self.sample(BASE + 500_000_000, 5, False),
|
|
event('sentinel', BASE + 500_000_000, type='endOfRoute')], lock=True)
|
|
result = analyze_route(self.root, selection(first), permitted=lambda: True)
|
|
self.assertFalse(result['complete'])
|
|
(self.root / first['segmentName'] / 'rlog.lock').unlink()
|
|
write(self.root, 1, [event('sentinel', BASE + 1_000_000_000, type='endOfRoute')])
|
|
result = analyze_route(self.root, selection(first), permitted=lambda: True)
|
|
self.assertFalse(result['complete'])
|
|
self.assertIn('segments changed', result['reason'])
|
|
|
|
def test_cancellation_is_incomplete(self):
|
|
first = write(self.root, 0, [*self.sample(BASE, 5, True), *self.sample(BASE + 500_000_000, 5, True),
|
|
event('sentinel', BASE + 500_000_000, type='endOfRoute')])
|
|
self.assertFalse(analyze_route(self.root, selection(first), permitted=lambda: False)['complete'])
|
|
|
|
def test_qlog_without_attention_samples_preserves_unknown_moments(self):
|
|
events = [event('sentinel', BASE, type='startOfRoute'),
|
|
event('carState', BASE + 2_000_000_000, vEgo=5),
|
|
event('selfdriveState', BASE + 2_000_000_000, enabled=True),
|
|
event('carState', BASE + 2_500_000_000, vEgo=5),
|
|
event('selfdriveState', BASE + 2_500_000_000, enabled=False),
|
|
event('sentinel', BASE + 3_000_000_000, type='endOfRoute')]
|
|
route = selection(write(self.root, 0, events, kind='qlog'))
|
|
result = analyze_route(self.root, route, permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['distanceMeters'], 2.5)
|
|
self.assertEqual(result['engagedSeconds'], .5)
|
|
self.assertIsNotNone(result['startTime'])
|
|
self.assertIsNone(result['distractedMoments'])
|
|
self.assertIsNone(result['unresponsiveMoments'])
|
|
|
|
def test_model_is_only_the_selection_recorded_in_init_data(self):
|
|
init = messaging.new_message('initData', valid=True)
|
|
init.logMonoTime = BASE
|
|
entries = init.initData.init('params').init('entries', 1)
|
|
entries[0].key = 'DrivingModelName'
|
|
entries[0].value = b'Recorded choice'
|
|
events = [init, event('carState', BASE, vEgo=5), event('selfdriveState', BASE, enabled=False),
|
|
event('carState', BASE + 500_000_000, vEgo=5),
|
|
event('selfdriveState', BASE + 500_000_000, enabled=False),
|
|
event('sentinel', BASE + 500_000_000, type='endOfRoute')]
|
|
route = selection(write(self.root, 0, events))
|
|
result = analyze_route(self.root, route, permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['model'], 'Selected: Recorded choice')
|
|
|
|
def model_events(self, big=False):
|
|
return [*self.sample(BASE, 5, True), event('drivingModelData', BASE + 100_000_000, big=big),
|
|
*self.sample(BASE + 500_000_000, 5, True), event('sentinel', BASE + 500_000_000, type='endOfRoute')]
|
|
|
|
def load_event(self, model_id='sc23', big=False, stamp=BASE):
|
|
message = messaging.new_message(None, valid=True)
|
|
message.logMonoTime = stamp
|
|
message.logMessage = json.dumps({'process': 123, 'msg': {'event': 'modeld.loaded', 'version': 1,
|
|
'pid': 123, 'processStartTicks': 17, 'loadedMonoNs': stamp, 'modelId': model_id,
|
|
'variant': 'chestnut' if big else 'small', 'artifactSha256': 'a' * 64, 'fallbackReason': None}})
|
|
return message
|
|
|
|
def test_qlog_reports_actual_small_fallback_without_a_recorded_identity(self):
|
|
events = self.model_events()
|
|
result = analyze_route(self.root, selection(write(self.root, 0, events, kind='qlog')), permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['model'], 'Small model (identity not recorded)')
|
|
self.assertEqual(result['analysisVersion'], drive_analysis.ANALYSIS_VERSION)
|
|
|
|
def test_retained_rlog_supplies_verified_identity_while_qlog_supplies_stats(self):
|
|
events = self.model_events()
|
|
segment = write(self.root, 0, events, kind='qlog')
|
|
rlog = self.root / segment['segmentName'] / 'rlog.zst'
|
|
rlog.write_bytes(zstd.compress(b''.join(item.as_reader().as_builder().to_bytes() for item in [self.load_event(), *events])))
|
|
segment['files']['rlog'] = True
|
|
result = analyze_route(self.root, selection(segment), permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['model'], f'{BY_ID["sc23"].name} (Small)')
|
|
rlog.write_bytes(b'corrupt optional metadata')
|
|
result = analyze_route(self.root, selection(segment), permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['model'], 'Small model (identity not recorded)')
|
|
|
|
def test_model_fallback_and_restart_do_not_keep_prior_identity(self):
|
|
events = [self.load_event('cinquev3', True), *self.sample(BASE, 5, True),
|
|
event('drivingModelData', BASE + 50_000_000, big=True),
|
|
self.load_event(stamp=BASE + 100_000_000), event('drivingModelData', BASE + 200_000_000, big=False),
|
|
*self.sample(BASE + 500_000_000, 5, True), event('sentinel', BASE + 500_000_000, type='endOfRoute')]
|
|
segment = write(self.root, 0, events)
|
|
result = analyze_route(self.root, selection(segment), permitted=lambda: True)
|
|
self.assertEqual(result['model'], f'{BY_ID["cinquev3"].name} (Chestnut big) → {BY_ID["sc23"].name} (Small)')
|
|
restart = messaging.new_message(None, valid=True)
|
|
restart.logMonoTime = BASE + 150_000_000
|
|
restart.logMessage = json.dumps({'msg': 'modeld init', 'module': 'modeld', 'process': 124})
|
|
events.insert(6, restart)
|
|
(self.root / segment['segmentName'] / 'rlog.zst').write_bytes(zstd.compress(b''.join(item.as_reader().as_builder().to_bytes() for item in events)))
|
|
result = analyze_route(self.root, selection(segment), permitted=lambda: True)
|
|
self.assertIn('Small model (identity not recorded)', result['model'])
|
|
self.assertNotIn(f'{BY_ID["sc23"].name} (Small)', result['model'])
|
|
|
|
def test_failed_or_unconfirmed_load_never_overrides_actual_output(self):
|
|
events = [self.load_event('cinquev3', True), *self.model_events()]
|
|
result = analyze_route(self.root, selection(write(self.root, 0, events)), permitted=lambda: True)
|
|
self.assertEqual(result['model'], 'Small model (identity not recorded)')
|
|
with mock.patch.object(drive_analysis, 'MAX_MODEL_METADATA_SECONDS', 0):
|
|
result = analyze_route(self.root, selection({'number': 0, 'segmentName': f'{ROUTE}--0', 'files': {'rlog': True}}),
|
|
permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertEqual(result['model'], 'Small model (identity not recorded)')
|
|
|
|
def test_sample_gap_and_missing_wall_time_cannot_be_complete(self):
|
|
events = [event('carState', BASE, vEgo=10), event('selfdriveState', BASE, enabled=True),
|
|
event('carState', BASE + 2_000_000_000, vEgo=10),
|
|
event('selfdriveState', BASE + 2_000_000_000, enabled=True),
|
|
event('sentinel', BASE + 2_000_000_000, type='endOfRoute')]
|
|
segment = write(self.root, 0, events, kind='qlog')
|
|
gap = analyze_route(self.root, selection(segment), permitted=lambda: True)
|
|
self.assertFalse(gap['complete'])
|
|
self.assertIsNone(gap['distanceMeters'])
|
|
self.assertIsNone(gap['engagedSeconds'])
|
|
self.assertIn('gapped', gap['reason'])
|
|
|
|
old_name = segment['segmentName']
|
|
no_date = selection(segment)
|
|
no_date['routeId'] = '00000231--8c5f1c4f8b'
|
|
no_date['segments'][0]['segmentName'] = '00000231--8c5f1c4f8b--0'
|
|
(self.root / old_name).rename(self.root / no_date['segments'][0]['segmentName'])
|
|
(self.root / no_date['segments'][0]['segmentName'] / 'qlog.zst').write_bytes(zstd.compress(b''.join(
|
|
item.as_reader().as_builder().to_bytes() for item in [event('carState', BASE, vEgo=10), event('selfdriveState', BASE, enabled=True),
|
|
event('carState', BASE + 500_000_000, vEgo=10),
|
|
event('selfdriveState', BASE + 500_000_000, enabled=False),
|
|
event('sentinel', BASE + 500_000_000, type='endOfRoute')])))
|
|
result = analyze_route(self.root, no_date, permitted=lambda: True)
|
|
self.assertFalse(result['complete'])
|
|
self.assertIsNone(result['startTime'])
|
|
self.assertIn('wall time', result['reason'])
|
|
|
|
def test_129_segment_qlog_route_and_source_guards(self):
|
|
segments = []
|
|
for number in range(129):
|
|
events = []
|
|
for sample in range(67):
|
|
stamp = BASE + number * 60_000_000_000 + sample * 900_000_000
|
|
events.extend([event('carState', stamp, vEgo=10), event('selfdriveState', stamp, enabled=True)])
|
|
if number == 128:
|
|
events.append(event('sentinel', stamp, type='endOfRoute'))
|
|
segments.append(write(self.root, number, events, kind='qlog'))
|
|
route = selection(*segments)
|
|
real_open = drive_analysis.os.open
|
|
real_close = drive_analysis.os.close
|
|
opened = set()
|
|
peak = [0]
|
|
def tracking_open(*args, **kwargs):
|
|
fd = real_open(*args, **kwargs)
|
|
opened.add(fd)
|
|
peak[0] = max(peak[0], len(opened))
|
|
return fd
|
|
def tracking_close(fd):
|
|
opened.discard(fd)
|
|
return real_close(fd)
|
|
with mock.patch.object(drive_analysis.os, 'open', side_effect=tracking_open), \
|
|
mock.patch.object(drive_analysis.os, 'close', side_effect=tracking_close):
|
|
result = analyze_route(self.root, route, permitted=lambda: True)
|
|
self.assertTrue(result['complete'], result)
|
|
self.assertLessEqual(peak[0], 3)
|
|
self.assertFalse(opened)
|
|
self.assertEqual(result['segmentCount'], 129)
|
|
self.assertEqual(result['distanceMeters'], 77394)
|
|
self.assertEqual(result['durationSeconds'], 7739.4)
|
|
self.assertEqual(result['engagedSeconds'], 7739.4)
|
|
|
|
middle = self.root / segments[64]['segmentName']
|
|
original = (middle / 'qlog.zst').read_bytes()
|
|
(middle / 'qlog.zst').unlink()
|
|
middle.rmdir()
|
|
self.assertFalse(analyze_route(self.root, route, permitted=lambda: True)['complete'])
|
|
middle.mkdir()
|
|
(middle / 'qlog.zst').write_bytes(original)
|
|
(middle / 'qlog.lock').touch()
|
|
self.assertFalse(analyze_route(self.root, route, permitted=lambda: True)['complete'])
|
|
(middle / 'qlog.lock').unlink()
|
|
|
|
first_log = self.root / segments[0]['segmentName'] / 'qlog.zst'
|
|
decode = drive_analysis.decode_segment
|
|
changed = [False]
|
|
def changing_decode(*args, **kwargs):
|
|
if not changed[0]:
|
|
changed[0] = True
|
|
first_log.write_bytes(first_log.read_bytes())
|
|
return decode(*args, **kwargs)
|
|
with mock.patch.object(drive_analysis, 'decode_segment', side_effect=changing_decode):
|
|
result = analyze_route(self.root, route, permitted=lambda: True)
|
|
self.assertFalse(result['complete'])
|
|
self.assertIn('source changed', result['reason'].lower())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|