mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-11-06 12:33:51 +08:00
Dom
298 lines
17 KiB
Python
298 lines
17 KiB
Python
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
spec = importlib.util.spec_from_file_location('private_lifecycle_shards', Path(__file__).parents[1] / 'lifecycle_shards.py')
|
|
assert spec is not None and spec.loader is not None
|
|
helper = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(helper)
|
|
|
|
|
|
class LifecycleShardTests(unittest.TestCase):
|
|
def catalog(self):
|
|
targets = ['a.py', 'b.py', 'c.py', 'd.py']
|
|
identities = [f'{name}.Test.test_{i}' for name in 'abcd' for i in range(3)]
|
|
return targets, identities
|
|
|
|
def test_partition_preserves_whole_modules_and_all_cases(self):
|
|
targets, identities = self.catalog()
|
|
shards = helper.partition(targets, identities, 4)
|
|
self.assertEqual(sorted(target for shard in shards for target in shard), targets)
|
|
self.assertEqual([len(shard) for shard in shards], [1] * 4)
|
|
self.assertEqual(shards, helper.partition(list(reversed(targets)), identities, 4))
|
|
|
|
def test_invalid_catalogs_are_rejected(self):
|
|
targets, identities = self.catalog()
|
|
for bad_targets, bad_ids, count in [(targets, identities, 0), (targets + targets, identities, 4),
|
|
(targets, identities + identities, 4), (targets, ['foreign.test'], 4),
|
|
(targets + ['empty.py'], identities, 4)]:
|
|
with self.subTest(targets=bad_targets, count=count), self.assertRaises(ValueError):
|
|
helper.partition(bad_targets, bad_ids, count)
|
|
|
|
def test_measured_native_callers_keep_whole_module_and_outweigh_case_count(self):
|
|
module = 'opendbc_repo/opendbc/car/mazda/tests/test_native_engagement_callers.py'
|
|
policy = helper.workload_policy()
|
|
measured = policy['modules'][module]
|
|
self.assertNotIn(module, policy['function_groups'])
|
|
self.assertEqual(measured['cases'], 48)
|
|
self.assertGreater(measured['seconds'], 56 * policy['unknown_seconds_per_case'])
|
|
self.assertEqual(len(measured['source_sha256']), 64)
|
|
self.assertEqual(measured['reference']['artifact_id'], 12520491388)
|
|
prefix = module.removesuffix('.py').replace('/', '.') + '.'
|
|
identities = [prefix + f'test_case[{i}]' for i in range(48)]
|
|
identities += [f'other.test_case[{i}]' for i in range(56)]
|
|
shards = helper.partition([module, 'other.py'], identities, 2)
|
|
self.assertEqual(shards, [[module], ['other.py']])
|
|
selected = [identity for shard in shards for identity in identities
|
|
if any(helper.matches(identity, target) for target in shard)]
|
|
self.assertCountEqual(selected, identities)
|
|
|
|
def test_measured_pedal_callers_keep_complete_module(self):
|
|
module = 'openpilot/starpilot/car/toyota/tests/test_pedal_parent_callers.py'
|
|
policy = helper.workload_policy()
|
|
measured = policy['modules'][module]
|
|
self.assertNotIn(module, policy['function_groups'])
|
|
self.assertEqual(measured['cases'], 92)
|
|
self.assertGreater(measured['seconds'], 100 * policy['unknown_seconds_per_case'])
|
|
self.assertEqual(len(measured['source_sha256']), 64)
|
|
self.assertEqual(measured['reference']['artifact_id'], 12574163616)
|
|
prefix = module.removesuffix('.py').replace('/', '.') + '.'
|
|
identities = [prefix + f'test_case[{index}]' for index in range(92)]
|
|
identities += [f'other.test_case[{index}]' for index in range(100)]
|
|
targets = [module, 'other.py']
|
|
shards = helper.partition(targets, identities, 2)
|
|
self.assertEqual(shards, [[module], ['other.py']])
|
|
self.assertCountEqual([target for shard in shards for target in shard], targets)
|
|
selected = [identity for shard in shards for identity in identities
|
|
if any(helper.matches(identity, target) for target in shard)]
|
|
self.assertCountEqual(selected, identities)
|
|
self.assertEqual(shards, helper.partition(list(reversed(targets)), list(reversed(identities)), 2))
|
|
|
|
def test_only_large_reviewed_heavy_functions_split_into_exact_parameter_leaves(self):
|
|
heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'
|
|
prefix = heavy.removesuffix('.py').replace('/', '.') + '.'
|
|
full = [prefix + f'test_nominal[{index}]' for index in range(126)]
|
|
full += [prefix + f'test_veto[{index}]' for index in range(32)]
|
|
full += ['other.Test.test_one', 'other.Test.test_two']
|
|
targets = [heavy, 'other.py']
|
|
shards = helper.partition(targets, full, 4)
|
|
selectors = [target for shard in shards for target in shard]
|
|
self.assertIn(heavy + '::test_veto', selectors)
|
|
self.assertIn('other.py', selectors)
|
|
self.assertEqual(len(selectors), 128)
|
|
self.assertTrue(all(heavy + f'::test_nominal[{index}]' in selectors for index in range(126)))
|
|
selected = [[identity for identity in full if any(helper.matches(identity, target) for target in shard)] for shard in shards]
|
|
self.assertEqual(sorted(identity for shard in selected for identity in shard), sorted(full))
|
|
self.assertEqual(shards, helper.partition(list(reversed(targets)), list(reversed(full)), 4))
|
|
self.assertFalse(helper.matches(prefix + 'test_nominal_extra[1]', heavy + '::test_nominal'))
|
|
self.assertFalse(helper.matches('foreign.test_nominal[1]', heavy + '::test_nominal'))
|
|
self.assertTrue(helper.matches(prefix + 'test_nominal[1]', heavy + '::test_nominal[1]'))
|
|
self.assertFalse(helper.matches(prefix + 'test_nominal[10]', heavy + '::test_nominal[1]'))
|
|
self.assertTrue(helper.matches(prefix + 'test_nominal[::complex/parameter]', heavy + '::test_nominal[::complex/parameter]'))
|
|
|
|
def test_current_ford_and_stock_location_cases_split_without_omission(self):
|
|
for module, size in (('openpilot/starpilot/tests/test_ford_configuration_parity.py', 112),
|
|
('openpilot/starpilot/aol/tests/test_stock_location_recovery.py', 135)):
|
|
with self.subTest(module=module):
|
|
prefix = module.removesuffix('.py').replace('/', '.') + '.'
|
|
full = [prefix + f'test_actual_lifecycle[{index}]' for index in range(size)]
|
|
full += [prefix + 'test_neighbor']
|
|
shards = helper.partition([module], full, 8)
|
|
selected = [[identity for identity in full if any(helper.matches(identity, target) for target in shard)]
|
|
for shard in shards]
|
|
self.assertEqual(sorted(identity for shard in selected for identity in shard), sorted(full))
|
|
self.assertEqual(len({identity for shard in selected for identity in shard}), len(full))
|
|
self.assertTrue(all(len(shard) >= 1 for shard in selected))
|
|
selectors = [target for shard in shards for target in shard]
|
|
self.assertEqual(len(selectors), len(full))
|
|
self.assertEqual(shards, helper.partition([module], list(reversed(full)), 8))
|
|
|
|
def test_heavy_function_rule_never_splits_classes_or_accepts_forged_targets(self):
|
|
heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'
|
|
prefix = heavy.removesuffix('.py').replace('/', '.') + '.'
|
|
for targets, ids in (([heavy], [prefix + 'TestClass.test_case']),
|
|
([heavy + '::test_case'], [prefix + 'test_case']),
|
|
([heavy], [prefix + 'test_case', prefix + 'test_case'])):
|
|
with self.subTest(targets=targets), self.assertRaises(ValueError):
|
|
helper.partition(targets, ids, 4)
|
|
with self.assertRaises(ValueError):
|
|
helper.partition(['other.py'], ['other.test_case'], 17)
|
|
|
|
def test_future_declared_module_does_not_add_any_cases(self):
|
|
targets, identities = self.catalog()
|
|
shards = helper.partition(targets, identities, 8)
|
|
self.assertEqual(sorted(target for shard in shards for target in shard), targets)
|
|
camera = 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py'
|
|
self.assertFalse(any(camera in target for shard in shards for target in shard))
|
|
|
|
def documents(self, suite="aol-lifecycle"):
|
|
targets, full = self.catalog()
|
|
partitioner = helper.release_partition if suite == "safety-release" else helper.partition
|
|
matcher = helper.release_matches if suite == "safety-release" else helper.matches
|
|
partition = partitioner(targets, full, 4)
|
|
source: dict = dict.fromkeys(('revision', 'inputs', 'manifest_sha256', 'dependencies', 'runner_sha256',
|
|
'suite_runner_sha256', 'pytest_runner_sha256', 'shard_runner_sha256'), 'same')
|
|
inventory_bytes = Path(__file__).parents[1].joinpath('lifecycle_build_outputs.json').read_bytes()
|
|
inventory = json.loads(inventory_bytes)
|
|
source['build_identity'] = {'inputs': {'revision': 'same', 'tree': 'same', 'tracked_inputs': {'source.c': {'sha256': 'c' * 64, 'mode': 420}},
|
|
'changes': {}, 'untracked': {}, 'recipes': {'SConstruct': {'sha256': 'd' * 64, 'mode': 420}},
|
|
'output_inventory_sha256': hashlib.sha256(inventory_bytes).hexdigest()},
|
|
'declared_outputs': {name: {'sha256': 'a' * 64, 'mode': 420} for name in inventory}}
|
|
can_sources = dict.fromkeys(('opendbc_repo/opendbc/can/packer.py', 'opendbc_repo/opendbc/can/parser.py'), 'e' * 64)
|
|
source['build_identity']['inputs']['tracked_inputs'].update({name: {'sha256': checksum, 'mode': 420}
|
|
for name, checksum in can_sources.items()})
|
|
docs = []
|
|
for index, assigned in enumerate(partition):
|
|
ids = [identity for identity in full if any(matcher(identity, target) for target in assigned)]
|
|
doc = {'suite': suite, 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4},
|
|
'targets': assigned, 'selected_test_ids': ids, 'full_catalog_targets': targets, 'full_catalog_test_ids': full,
|
|
'collection_errors': [], 'gate_errors': [], 'source': source, 'source_after': source,
|
|
'native': {'can_sources': can_sources, 'pycapnp': {'sha256': 'b' * 64}, 'safety': {'sha256': '1' * 64,
|
|
'variant': 'release', 'all_output_hook_available': False}, 'aol_protocol': {'sha256': '2' * 64}}}
|
|
result = {'selected_test_ids': ids, 'exit_code': 0, 'collection_errors': [], 'gate_errors': [],
|
|
'results': [{'id': identity, 'status': 'passed'} for identity in ids]}
|
|
docs.append((doc, result))
|
|
return docs
|
|
|
|
def check(self, docs, artifact_mutation=None):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
for index, (doc, result) in enumerate(docs):
|
|
path = Path(directory) / str(index)
|
|
path.mkdir()
|
|
digest = hashlib.sha256(json.dumps(doc['source']['build_identity']['inputs'], sort_keys=True).encode()).hexdigest()
|
|
for kind in ('safety', 'aol_protocol'):
|
|
native = doc['native'][kind]
|
|
name = 'libsafety-debug.so' if kind == 'safety' else 'test_aol_protocol'
|
|
artifact = path / name
|
|
artifact.write_bytes(kind.encode())
|
|
native.update(path=str(artifact), sha256=hashlib.sha256(artifact.read_bytes()).hexdigest(),
|
|
build_inputs_sha256=digest, commands=[['cc', 'actual.c']] if kind == 'safety' else None,
|
|
command=['c++', 'actual.cc'])
|
|
(path / 'results.json').write_text(json.dumps(result))
|
|
doc['result_sha256'] = hashlib.sha256((path / 'results.json').read_bytes()).hexdigest()
|
|
(path / 'coverage.json').write_text(json.dumps(doc))
|
|
if artifact_mutation is not None:
|
|
artifact_mutation(Path(directory))
|
|
return helper.aggregate(directory, 4, docs[0][0]["suite"])
|
|
|
|
def test_release_partition_accepts_real_class_selectors_and_import_aliases(self):
|
|
targets = ['opendbc_repo/opendbc/safety/tests/a.py::TestRelease', 'b.py::test_case[one]', 'c.py']
|
|
ids = ['opendbc.safety.tests.a.TestRelease.test_denied', 'b.test_case[one]', 'c.Test.test_one']
|
|
shards = helper.release_partition(targets, ids, 4)
|
|
self.assertCountEqual([target for shard in shards for target in shard], targets)
|
|
self.assertEqual(sum(sum(any(helper.release_matches(identity, target) for target in shard) for identity in ids) for shard in shards), len(ids))
|
|
for bad_targets, bad_ids in ((targets, ids + ids), (targets + ['c.py::Test'], ids), (targets, ids + ['foreign.test']),
|
|
(targets + ['empty.py'], ids), (['../escape.py'], ['escape.test'])):
|
|
with self.subTest(targets=bad_targets), self.assertRaises(ValueError):
|
|
helper.release_partition(bad_targets, bad_ids, 4)
|
|
|
|
def test_release_aggregate_retains_all_outcome_and_native_guards(self):
|
|
docs = self.documents('safety-release')
|
|
self.assertEqual(self.check(copy.deepcopy(docs))['cases_passed'], 12)
|
|
for change in ('missing', 'duplicate', 'debug', 'allOutput', 'wrong_suite'):
|
|
bad = json.loads(json.dumps(docs))
|
|
if change == 'missing':
|
|
bad[0][1]['results'].pop()
|
|
elif change == 'duplicate':
|
|
bad[0][1]['results'].append(bad[0][1]['results'][0])
|
|
elif change == 'skip':
|
|
bad[0][1]['results'][0]['status'] = 'skipped'
|
|
elif change == 'debug':
|
|
bad[0][0]['native']['safety']['variant'] = 'ALLOW_DEBUG'
|
|
elif change == 'allOutput':
|
|
bad[0][0]['native']['safety']['all_output_hook_available'] = True
|
|
else:
|
|
bad[0][0]['suite'] = 'interfaces'
|
|
with self.subTest(change=change), self.assertRaises((ValueError, KeyError, StopIteration)):
|
|
self.check(bad)
|
|
|
|
def test_release_preserves_existing_skips_without_counting_them_as_passed(self):
|
|
docs = self.documents('safety-release')
|
|
docs[0][1]['results'][0].update(status='skipped', detail='DEBUG classic Bosch owner')
|
|
verdict = self.check(copy.deepcopy(docs))
|
|
self.assertEqual(verdict['cases_passed'], 11)
|
|
self.assertEqual(verdict['cases_skipped'], 1)
|
|
self.assertEqual(verdict['cases_total'], 12)
|
|
for status in ('failed', 'error', 'xfailed', 'xpassed', 'not_run', 'timed_out'):
|
|
bad = copy.deepcopy(docs)
|
|
bad[0][1]['results'][0]['status'] = status
|
|
with self.subTest(status=status), self.assertRaises(ValueError):
|
|
self.check(bad)
|
|
for doc, _ in docs:
|
|
doc['suite'] = 'aol-lifecycle'
|
|
with self.assertRaises(ValueError):
|
|
self.check(docs)
|
|
|
|
def test_complete_catalog_passes(self):
|
|
self.assertEqual(self.check(self.documents())['cases_passed'], 12)
|
|
|
|
def test_parameter_leaf_aggregate_requires_exact_complete_passed_outcomes(self):
|
|
heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'
|
|
prefix = heavy.removesuffix('.py').replace('/', '.') + '.'
|
|
full = [prefix + f'test_nominal[{index}]' for index in range(126)] + ['other.Test.test_case']
|
|
with patch.object(self, 'catalog', return_value=([heavy, 'other.py'], full)):
|
|
docs = self.documents()
|
|
self.assertEqual(self.check(copy.deepcopy(docs))['cases_passed'], 127)
|
|
for change in ('missing', 'duplicate', 'skip', 'forged_catalog', 'forged_leaf'):
|
|
bad = json.loads(json.dumps(docs))
|
|
if change == 'missing':
|
|
bad[0][1]['results'].pop()
|
|
elif change == 'duplicate':
|
|
bad[0][1]['results'].append(bad[0][1]['results'][0])
|
|
elif change == 'skip':
|
|
bad[0][1]['results'][0]['status'] = 'skipped'
|
|
elif change == 'forged_catalog':
|
|
bad[0][0]['full_catalog_test_ids'].pop()
|
|
else:
|
|
bad[0][0]['targets'][0] += '-forged'
|
|
with self.subTest(change=change), self.assertRaises((ValueError, KeyError, StopIteration)):
|
|
self.check(bad)
|
|
|
|
def test_missing_modified_or_forged_native_artifacts_are_denied(self):
|
|
def remove(root):
|
|
(root / '0/libsafety-debug.so').unlink()
|
|
def modify(root):
|
|
(root / '0/test_aol_protocol').write_bytes(b'foreign binary')
|
|
def forge(root):
|
|
path = root / '0/coverage.json'
|
|
data = json.loads(path.read_text())
|
|
data['native']['safety']['build_inputs_sha256'] = 'f' * 64
|
|
path.write_text(json.dumps(data))
|
|
for mutate in (remove, modify, forge):
|
|
with self.subTest(mutation=mutate.__name__), self.assertRaises(ValueError):
|
|
self.check(self.documents(), mutate)
|
|
|
|
def test_bad_outcomes_and_provenance_are_denied(self):
|
|
mutations = [lambda d: d.pop(),
|
|
lambda d: d[0][0]['shard'].update(index=1),
|
|
lambda d: d[0][0].update(exit_code=1),
|
|
lambda d: d[0][0].update(gate_errors=['failed gate']),
|
|
lambda d: d[0][0]['full_catalog_test_ids'].pop(),
|
|
lambda d: d[0][0].update(targets=['foreign.py']),
|
|
lambda d: d[0][0]['source'].update(revision='foreign'),
|
|
lambda d: d[0][0]['source_after'].update(inputs='changed'),
|
|
lambda d: d[0][0]['source']['build_identity']['declared_outputs'].clear(),
|
|
lambda d: d[0][0]['native']['can_sources'].clear(),
|
|
lambda d: d[0][0]['native']['pycapnp'].update(sha256='f' * 64),
|
|
lambda d: d[0][1]['results'].pop(),
|
|
lambda d: d[0][1]['results'].append(d[0][1]['results'][0]),
|
|
lambda d: d[0][1]['results'][0].update(status='skipped'),
|
|
lambda d: d[0][1]['results'][0].update(status='failed'),
|
|
lambda d: d[0][1].update(exit_code=1)]
|
|
for index, mutate in enumerate(mutations):
|
|
docs = copy.deepcopy(self.documents())
|
|
# Break intentional shared catalog/source aliases to exercise genuine cross-shard disagreement.
|
|
docs = json.loads(json.dumps(docs))
|
|
mutate(docs)
|
|
with self.subTest(index=index), self.assertRaises((ValueError, KeyError, StopIteration)):
|
|
self.check(docs)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|