mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-10-27 08:33:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Keep CI fixtures current and partition complete RELEASE contracts

Browse Source
This commit is contained in:
firestar5683
2026-10-23 23:24:10 -05:00
parent 04c590f7c4
commit 6efe2e5379
7 changed files with 162 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tests.yaml
+52 -1
View File
@@ -395,7 +395,7 @@ jobs:
strategy:
fail-fast: false
matrix:
suite: [interfaces, safety-debug, safety-release]
suite: [interfaces, safety-debug]
env:
FUZZ_SEED: '0'
steps:
@@ -421,6 +421,57 @@ jobs:
path: ci-results/${{ matrix.suite }}/
if-no-files-found: error
safety_release_contracts:
name: RELEASE vehicle contracts (${{ matrix.shard }})
runs-on: ubuntu-24.04
timeout-minutes: 35
permissions:
contents: read
strategy:
fail-fast: false
matrix:
shard: [0, 1, 2, 3]
env:
FUZZ_SEED: '0'
steps:
- uses: actions/checkout@v7
- run: ./tools/op.sh setup
- name: Build native host dependencies
run: scons
- name: Run every assigned RELEASE target serially
timeout-minutes: 20
run: python3 tools/ci/run_vehicle_tests.py --suite safety-release --shard-index "${{ matrix.shard }}" --shard-count 4 --output "ci-results/safety-release-${{ matrix.shard }}"
- name: Save every RELEASE outcome and native build
if: always()
uses: actions/upload-artifact@v4
with:
name: safety-release-${{ matrix.shard }}
path: ci-results/safety-release-${{ matrix.shard }}/
if-no-files-found: error
safety_release_complete:
name: complete RELEASE vehicle catalog
needs: safety_release_contracts
if: ${{ always() && !cancelled() }}
runs-on: ubuntu-24.04
permissions:
contents: read
steps:
- uses: actions/checkout@v7
- uses: actions/download-artifact@v4
with:
pattern: safety-release-*
path: ci-results/safety-release-shards
- name: Require every RELEASE case exactly once and passed
run: python3 tools/ci/lifecycle_shards.py ci-results/safety-release-shards --suite safety-release --count 4 --output ci-results/safety-release-complete.json
- name: Save complete RELEASE verdict
if: always()
uses: actions/upload-artifact@v4
with:
name: safety-release-complete
path: ci-results/safety-release-complete.json
if-no-files-found: error
recorded_vehicles:
name: recorded vehicles (${{ matrix.variant }})
runs-on: ubuntu-24.04
openpilot/starpilot/car/toyota/tests/test_legacy_sdsu.py
+4 -2
View File
@@ -3,7 +3,7 @@ from opendbc.car import gen_empty_fingerprint, structs
from opendbc.car.toyota import legacy_sdsu
from opendbc.car.toyota.carstate import CarState
from opendbc.car.toyota.interface import CarInterface, apply_toyota_auto_hold
from opendbc.car.toyota.values import CAR, CarControllerParams, EPS_SCALE
from opendbc.car.toyota.values import CAR, CarControllerParams, EPS_SCALE, TOYOTA_AUTO_HOLD_CARS
from openpilot.starpilot.car.toyota.tests.test_pedal_caller_native import firmware
from openpilot.starpilot.car.toyota.aol import qualified
@@ -37,7 +37,9 @@ def test_full_configuration_and_tune_preserved(candidate, alpha, hybrid):
assert before == after
old_limits, new_limits = CarControllerParams(baseline), CarControllerParams(actual)
assert (old_limits.ACCEL_MIN, old_limits.ACCEL_MAX) == (new_limits.ACCEL_MIN, new_limits.ACCEL_MAX)
assert not apply_toyota_auto_hold(actual, True)
hold = actual.as_reader().as_builder()
assert apply_toyota_auto_hold(hold, True) == (candidate in TOYOTA_AUTO_HOLD_CARS)
assert actual.to_dict() == CarInterface.get_params(candidate, fp, fw, alpha, False, False).to_dict()
assert legacy_sdsu.configured(actual)
assert legacy_sdsu.prepare_stock(actual) and legacy_sdsu.configured(actual) and qualified(actual)
assert actual.safetyConfigs[0].safetyParam == legacy_sdsu.PROFILES[candidate][1]
openpilot/starpilot/flm/tests/test_tracking_source_context.py
+4 -4
View File
@@ -58,20 +58,20 @@ def test_legacy_point_precision_is_unchanged():
def test_global_thirty_coordinates_require_new_unselected_baseline(name):
o = owner()
kind = 'subtle' if name.startswith('center_') or name == 'highway_center_taper_max' else 'large'
count = 9 if kind == 'subtle' else 3
count = 3
discovery = set()
for i in range(2 * count):
e = episode(o, i, name, kind=kind, error=0.2)
e = replace(e, normalization_accel=max(0.1, e.goal_rms))
refresh(o, e.end_ns)
o.observe_episode(e, now_ns=e.end_ns)
if o.discovery is not None and not discovery:
discovery.update(o.selector_hint['source_episode_ids'])
if i < count:
discovery.add(e.episode_id)
if i == count - 1:
assert o.offer_state is None
selected = o.offer(e.end_ns)
assert selected is not None and selected.names == (name,) and len(o.before) == count
assert discovery.isdisjoint(e.episode_id for e in o.before)
assert discovery and discovery.isdisjoint(e.episode_id for e in o.before)
assert o.offer_state['alpha'] == 0.0 and o.offer_state['critical'] == 0.0
assert 'noise_prior' not in o.offer_state and len(selected.target) == 30 and not any(o.accepted.values())
changes = [n for n, v in selected.target if v != dict(selected.baseline)[n]]
prebuilt.json
+1 -1
View File
@@ -1 +1 @@
{"protected_inventory_sha256": "0a9d8f003500d6b80d3d0fa788ac3f029aa065a2310269ba2cff8b29059ee30c", "schema_version": 1}
{"protected_inventory_sha256": "068a87c1fcc37d4f6a4bdec4c063cbf0189a8e439230eb7bb4e4659b0a39653d", "schema_version": 1}
tools/ci/lifecycle_shards.py
+49 -7
View File
@@ -167,21 +167,59 @@ def partition(targets, identities, count):
return shards
def aggregate(directory, count):
def release_matches(identity, target):
if identity.startswith('opendbc.'):
identity = 'opendbc_repo.' + identity
module, *selectors = target.split('::')
if not module.endswith('.py') or module.startswith('/') or '..' in module.split('/'):
raise ValueError('Invalid RELEASE target')
prefix = module.removesuffix('.py').replace('/', '.')
if selectors:
prefix += '.' + '.'.join(selectors)
return identity == prefix or identity.startswith((prefix + '.', prefix + '['))
return identity.startswith(prefix + '.')
def release_partition(targets, identities, count):
"""Keep every declared module/class/function intact; balance by actual case count."""
if type(count) is not int or not 1 <= count <= 16 or len(set(targets)) != len(targets) or len(set(identities)) != len(identities):
raise ValueError('Invalid RELEASE catalog')
groups = {target: [] for target in targets}
for identity in identities:
owners = [target for target in targets if release_matches(identity, target)]
if len(owners) != 1:
raise ValueError('RELEASE case needs exactly one declared owner')
groups[owners[0]].append(identity)
if not groups or any(not cases for cases in groups.values()):
raise ValueError('RELEASE target collected no cases')
shards, loads = [[] for _ in range(count)], [0] * count
for target in sorted(groups, key=lambda target: (-len(groups[target]), target)):
index = min(range(count), key=lambda i: (loads[i], i))
shards[index].append(target)
loads[index] += len(groups[target])
return shards
def aggregate(directory, count, suite='aol-lifecycle'):
if suite not in ('aol-lifecycle', 'safety-release'):
raise ValueError('Invalid aggregate suite')
partitioner = release_partition if suite == 'safety-release' else partition
matcher = release_matches if suite == 'safety-release' else matches
native_kinds = ('safety',) if suite == 'safety-release' else ('safety', 'aol_protocol')
documents = [json.loads(path.read_text()) for path in sorted(Path(directory).rglob('coverage.json'))]
if len(documents) != count:
raise ValueError('Missing or duplicate lifecycle shard artifacts')
first = documents[0]
full = first['full_catalog_test_ids']
targets = first['full_catalog_targets']
expected = partition(targets, full, count)
expected = partitioner(targets, full, count)
seen, indices = [], set()
for doc in documents:
index = doc['shard']['index']
if index in indices or not 0 <= index < count or doc['shard']['count'] != count:
raise ValueError('Invalid lifecycle shard index')
indices.add(index)
if doc['suite'] != 'aol-lifecycle' or doc['status'] != 'completed' or doc['exit_code'] != 0:
if doc['suite'] != suite or doc['status'] != 'completed' or doc['exit_code'] != 0:
raise ValueError('Lifecycle shard failed')
if doc['collection_errors'] or doc['gate_errors']:
raise ValueError('Lifecycle shard errors')
@@ -206,14 +244,17 @@ def aggregate(directory, count):
capnp = doc['native'].get('pycapnp', {})
if capnp.get('sha256') != first['native'].get('pycapnp', {}).get('sha256') or len(capnp.get('sha256', '')) != 64:
raise ValueError('Native pycapnp import provenance changed')
for key in ('safety', 'aol_protocol'):
if suite == 'safety-release' and (doc['native']['safety'].get('variant') != 'release' or
doc['native']['safety'].get('all_output_hook_available') is not False):
raise ValueError('RELEASE shard must use the actual release registry')
for key in native_kinds:
if not doc['native'][key]['sha256']:
raise ValueError('Missing genuine native build evidence')
ids = doc['selected_test_ids']
coverage_path = next(path for path in Path(directory).rglob('coverage.json')
if json.loads(path.read_text())['shard']['index'] == index)
build_input_digest = digest(json.dumps(doc['source']['build_identity']['inputs'], sort_keys=True).encode())
for kind in ('safety', 'aol_protocol'):
for kind in native_kinds:
native = doc['native'][kind]
if native['build_inputs_sha256'] != build_input_digest:
raise ValueError('Native build input source provenance changed')
@@ -232,7 +273,7 @@ def aggregate(directory, count):
raise ValueError('Lifecycle outcome report failed')
if result['selected_test_ids'] != ids or Counter(row['id'] for row in records) != Counter(ids) or any(row['status'] != 'passed' for row in records):
raise ValueError('Lifecycle missing, duplicate or unsuccessful outcomes')
if ids != [identity for identity in full if any(matches(identity, target) for target in expected[index])]:
if ids != [identity for identity in full if any(matcher(identity, target) for target in expected[index])]:
raise ValueError('Lifecycle shard cases differ from the full catalog')
seen.extend(ids)
if Counter(seen) != Counter(full) or len(set(full)) != len(full):
@@ -244,9 +285,10 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('directory', type=Path)
parser.add_argument('--count', type=int, default=4)
parser.add_argument('--suite', choices=('aol-lifecycle', 'safety-release'), default='aol-lifecycle')
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
result = aggregate(args.directory, args.count)
result = aggregate(args.directory, args.count, args.suite)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2) + '\n')
print(json.dumps(result))
tools/ci/run_vehicle_tests.py
+12 -10
View File
@@ -716,7 +716,7 @@ def build_aol_protocol(output):
def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
if shard_index is not None and (suite != "aol-lifecycle" or not 0 <= shard_index < shard_count):
if shard_index is not None and (suite not in ("aol-lifecycle", "safety-release") or not 0 <= shard_index < shard_count):
raise ValueError("Lifecycle shard index is outside its catalog")
output = Path(output).resolve()
output.mkdir(parents=True, exist_ok=True)
@@ -731,7 +731,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
"independent lateral/longitudinal feature transitions", "device firmware execution"],
"targets": [], "selected_test_ids": [], "collection_errors": [],
"fuzz_seed": os.environ.get("FUZZ_SEED"), "max_examples_override": os.environ.get("MAX_EXAMPLES")}
if suite == "aol-lifecycle":
if suite in ("aol-lifecycle", "safety-release"):
plan["scope"] = "selected actual vehicle callers with synthetic receive/readiness timelines; native hooks where exercised by the selected suites"
plan["lifecycle_rows"] = AOL_LIFECYCLE_SCOPE
plan["uncovered"] = ["complete recorded routes and real-time device deadlines", "physical firmware/harness acceptance",
@@ -743,7 +743,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
try:
plan["targets"] = targets_for(suite)
plan["source"] = source_provenance()
if suite == "aol-lifecycle":
if suite in ("aol-lifecycle", "safety-release"):
plan["source"]["build_identity"] = lifecycle_shards.build_identity(ROOT)
plan.update(vehicle_inventory())
plan["native"] = native_provenance()
@@ -751,9 +751,9 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
plan["native"]["safety"] = build_safety_library(suite == "safety-release", output)
if suite == "aol-lifecycle" and not collect_only:
plan["native"]["aol_protocol"] = build_aol_protocol(output)
if suite == "aol-lifecycle":
if suite in ("aol-lifecycle", "safety-release"):
build_input_digest = hashlib.sha256(json.dumps(plan["source"]["build_identity"]["inputs"], sort_keys=True).encode()).hexdigest()
for name in ('safety', 'aol_protocol'):
for name in (('safety',) if suite == 'safety-release' else ('safety', 'aol_protocol')):
if name in plan["native"]:
plan["native"][name]["build_inputs_sha256"] = build_input_digest
phase = "collection"
@@ -765,13 +765,15 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
ids = [test.id() for test in tests] + [vehicle_pytest.test_id(nodeid) for nodeid in pytest_nodeids]
if shard_index is not None and not errors:
full_targets = list(plan["targets"])
shards = lifecycle_shards.partition(full_targets, ids, shard_count)
partitioner = lifecycle_shards.release_partition if suite == 'safety-release' else lifecycle_shards.partition
matcher = lifecycle_shards.release_matches if suite == 'safety-release' else lifecycle_shards.matches
shards = partitioner(full_targets, ids, shard_count)
plan.update(full_catalog_targets=full_targets, full_catalog_test_ids=list(ids),
shard={"index": shard_index, "count": shard_count})
assigned_targets = shards[shard_index]
plan["targets"] = assigned_targets
def assigned(identity):
return any(lifecycle_shards.matches(identity, target) for target in assigned_targets)
return any(matcher(identity, target) for target in assigned_targets)
tests = [test for test in tests if assigned(test.id())]
pytest_nodeids = [nodeid for nodeid in pytest_nodeids if assigned(vehicle_pytest.test_id(nodeid))]
ids = [test.id() for test in tests] + [vehicle_pytest.test_id(nodeid) for nodeid in pytest_nodeids]
@@ -797,7 +799,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
errors.extend(pytest_errors)
phase = "source verification"
plan["source_after"] = source_provenance()
if suite == "aol-lifecycle":
if suite in ("aol-lifecycle", "safety-release"):
plan["source_after"]["build_identity"] = lifecycle_shards.build_identity(ROOT)
if plan["source"]["build_identity"] != plan["source_after"]["build_identity"]:
gate_errors.append("Lifecycle source/build identity changed during the suite")
@@ -810,11 +812,11 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
records = test_runner.account_for_tests(ids, records, tests)
if not errors and suite == "interfaces":
gate_errors.extend(interface_execution_errors(platform_tests, records))
if not collect_only and suite == "aol-lifecycle":
if not collect_only and suite in ("aol-lifecycle", "safety-release"):
gate_errors.extend(f"Required lifecycle case did not pass: {record['id']} ({record['status']})"
for record in records if record["status"] != "passed")
for target in plan["targets"]:
if not any(lifecycle_shards.matches(identity, target) for identity in ids):
if not any((lifecycle_shards.release_matches if suite == 'safety-release' else lifecycle_shards.matches)(identity, target) for identity in ids):
gate_errors.append(f"Required lifecycle module collected no cases: {target}")
elapsed = time.monotonic() - started
exit_code = test_runner.report(records, errors + gate_errors, 10, elapsed)
tools/ci/tests/test_lifecycle_shards.py
+40 -6
View File
@@ -109,9 +109,11 @@ class LifecycleShardTests(unittest.TestCase):
camera = 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py'
self.assertFalse(any(camera in target for shard in shards for target in shard))
def documents(self):
def documents(self, suite="aol-lifecycle"):
targets, full = self.catalog()
partition = helper.partition(targets, full, 4)
partitioner = helper.release_partition if suite == "safety-release" else helper.partition
matcher = helper.release_matches if suite == "safety-release" else helper.matches
partition = partitioner(targets, full, 4)
source: dict = dict.fromkeys(('revision', 'inputs', 'manifest_sha256', 'dependencies', 'runner_sha256',
'suite_runner_sha256', 'pytest_runner_sha256', 'shard_runner_sha256'), 'same')
inventory_bytes = Path(__file__).parents[1].joinpath('lifecycle_build_outputs.json').read_bytes()
@@ -125,11 +127,12 @@ class LifecycleShardTests(unittest.TestCase):
for name, checksum in can_sources.items()})
docs = []
for index, assigned in enumerate(partition):
ids = [identity for identity in full if any(helper.matches(identity, target) for target in assigned)]
doc = {'suite': 'aol-lifecycle', 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4},
ids = [identity for identity in full if any(matcher(identity, target) for target in assigned)]
doc = {'suite': suite, 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4},
'targets': assigned, 'selected_test_ids': ids, 'full_catalog_targets': targets, 'full_catalog_test_ids': full,
'collection_errors': [], 'gate_errors': [], 'source': source, 'source_after': source,
'native': {'can_sources': can_sources, 'pycapnp': {'sha256': 'b' * 64}, 'safety': {'sha256': '1' * 64}, 'aol_protocol': {'sha256': '2' * 64}}}
'native': {'can_sources': can_sources, 'pycapnp': {'sha256': 'b' * 64}, 'safety': {'sha256': '1' * 64,
'variant': 'release', 'all_output_hook_available': False}, 'aol_protocol': {'sha256': '2' * 64}}}
result = {'selected_test_ids': ids, 'exit_code': 0, 'collection_errors': [], 'gate_errors': [],
'results': [{'id': identity, 'status': 'passed'} for identity in ids]}
docs.append((doc, result))
@@ -154,7 +157,38 @@ class LifecycleShardTests(unittest.TestCase):
(path / 'coverage.json').write_text(json.dumps(doc))
if artifact_mutation is not None:
artifact_mutation(Path(directory))
return helper.aggregate(directory, 4)
return helper.aggregate(directory, 4, docs[0][0]["suite"])
def test_release_partition_accepts_real_class_selectors_and_import_aliases(self):
targets = ['opendbc_repo/opendbc/safety/tests/a.py::TestRelease', 'b.py::test_case[one]', 'c.py']
ids = ['opendbc.safety.tests.a.TestRelease.test_denied', 'b.test_case[one]', 'c.Test.test_one']
shards = helper.release_partition(targets, ids, 4)
self.assertCountEqual([target for shard in shards for target in shard], targets)
self.assertEqual(sum(sum(any(helper.release_matches(identity, target) for target in shard) for identity in ids) for shard in shards), len(ids))
for bad_targets, bad_ids in ((targets, ids + ids), (targets + ['c.py::Test'], ids), (targets, ids + ['foreign.test']),
(targets + ['empty.py'], ids), (['../escape.py'], ['escape.test'])):
with self.subTest(targets=bad_targets), self.assertRaises(ValueError):
helper.release_partition(bad_targets, bad_ids, 4)
def test_release_aggregate_retains_all_outcome_and_native_guards(self):
docs = self.documents('safety-release')
self.assertEqual(self.check(copy.deepcopy(docs))['cases_passed'], 12)
for change in ('missing', 'duplicate', 'skip', 'debug', 'allOutput', 'wrong_suite'):
bad = json.loads(json.dumps(docs))
if change == 'missing':
bad[0][1]['results'].pop()
elif change == 'duplicate':
bad[0][1]['results'].append(bad[0][1]['results'][0])
elif change == 'skip':
bad[0][1]['results'][0]['status'] = 'skipped'
elif change == 'debug':
bad[0][0]['native']['safety']['variant'] = 'ALLOW_DEBUG'
elif change == 'allOutput':
bad[0][0]['native']['safety']['all_output_hook_available'] = True
else:
bad[0][0]['suite'] = 'interfaces'
with self.subTest(change=change), self.assertRaises((ValueError, KeyError, StopIteration)):
self.check(bad)
def test_complete_catalog_passes(self):
self.assertEqual(self.check(self.documents())['cases_passed'], 12)