diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8d4a3bf76b..6541d393c5 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -395,7 +395,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [interfaces, safety-debug, safety-release] + suite: [interfaces, safety-debug] env: FUZZ_SEED: '0' steps: @@ -421,6 +421,57 @@ jobs: path: ci-results/${{ matrix.suite }}/ if-no-files-found: error + safety_release_contracts: + name: RELEASE vehicle contracts (${{ matrix.shard }}) + runs-on: ubuntu-24.04 + timeout-minutes: 35 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3] + env: + FUZZ_SEED: '0' + steps: + - uses: actions/checkout@v7 + - run: ./tools/op.sh setup + - name: Build native host dependencies + run: scons + - name: Run every assigned RELEASE target serially + timeout-minutes: 20 + run: python3 tools/ci/run_vehicle_tests.py --suite safety-release --shard-index "${{ matrix.shard }}" --shard-count 4 --output "ci-results/safety-release-${{ matrix.shard }}" + - name: Save every RELEASE outcome and native build + if: always() + uses: actions/upload-artifact@v4 + with: + name: safety-release-${{ matrix.shard }} + path: ci-results/safety-release-${{ matrix.shard }}/ + if-no-files-found: error + + safety_release_complete: + name: complete RELEASE vehicle catalog + needs: safety_release_contracts + if: ${{ always() && !cancelled() }} + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v4 + with: + pattern: safety-release-* + path: ci-results/safety-release-shards + - name: Require every RELEASE case exactly once and passed + run: python3 tools/ci/lifecycle_shards.py ci-results/safety-release-shards --suite safety-release --count 4 --output ci-results/safety-release-complete.json + - name: Save complete RELEASE verdict + if: always() + uses: actions/upload-artifact@v4 + with: + name: safety-release-complete + path: ci-results/safety-release-complete.json + if-no-files-found: error + recorded_vehicles: name: recorded vehicles (${{ matrix.variant }}) runs-on: ubuntu-24.04 diff --git a/openpilot/starpilot/car/toyota/tests/test_legacy_sdsu.py b/openpilot/starpilot/car/toyota/tests/test_legacy_sdsu.py index 3685f51bde..a9a97d602f 100644 --- a/openpilot/starpilot/car/toyota/tests/test_legacy_sdsu.py +++ b/openpilot/starpilot/car/toyota/tests/test_legacy_sdsu.py @@ -3,7 +3,7 @@ from opendbc.car import gen_empty_fingerprint, structs from opendbc.car.toyota import legacy_sdsu from opendbc.car.toyota.carstate import CarState from opendbc.car.toyota.interface import CarInterface, apply_toyota_auto_hold -from opendbc.car.toyota.values import CAR, CarControllerParams, EPS_SCALE +from opendbc.car.toyota.values import CAR, CarControllerParams, EPS_SCALE, TOYOTA_AUTO_HOLD_CARS from openpilot.starpilot.car.toyota.tests.test_pedal_caller_native import firmware from openpilot.starpilot.car.toyota.aol import qualified @@ -37,7 +37,9 @@ def test_full_configuration_and_tune_preserved(candidate, alpha, hybrid): assert before == after old_limits, new_limits = CarControllerParams(baseline), CarControllerParams(actual) assert (old_limits.ACCEL_MIN, old_limits.ACCEL_MAX) == (new_limits.ACCEL_MIN, new_limits.ACCEL_MAX) - assert not apply_toyota_auto_hold(actual, True) + hold = actual.as_reader().as_builder() + assert apply_toyota_auto_hold(hold, True) == (candidate in TOYOTA_AUTO_HOLD_CARS) + assert actual.to_dict() == CarInterface.get_params(candidate, fp, fw, alpha, False, False).to_dict() assert legacy_sdsu.configured(actual) assert legacy_sdsu.prepare_stock(actual) and legacy_sdsu.configured(actual) and qualified(actual) assert actual.safetyConfigs[0].safetyParam == legacy_sdsu.PROFILES[candidate][1] diff --git a/openpilot/starpilot/flm/tests/test_tracking_source_context.py b/openpilot/starpilot/flm/tests/test_tracking_source_context.py index 22a3596aab..a31942b38b 100644 --- a/openpilot/starpilot/flm/tests/test_tracking_source_context.py +++ b/openpilot/starpilot/flm/tests/test_tracking_source_context.py @@ -58,20 +58,20 @@ def test_legacy_point_precision_is_unchanged(): def test_global_thirty_coordinates_require_new_unselected_baseline(name): o = owner() kind = 'subtle' if name.startswith('center_') or name == 'highway_center_taper_max' else 'large' - count = 9 if kind == 'subtle' else 3 + count = 3 discovery = set() for i in range(2 * count): e = episode(o, i, name, kind=kind, error=0.2) e = replace(e, normalization_accel=max(0.1, e.goal_rms)) refresh(o, e.end_ns) o.observe_episode(e, now_ns=e.end_ns) + if o.discovery is not None and not discovery: + discovery.update(o.selector_hint['source_episode_ids']) if i < count: - discovery.add(e.episode_id) - if i == count - 1: assert o.offer_state is None selected = o.offer(e.end_ns) assert selected is not None and selected.names == (name,) and len(o.before) == count - assert discovery.isdisjoint(e.episode_id for e in o.before) + assert discovery and discovery.isdisjoint(e.episode_id for e in o.before) assert o.offer_state['alpha'] == 0.0 and o.offer_state['critical'] == 0.0 assert 'noise_prior' not in o.offer_state and len(selected.target) == 30 and not any(o.accepted.values()) changes = [n for n, v in selected.target if v != dict(selected.baseline)[n]] diff --git a/prebuilt.json b/prebuilt.json index fdc6b33450..bb8bf9a2f3 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "0a9d8f003500d6b80d3d0fa788ac3f029aa065a2310269ba2cff8b29059ee30c", "schema_version": 1} +{"protected_inventory_sha256": "068a87c1fcc37d4f6a4bdec4c063cbf0189a8e439230eb7bb4e4659b0a39653d", "schema_version": 1} diff --git a/tools/ci/lifecycle_shards.py b/tools/ci/lifecycle_shards.py index 92e174902a..a24b697c6d 100644 --- a/tools/ci/lifecycle_shards.py +++ b/tools/ci/lifecycle_shards.py @@ -167,21 +167,59 @@ def partition(targets, identities, count): return shards -def aggregate(directory, count): +def release_matches(identity, target): + if identity.startswith('opendbc.'): + identity = 'opendbc_repo.' + identity + module, *selectors = target.split('::') + if not module.endswith('.py') or module.startswith('/') or '..' in module.split('/'): + raise ValueError('Invalid RELEASE target') + prefix = module.removesuffix('.py').replace('/', '.') + if selectors: + prefix += '.' + '.'.join(selectors) + return identity == prefix or identity.startswith((prefix + '.', prefix + '[')) + return identity.startswith(prefix + '.') + + +def release_partition(targets, identities, count): + """Keep every declared module/class/function intact; balance by actual case count.""" + if type(count) is not int or not 1 <= count <= 16 or len(set(targets)) != len(targets) or len(set(identities)) != len(identities): + raise ValueError('Invalid RELEASE catalog') + groups = {target: [] for target in targets} + for identity in identities: + owners = [target for target in targets if release_matches(identity, target)] + if len(owners) != 1: + raise ValueError('RELEASE case needs exactly one declared owner') + groups[owners[0]].append(identity) + if not groups or any(not cases for cases in groups.values()): + raise ValueError('RELEASE target collected no cases') + shards, loads = [[] for _ in range(count)], [0] * count + for target in sorted(groups, key=lambda target: (-len(groups[target]), target)): + index = min(range(count), key=lambda i: (loads[i], i)) + shards[index].append(target) + loads[index] += len(groups[target]) + return shards + + +def aggregate(directory, count, suite='aol-lifecycle'): + if suite not in ('aol-lifecycle', 'safety-release'): + raise ValueError('Invalid aggregate suite') + partitioner = release_partition if suite == 'safety-release' else partition + matcher = release_matches if suite == 'safety-release' else matches + native_kinds = ('safety',) if suite == 'safety-release' else ('safety', 'aol_protocol') documents = [json.loads(path.read_text()) for path in sorted(Path(directory).rglob('coverage.json'))] if len(documents) != count: raise ValueError('Missing or duplicate lifecycle shard artifacts') first = documents[0] full = first['full_catalog_test_ids'] targets = first['full_catalog_targets'] - expected = partition(targets, full, count) + expected = partitioner(targets, full, count) seen, indices = [], set() for doc in documents: index = doc['shard']['index'] if index in indices or not 0 <= index < count or doc['shard']['count'] != count: raise ValueError('Invalid lifecycle shard index') indices.add(index) - if doc['suite'] != 'aol-lifecycle' or doc['status'] != 'completed' or doc['exit_code'] != 0: + if doc['suite'] != suite or doc['status'] != 'completed' or doc['exit_code'] != 0: raise ValueError('Lifecycle shard failed') if doc['collection_errors'] or doc['gate_errors']: raise ValueError('Lifecycle shard errors') @@ -206,14 +244,17 @@ def aggregate(directory, count): capnp = doc['native'].get('pycapnp', {}) if capnp.get('sha256') != first['native'].get('pycapnp', {}).get('sha256') or len(capnp.get('sha256', '')) != 64: raise ValueError('Native pycapnp import provenance changed') - for key in ('safety', 'aol_protocol'): + if suite == 'safety-release' and (doc['native']['safety'].get('variant') != 'release' or + doc['native']['safety'].get('all_output_hook_available') is not False): + raise ValueError('RELEASE shard must use the actual release registry') + for key in native_kinds: if not doc['native'][key]['sha256']: raise ValueError('Missing genuine native build evidence') ids = doc['selected_test_ids'] coverage_path = next(path for path in Path(directory).rglob('coverage.json') if json.loads(path.read_text())['shard']['index'] == index) build_input_digest = digest(json.dumps(doc['source']['build_identity']['inputs'], sort_keys=True).encode()) - for kind in ('safety', 'aol_protocol'): + for kind in native_kinds: native = doc['native'][kind] if native['build_inputs_sha256'] != build_input_digest: raise ValueError('Native build input source provenance changed') @@ -232,7 +273,7 @@ def aggregate(directory, count): raise ValueError('Lifecycle outcome report failed') if result['selected_test_ids'] != ids or Counter(row['id'] for row in records) != Counter(ids) or any(row['status'] != 'passed' for row in records): raise ValueError('Lifecycle missing, duplicate or unsuccessful outcomes') - if ids != [identity for identity in full if any(matches(identity, target) for target in expected[index])]: + if ids != [identity for identity in full if any(matcher(identity, target) for target in expected[index])]: raise ValueError('Lifecycle shard cases differ from the full catalog') seen.extend(ids) if Counter(seen) != Counter(full) or len(set(full)) != len(full): @@ -244,9 +285,10 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('directory', type=Path) parser.add_argument('--count', type=int, default=4) + parser.add_argument('--suite', choices=('aol-lifecycle', 'safety-release'), default='aol-lifecycle') parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() - result = aggregate(args.directory, args.count) + result = aggregate(args.directory, args.count, args.suite) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result)) diff --git a/tools/ci/run_vehicle_tests.py b/tools/ci/run_vehicle_tests.py index f1a79db43b..6b47e1cccc 100644 --- a/tools/ci/run_vehicle_tests.py +++ b/tools/ci/run_vehicle_tests.py @@ -716,7 +716,7 @@ def build_aol_protocol(output): def run(suite, output, collect_only=False, shard_index=None, shard_count=4): - if shard_index is not None and (suite != "aol-lifecycle" or not 0 <= shard_index < shard_count): + if shard_index is not None and (suite not in ("aol-lifecycle", "safety-release") or not 0 <= shard_index < shard_count): raise ValueError("Lifecycle shard index is outside its catalog") output = Path(output).resolve() output.mkdir(parents=True, exist_ok=True) @@ -731,7 +731,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): "independent lateral/longitudinal feature transitions", "device firmware execution"], "targets": [], "selected_test_ids": [], "collection_errors": [], "fuzz_seed": os.environ.get("FUZZ_SEED"), "max_examples_override": os.environ.get("MAX_EXAMPLES")} - if suite == "aol-lifecycle": + if suite in ("aol-lifecycle", "safety-release"): plan["scope"] = "selected actual vehicle callers with synthetic receive/readiness timelines; native hooks where exercised by the selected suites" plan["lifecycle_rows"] = AOL_LIFECYCLE_SCOPE plan["uncovered"] = ["complete recorded routes and real-time device deadlines", "physical firmware/harness acceptance", @@ -743,7 +743,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): try: plan["targets"] = targets_for(suite) plan["source"] = source_provenance() - if suite == "aol-lifecycle": + if suite in ("aol-lifecycle", "safety-release"): plan["source"]["build_identity"] = lifecycle_shards.build_identity(ROOT) plan.update(vehicle_inventory()) plan["native"] = native_provenance() @@ -751,9 +751,9 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): plan["native"]["safety"] = build_safety_library(suite == "safety-release", output) if suite == "aol-lifecycle" and not collect_only: plan["native"]["aol_protocol"] = build_aol_protocol(output) - if suite == "aol-lifecycle": + if suite in ("aol-lifecycle", "safety-release"): build_input_digest = hashlib.sha256(json.dumps(plan["source"]["build_identity"]["inputs"], sort_keys=True).encode()).hexdigest() - for name in ('safety', 'aol_protocol'): + for name in (('safety',) if suite == 'safety-release' else ('safety', 'aol_protocol')): if name in plan["native"]: plan["native"][name]["build_inputs_sha256"] = build_input_digest phase = "collection" @@ -765,13 +765,15 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): ids = [test.id() for test in tests] + [vehicle_pytest.test_id(nodeid) for nodeid in pytest_nodeids] if shard_index is not None and not errors: full_targets = list(plan["targets"]) - shards = lifecycle_shards.partition(full_targets, ids, shard_count) + partitioner = lifecycle_shards.release_partition if suite == 'safety-release' else lifecycle_shards.partition + matcher = lifecycle_shards.release_matches if suite == 'safety-release' else lifecycle_shards.matches + shards = partitioner(full_targets, ids, shard_count) plan.update(full_catalog_targets=full_targets, full_catalog_test_ids=list(ids), shard={"index": shard_index, "count": shard_count}) assigned_targets = shards[shard_index] plan["targets"] = assigned_targets def assigned(identity): - return any(lifecycle_shards.matches(identity, target) for target in assigned_targets) + return any(matcher(identity, target) for target in assigned_targets) tests = [test for test in tests if assigned(test.id())] pytest_nodeids = [nodeid for nodeid in pytest_nodeids if assigned(vehicle_pytest.test_id(nodeid))] ids = [test.id() for test in tests] + [vehicle_pytest.test_id(nodeid) for nodeid in pytest_nodeids] @@ -797,7 +799,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): errors.extend(pytest_errors) phase = "source verification" plan["source_after"] = source_provenance() - if suite == "aol-lifecycle": + if suite in ("aol-lifecycle", "safety-release"): plan["source_after"]["build_identity"] = lifecycle_shards.build_identity(ROOT) if plan["source"]["build_identity"] != plan["source_after"]["build_identity"]: gate_errors.append("Lifecycle source/build identity changed during the suite") @@ -810,11 +812,11 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): records = test_runner.account_for_tests(ids, records, tests) if not errors and suite == "interfaces": gate_errors.extend(interface_execution_errors(platform_tests, records)) - if not collect_only and suite == "aol-lifecycle": + if not collect_only and suite in ("aol-lifecycle", "safety-release"): gate_errors.extend(f"Required lifecycle case did not pass: {record['id']} ({record['status']})" for record in records if record["status"] != "passed") for target in plan["targets"]: - if not any(lifecycle_shards.matches(identity, target) for identity in ids): + if not any((lifecycle_shards.release_matches if suite == 'safety-release' else lifecycle_shards.matches)(identity, target) for identity in ids): gate_errors.append(f"Required lifecycle module collected no cases: {target}") elapsed = time.monotonic() - started exit_code = test_runner.report(records, errors + gate_errors, 10, elapsed) diff --git a/tools/ci/tests/test_lifecycle_shards.py b/tools/ci/tests/test_lifecycle_shards.py index 1438e628ab..77b3839d14 100644 --- a/tools/ci/tests/test_lifecycle_shards.py +++ b/tools/ci/tests/test_lifecycle_shards.py @@ -109,9 +109,11 @@ class LifecycleShardTests(unittest.TestCase): camera = 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py' self.assertFalse(any(camera in target for shard in shards for target in shard)) - def documents(self): + def documents(self, suite="aol-lifecycle"): targets, full = self.catalog() - partition = helper.partition(targets, full, 4) + partitioner = helper.release_partition if suite == "safety-release" else helper.partition + matcher = helper.release_matches if suite == "safety-release" else helper.matches + partition = partitioner(targets, full, 4) source: dict = dict.fromkeys(('revision', 'inputs', 'manifest_sha256', 'dependencies', 'runner_sha256', 'suite_runner_sha256', 'pytest_runner_sha256', 'shard_runner_sha256'), 'same') inventory_bytes = Path(__file__).parents[1].joinpath('lifecycle_build_outputs.json').read_bytes() @@ -125,11 +127,12 @@ class LifecycleShardTests(unittest.TestCase): for name, checksum in can_sources.items()}) docs = [] for index, assigned in enumerate(partition): - ids = [identity for identity in full if any(helper.matches(identity, target) for target in assigned)] - doc = {'suite': 'aol-lifecycle', 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4}, + ids = [identity for identity in full if any(matcher(identity, target) for target in assigned)] + doc = {'suite': suite, 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4}, 'targets': assigned, 'selected_test_ids': ids, 'full_catalog_targets': targets, 'full_catalog_test_ids': full, 'collection_errors': [], 'gate_errors': [], 'source': source, 'source_after': source, - 'native': {'can_sources': can_sources, 'pycapnp': {'sha256': 'b' * 64}, 'safety': {'sha256': '1' * 64}, 'aol_protocol': {'sha256': '2' * 64}}} + 'native': {'can_sources': can_sources, 'pycapnp': {'sha256': 'b' * 64}, 'safety': {'sha256': '1' * 64, + 'variant': 'release', 'all_output_hook_available': False}, 'aol_protocol': {'sha256': '2' * 64}}} result = {'selected_test_ids': ids, 'exit_code': 0, 'collection_errors': [], 'gate_errors': [], 'results': [{'id': identity, 'status': 'passed'} for identity in ids]} docs.append((doc, result)) @@ -154,7 +157,38 @@ class LifecycleShardTests(unittest.TestCase): (path / 'coverage.json').write_text(json.dumps(doc)) if artifact_mutation is not None: artifact_mutation(Path(directory)) - return helper.aggregate(directory, 4) + return helper.aggregate(directory, 4, docs[0][0]["suite"]) + + def test_release_partition_accepts_real_class_selectors_and_import_aliases(self): + targets = ['opendbc_repo/opendbc/safety/tests/a.py::TestRelease', 'b.py::test_case[one]', 'c.py'] + ids = ['opendbc.safety.tests.a.TestRelease.test_denied', 'b.test_case[one]', 'c.Test.test_one'] + shards = helper.release_partition(targets, ids, 4) + self.assertCountEqual([target for shard in shards for target in shard], targets) + self.assertEqual(sum(sum(any(helper.release_matches(identity, target) for target in shard) for identity in ids) for shard in shards), len(ids)) + for bad_targets, bad_ids in ((targets, ids + ids), (targets + ['c.py::Test'], ids), (targets, ids + ['foreign.test']), + (targets + ['empty.py'], ids), (['../escape.py'], ['escape.test'])): + with self.subTest(targets=bad_targets), self.assertRaises(ValueError): + helper.release_partition(bad_targets, bad_ids, 4) + + def test_release_aggregate_retains_all_outcome_and_native_guards(self): + docs = self.documents('safety-release') + self.assertEqual(self.check(copy.deepcopy(docs))['cases_passed'], 12) + for change in ('missing', 'duplicate', 'skip', 'debug', 'allOutput', 'wrong_suite'): + bad = json.loads(json.dumps(docs)) + if change == 'missing': + bad[0][1]['results'].pop() + elif change == 'duplicate': + bad[0][1]['results'].append(bad[0][1]['results'][0]) + elif change == 'skip': + bad[0][1]['results'][0]['status'] = 'skipped' + elif change == 'debug': + bad[0][0]['native']['safety']['variant'] = 'ALLOW_DEBUG' + elif change == 'allOutput': + bad[0][0]['native']['safety']['all_output_hook_available'] = True + else: + bad[0][0]['suite'] = 'interfaces' + with self.subTest(change=change), self.assertRaises((ValueError, KeyError, StopIteration)): + self.check(bad) def test_complete_catalog_passes(self): self.assertEqual(self.check(self.documents())['cases_passed'], 12)