From 2d2f662bdf482be70034f4808a1473db548d9ccf Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:06:24 -0500 Subject: [PATCH] Balance independent Toyota cases and audit existing map GPU guards --- prebuilt.json | 2 +- tools/ci/custom_pytest_shards.py | 27 +++++++++++- tools/ci/custom_pytest_workloads.json | 36 +++++++++++++++ tools/ci/tests/test_custom_pytest_shards.py | 49 ++++++++++++++++++--- 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/prebuilt.json b/prebuilt.json index a47656a5c9..6a21bed2e0 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "531d26d0ac61733669544b97fa74454ebb23fe74ac9e9eb0fec0512c1a495a4c", "schema_version": 1} +{"protected_inventory_sha256": "15b50c0ee035548e7306539080164ea96649e4fb6a0d67e45303daf97610054e", "schema_version": 1} diff --git a/tools/ci/custom_pytest_shards.py b/tools/ci/custom_pytest_shards.py index 75cc556a7b..93c5b0bdde 100644 --- a/tools/ci/custom_pytest_shards.py +++ b/tools/ci/custom_pytest_shards.py @@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[2] HEAVY_MODULES = { 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py', 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py', + 'openpilot/starpilot/car/toyota/tests/test_advert_filter_callers.py', } CAPABILITY_NODES = { @@ -36,6 +37,16 @@ CAPABILITY_NODES = { 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-18-0-0]', 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-18-4-2]', 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-20-3-5]', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_draws_heading_up_roads_route_and_soft_edges', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_opacity_fades_map_background', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_without_gps_waits_without_drawing_roads', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_route_stays_bright_at_minimum_map_opacity', +} +MAP_CAPABILITY_NODES = { + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_draws_heading_up_roads_route_and_soft_edges', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_opacity_fades_map_background', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_without_gps_waits_without_drawing_roads', + 'openpilot/starpilot/ui/tests/test_onroad_map.py::test_route_stays_bright_at_minimum_map_opacity', } @@ -56,6 +67,11 @@ def policy(): if (value['version'] != 1 or set(value['independent_functions']) != HEAVY_MODULES or set(value['capability_skips']) != CAPABILITY_NODES): raise ValueError('Unreviewed custom pytest grouping policy') + for node in MAP_CAPABILITY_NODES: + row = value['capability_skips'][node] + if (row.get('guard_source_path') != 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py' or + len(row.get('guard_source_blob', '')) != 40 or row['guard'] != 'gpu_env'): + raise ValueError('Map capability requires exact shared GPU fixture definition') if type(value['independent_max_cases']) is not int or not 1 <= value['independent_max_cases'] <= 60: raise ValueError('Invalid independent parameter group bound') numbers = [value['unknown_seconds_per_case'], value['independent_max_seconds']] @@ -161,6 +177,8 @@ def capability_state(selected, root=ROOT): row = inventory[node] state[node] = {'source_blob': blob_id(Path(root) / node.split('::')[0]), 'guard': row['guard'], 'observation': guard_observation(row['guard'], root)} + if node in MAP_CAPABILITY_NODES: + state[node]['guard_source_blob'] = blob_id(Path(root) / row['guard_source_path']) return state @@ -170,10 +188,13 @@ def validate_capabilities(selected, state, root=ROOT): raise ValueError('Missing or forged audited capability nodes') for node, record in state.items(): row = inventory[node] - if set(record) != {'source_blob', 'guard', 'observation'} or record['guard'] != row['guard']: + fields = {'source_blob', 'guard', 'observation'} | ({'guard_source_blob'} if node in MAP_CAPABILITY_NODES else set()) + if set(record) != fields or record['guard'] != row['guard']: raise ValueError('Forged audited capability guard') if record['source_blob'] != blob_id(Path(root) / node.split('::')[0]): raise ValueError('Audited capability source differs from checkout') + if node in MAP_CAPABILITY_NODES and record['guard_source_blob'] != blob_id(Path(root) / row['guard_source_path']): + raise ValueError('Shared GPU capability guard source differs from checkout') guard_unavailable(record['guard'], record['observation']) @@ -191,6 +212,7 @@ def validate_outcomes(selected, reports, capabilities=None): row = inventory.get(node) record = capabilities.get(node) if (row is None or record is None or record['source_blob'] != row['source_blob'] or + (node in MAP_CAPABILITY_NODES and record.get('guard_source_blob') != row['guard_source_blob']) or record['guard'] != row['guard'] or not guard_unavailable(record['guard'], record['observation'])): raise ValueError('Failed, skipped or erroneous nonexempt custom pytest node') if Counter(report['phase'] for report in phases) != Counter({'setup': 1, 'teardown': 1}): @@ -199,9 +221,10 @@ def validate_outcomes(selected, reports, capabilities=None): teardown = next(report for report in phases if report['phase'] == 'teardown') if setup['status'] != 'skipped' or setup.get('reason') != row['reason'] or teardown['status'] != 'passed': raise ValueError('Audited capability skip has wrong reason or outcome') + shared_guard = {key: row[key] for key in ('guard_source_path', 'guard_source_blob')} if node in MAP_CAPABILITY_NODES else {} skipped.append({'id': node, 'status': 'skipped', 'disposition': 'capability-unavailable', 'reason': row['reason'], 'source_blob': row['source_blob'], 'guard': row['guard'], - 'observation': record['observation']}) + 'observation': record['observation'], **shared_guard}) return skipped diff --git a/tools/ci/custom_pytest_workloads.json b/tools/ci/custom_pytest_workloads.json index 435234b8ca..3c67f90d63 100644 --- a/tools/ci/custom_pytest_workloads.json +++ b/tools/ci/custom_pytest_workloads.json @@ -95,9 +95,45 @@ "guard": "gpu_env", "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_draws_heading_up_roads_route_and_soft_edges": { + "basis": "Existing map fixture alias gl=test_gpu_nv12.gpu; both consumer and shared guard definition pinned; unavailable remains SKIPPED, never passed", + "guard": "gpu_env", + "guard_source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6", + "guard_source_path": "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "e0098c87f63d8f56ebdfa84faa6e49d7811e6e51" + }, + "openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_opacity_fades_map_background": { + "basis": "Existing map fixture alias gl=test_gpu_nv12.gpu; both consumer and shared guard definition pinned; unavailable remains SKIPPED, never passed", + "guard": "gpu_env", + "guard_source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6", + "guard_source_path": "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "e0098c87f63d8f56ebdfa84faa6e49d7811e6e51" + }, + "openpilot/starpilot/ui/tests/test_onroad_map.py::test_overlay_without_gps_waits_without_drawing_roads": { + "basis": "Existing map fixture alias gl=test_gpu_nv12.gpu; both consumer and shared guard definition pinned; unavailable remains SKIPPED, never passed", + "guard": "gpu_env", + "guard_source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6", + "guard_source_path": "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "e0098c87f63d8f56ebdfa84faa6e49d7811e6e51" + }, + "openpilot/starpilot/ui/tests/test_onroad_map.py::test_route_stays_bright_at_minimum_map_opacity": { + "basis": "Existing map fixture alias gl=test_gpu_nv12.gpu; both consumer and shared guard definition pinned; unavailable remains SKIPPED, never passed", + "guard": "gpu_env", + "guard_source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6", + "guard_source_path": "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "e0098c87f63d8f56ebdfa84faa6e49d7811e6e51" } }, "independent_functions": { + "openpilot/starpilot/car/toyota/tests/test_advert_filter_callers.py": { + "basis": "7d2 hosted shard 2: 272 cases took 800.08457733 seconds; Toyota owner verified fresh per-case Params/native hooks/bridge/peer state and teardown.", + "seconds_per_case": 4.0 + }, "openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py": { "basis": "Owner estimate164cases274s; only actual collected IDs when present", "seconds_per_case": 2.0 diff --git a/tools/ci/tests/test_custom_pytest_shards.py b/tools/ci/tests/test_custom_pytest_shards.py index efdf7f4b84..248454e515 100644 --- a/tools/ci/tests/test_custom_pytest_shards.py +++ b/tools/ci/tests/test_custom_pytest_shards.py @@ -327,7 +327,8 @@ def test_each_original_capability_skip_remains_unavailable_never_passed(capabili result = shards.validate_outcomes([node], skipped_reports(node, row['reason']), state) assert result == [{'id': node, 'status': 'skipped', 'disposition': 'capability-unavailable', 'reason': row['reason'], 'source_blob': row['source_blob'], 'guard': row['guard'], - 'observation': unavailable_observation(row['guard'])}] + 'observation': unavailable_observation(row['guard']), + **({key: row[key] for key in ('guard_source_path', 'guard_source_blob')} if node in shards.MAP_CAPABILITY_NODES else {})}] assert shards.validate_outcomes([node], reports([node]), state) == [] @@ -365,7 +366,7 @@ def test_forged_capability_node_and_condition_denied(capability_guards): shards.validate_capabilities([node], state) -def test_audited16_counts_are_skipped_not_passed_in_exact_aggregate(tmp_path, monkeypatch, capability_guards): +def test_audited20_counts_are_skipped_not_passed_in_exact_aggregate(tmp_path, monkeypatch, capability_guards): monkeypatch.setattr(shards, 'source_snapshot', lambda root=shards.ROOT: source()) monkeypatch.setattr(shards, 'same_build_inputs', lambda a, b: a == b) catalog = sorted(capability_guards) + ['required.py::test_required'] @@ -383,10 +384,10 @@ def test_audited16_counts_are_skipped_not_passed_in_exact_aggregate(tmp_path, mo junit_hash = shards.checksum((path / 'diagnostics/pytest.xml').read_bytes()) rewrite(path, 'receipt.json', lambda d, caps=state, digest=junit_hash: d.update(capabilities=caps, junit_sha256=digest)) result = shards.aggregate(tmp_path, 2) - assert result['cases_collected'] == 17 + assert result['cases_collected'] == 21 assert result['cases_passed'] == 1 - assert result['cases_audited_skipped'] == 16 - assert len(result['audited_skips']) == 16 + assert result['cases_audited_skipped'] == 20 + assert len(result['audited_skips']) == 20 assert all(row['status'] == 'skipped' for row in result['audited_skips']) @@ -431,7 +432,7 @@ def test_junit_false_success_and_unaudited_skips_denied(jobs, change): shards.aggregate(directory, 2) -def test_real_pytest_all16_audited_guards_have_skipped_junit_not_passes(tmp_path): +def test_real_pytest_all20_audited_guards_have_skipped_junit_not_passes(tmp_path): """Real pytest phases/JUnit with controlled guard witnesses; no graphics or private APK is tested.""" import shutil package = tmp_path / 'tools/ci' @@ -469,6 +470,9 @@ def test_real_pytest_all16_audited_guards_have_skipped_junit_not_passes(tmp_path source_path.write_text('\n'.join(lines) + '\n') for node in nodes: config['capability_skips'][node]['source_blob'] = shards.blob_id(source_path) + for node in shards.MAP_CAPABILITY_NODES: + row = config['capability_skips'][node] + row['guard_source_blob'] = shards.blob_id(tmp_path / row['guard_source_path']) shards.write_json(package / 'custom_pytest_workloads.json', config) (tmp_path / 'guard_witness.py').write_text('''def pytest_configure(config): from tools.ci import custom_pytest_shards as s @@ -488,7 +492,7 @@ def test_real_pytest_all16_audited_guards_have_skipped_junit_not_passes(tmp_path outcome = json.loads((output / 'outcomes.json').read_text()) assert child.returncode == 0, child.stdout.decode() + child.stderr.decode() assert outcome['status'] == 'completed' - assert len(outcome['catalog']) == len(outcome['audited_skips']) == 16 + assert len(outcome['catalog']) == len(outcome['audited_skips']) == 20 assert all(row['status'] == 'skipped' for row in outcome['audited_skips']) shards.validate_junit(output / 'diagnostics/pytest.xml', outcome['selected'], outcome['audited_skips']) progress = json.loads((output / 'diagnostics/pytest-collection.json').read_text()) @@ -552,3 +556,34 @@ print(json.dumps({'parent': parent, 'worker': worker})) aggregate_line = next(line for line in workflow.splitlines() if 'tools.ci.custom_pytest_shards aggregate ' in line) assert aggregate_line.split('run: ', 1)[1].startswith('python3 -m ') assert 'passed or audited capability-unavailable' in workflow + + +@pytest.mark.parametrize('node', sorted(shards.MAP_CAPABILITY_NODES)) +def test_map_skip_requires_both_consumer_and_shared_fixture_blobs(capability_guards, node): + row = capability_guards[node] + state = shards.capability_state([node]) + shards.validate_capabilities([node], state) + state[node]['guard_source_blob'] = '0' * 40 + with pytest.raises(ValueError, match='Shared GPU'): + shards.validate_capabilities([node], state) + with pytest.raises(ValueError): + shards.validate_outcomes([node], skipped_reports(node, row['reason']), state) + + +def test_reviewed_advert_filter_functions_preserve_272_independent_nodes(): + module = 'openpilot/starpilot/car/toyota/tests/test_advert_filter_callers.py' + counts = {'test_each_advertised_filter_actual_callers': 64, + 'test_physical_saved_off_and_cold_lateral': 48, + 'test_ordinary_cruise_without_aol': 16, + 'test_alpha_does_not_remove_default_long': 16, + 'test_active_hybrid_resume': 8, + 'test_required_sources_withdraw_and_recover_through_actual_can': 120} + catalog = [f'{module}::{name}[{index}]' for name, count in counts.items() for index in range(count)] + assignment, loads = shards.partition(catalog, 24) + assert len(catalog) == 272 + assert sorted(node for group in assignment for node in group) == sorted(catalog) + assert sorted(len(group) for group in assignment if group) == [4, 8, 16, 16, 48, 60, 60, 60] + assert max(loads) <= 240 + reversed_assignment, reversed_loads = shards.partition(list(reversed(catalog)), 24) + assert [set(group) for group in reversed_assignment] == [set(group) for group in assignment] + assert reversed_loads == loads