diff --git a/prebuilt.json b/prebuilt.json index 802c8430a8..746122e7a5 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "d271bc09145447f9fc6d5d41d68a0d7e3fbc38be1f6765c21662dd79d8071b1e", "schema_version": 1} +{"protected_inventory_sha256": "079e92a64c345ad09f03d5a7ede05b13deea46058bbdcc4a29011799cf17a50d", "schema_version": 1} diff --git a/tools/ci/custom_pytest_shards.py b/tools/ci/custom_pytest_shards.py index 41c2cf6e0d..a7a1cccd06 100644 --- a/tools/ci/custom_pytest_shards.py +++ b/tools/ci/custom_pytest_shards.py @@ -15,6 +15,7 @@ from tools.ci.lifecycle_shards import build_identity, same_build_inputs ROOT = Path(__file__).resolve().parents[2] HEAVY_MODULES = { + 'openpilot/starpilot/tests/test_ford_bluepilot_library_lifecycle.py', 'openpilot/starpilot/car/toyota/tests/test_camera_pedal_extension_callers.py', 'openpilot/starpilot/car/toyota/tests/test_eps8_sdsu_callers.py', 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py', diff --git a/tools/ci/custom_pytest_workloads.json b/tools/ci/custom_pytest_workloads.json index 77e990b2a4..0fdacc5506 100644 --- a/tools/ci/custom_pytest_workloads.json +++ b/tools/ci/custom_pytest_workloads.json @@ -165,6 +165,10 @@ "openpilot/starpilot/car/toyota/tests/test_prius_sdsu_callers.py": { "basis": "90 actual DEBUG/RELEASE caller cases passed in four bounded phases (186.885s combined wall time including startup). Scheduling estimate 3s/case; fresh per-case Params, raw CAN, native configuration and C++ host reset; no shared outcome dependence.", "seconds_per_case": 3.0 + }, + "openpilot/starpilot/tests/test_ford_bluepilot_library_lifecycle.py": { + "basis": "736580721a hosted shard7: 248 completed Ford lifecycle cases used573.067107459 phase seconds; slowest case4.394150353s. Round up to5s. Function-scoped library_settings and ford_joined create fresh Params, host/native safety per case and tear down each case; full in-case timelines remain unchanged.", + "seconds_per_case": 5.0 } }, "independent_max_cases": 60, @@ -1982,12 +1986,6 @@ "cases": 936, "seconds": 1.740757 }, - "openpilot/starpilot/system/starport/tests/test_starport.py": { - "basis": "Exact 65660a4 hosted main40623875380: all 72 nodes passed, 22.037476s setup/call/teardown; 25% estimate margin, retaining any higher same-source prior estimate. Scheduling only; no test deadline change.", - "blob": "c4c3d21bea7d749438afeb20886652bca06208bb", - "cases": 72, - "seconds": 27.546846 - }, "openpilot/starpilot/system/starport/tests/test_bugsink_report.py": { "basis": "Exact 65660a4 hosted main40623875380: all 33 nodes passed, 0.737380s setup/call/teardown; 25% estimate margin, retaining any higher same-source prior estimate. Scheduling only; no test deadline change.", "blob": "b759b5e8dc1c52623809ff957bc282ca31c7e539", @@ -2006,6 +2004,12 @@ "cases": 30, "seconds": 0.161266 }, + "openpilot/starpilot/system/starport/tests/test_starport.py": { + "basis": "Exact 65660a4 hosted main40623875380: all 72 nodes passed, 22.037476s setup/call/teardown; 25% estimate margin, retaining any higher same-source prior estimate. Scheduling only; no test deadline change.", + "blob": "c4c3d21bea7d749438afeb20886652bca06208bb", + "cases": 72, + "seconds": 27.546846 + }, "openpilot/starpilot/tests/test_captured_adrv_diagnostic_startup.py": { "basis": "Exact 65660a4 hosted main40623875380: all 64 nodes passed, 12.454431s setup/call/teardown; 25% estimate margin, retaining any higher same-source prior estimate. Scheduling only; no test deadline change.", "blob": "7849a690cd1ac2f3064f60d72622ea7150015cb8", diff --git a/tools/ci/lifecycle_shards.py b/tools/ci/lifecycle_shards.py index 3f9921e930..b1384909c8 100644 --- a/tools/ci/lifecycle_shards.py +++ b/tools/ci/lifecycle_shards.py @@ -92,6 +92,7 @@ def workload_policy(): policy = json.loads(Path(__file__).with_name('lifecycle_workloads.json').read_text()) if policy['version'] != 1 or set(policy['function_groups']) != { 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py', + 'openpilot/starpilot/tests/test_ford_bluepilot_library_lifecycle.py', 'openpilot/starpilot/tests/test_ford_configuration_parity.py', 'openpilot/starpilot/aol/tests/test_stock_location_recovery.py', 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py', diff --git a/tools/ci/lifecycle_workloads.json b/tools/ci/lifecycle_workloads.json index 37a209f400..5afc2d4868 100644 --- a/tools/ci/lifecycle_workloads.json +++ b/tools/ci/lifecycle_workloads.json @@ -29,6 +29,11 @@ "basis": "Scheduling estimate2s/case: actual188 shard3 ordinal577..648 (all inside140case stock-location module) took91.261s; incomplete module timing, not a full measurement. Fresh function-scoped fixture creates CP/state/feedback and private Params per case; no shared outcome dependence; full in-case timelines retained.", "estimated_seconds_per_case": 2.0, "independent_parameters": true + }, + "openpilot/starpilot/tests/test_ford_bluepilot_library_lifecycle.py": { + "basis": "736580721a hosted shard7: 248 completed Ford lifecycle cases used573.067107459 phase seconds; slowest case4.394150353s. Round up to5s. Function-scoped library_settings and ford_joined create fresh Params, host/native safety per case and tear down each case; full in-case timelines remain unchanged.", + "estimated_seconds_per_case": 5.0, + "independent_parameters": true } }, "independent_function_max_cases": 60, diff --git a/tools/ci/tests/test_custom_pytest_shards.py b/tools/ci/tests/test_custom_pytest_shards.py index 5edb1e2226..30d97c77dd 100644 --- a/tools/ci/tests/test_custom_pytest_shards.py +++ b/tools/ci/tests/test_custom_pytest_shards.py @@ -621,3 +621,33 @@ def test_measured_slow_module_exact_function_and_leaf_groups(module, counts, exp def test_invalid_shard_count_still_denied(count): with pytest.raises(ValueError): shards.partition(['a.py::test_a'], count) + + +@pytest.mark.parametrize("count", [24, 32]) +def test_ford_library_cases_keep_complete_timelines_in_bounded_groups(count): + module = 'openpilot/starpilot/tests/test_ford_bluepilot_library_lifecycle.py' + counts = { + "test_library_angle_curvature_aol_native_lifecycle": 96, + "test_model_loss_still_withdraws_library_output_through_actual_callers": 3, + "test_gas_held_first_engagement": 8, + "test_sustained_real_loop_lag_still_disables_ford": 1, + "test_cancel_at_every_native_receipt_phase": 20, + "test_manual_eps_refusal_keeps_aol_intent_without_steering_through_fault": 2, + "test_cruise_resume_preserves_deliberate_lateral_pause": 2, + "test_manual_steering_retires_held_turn_limit_alert": 2, + "test_fresh_cruise_withdrawal_after_previous_output_expires": 4, + "test_reengaging_in_a_manual_curve_respects_native_convergence": 32, + "test_brake_retains_independent_lateral_intent": 48, + "test_starpilot_tuning_uses_current_native_aol_path": 32, + "test_starpilot_low_speed_assistance_survives_brake_with_native_limits": 8 + } + catalog = [f"{module}::{name}[{index}]" for name, cases in counts.items() for index in range(cases)] + assignment, loads = shards.partition(catalog, count) + assert len(catalog) == 258 + assert sorted(node for group in assignment for node in group) == sorted(catalog) + assert max(map(len, assignment)) <= 48 + assert max(loads) <= 240 + assert sum(loads) == 258 * 5 + reverse, reverse_loads = shards.partition(list(reversed(catalog)), count) + assert [set(group) for group in reverse] == [set(group) for group in assignment] + assert reverse_loads == loads diff --git a/tools/ci/tests/test_lifecycle_shards.py b/tools/ci/tests/test_lifecycle_shards.py index a7f45dec38..8d7a6b8880 100644 --- a/tools/ci/tests/test_lifecycle_shards.py +++ b/tools/ci/tests/test_lifecycle_shards.py @@ -73,23 +73,20 @@ class LifecycleShardTests(unittest.TestCase): self.assertCountEqual(selected, identities) self.assertEqual(shards, helper.partition(list(reversed(targets)), list(reversed(identities)), 2)) - def test_measured_bluepilot_lifecycle_is_not_packed_as_unknown_work(self): + def test_measured_bluepilot_lifecycle_cases_split_without_omission(self): module = 'openpilot/starpilot/tests/test_ford_bluepilot_library_lifecycle.py' policy = helper.workload_policy() - measured = policy['modules'][module] - self.assertNotIn(module, policy['function_groups']) - self.assertEqual(measured['cases'], 169) - self.assertGreater(measured['seconds'], 300 * policy['unknown_seconds_per_case']) - self.assertEqual(len(measured['source_sha256']), 64) - self.assertEqual(measured['reference']['artifact_id'], 13071624813) + self.assertTrue(policy['function_groups'][module]['independent_parameters']) + self.assertEqual(policy['function_groups'][module]['estimated_seconds_per_case'], 5.) prefix = module.removesuffix('.py').replace('/', '.') + '.' - identities = [prefix + f'test_case[{index}]' for index in range(169)] - identities += [f'other.test_case[{index}]' for index in range(300)] - shards = helper.partition([module, 'other.py'], identities, 2) - self.assertEqual(shards, [[module], ['other.py']]) + identities = [prefix + f'test_case[{index}]' for index in range(258)] + shards = helper.partition([module], identities, 16) selected = [identity for shard in shards for identity in identities if any(helper.matches(identity, target) for target in shard)] self.assertCountEqual(selected, identities) + self.assertEqual(len(selected), 258) + self.assertLessEqual(max(map(len, shards)), 17) + self.assertEqual(shards, helper.partition([module], list(reversed(identities)), 16)) def test_only_large_reviewed_heavy_functions_split_into_exact_parameter_leaves(self): heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'