Balance independent lifecycle workloads within existing deadlines

.github/workflows/tests.yaml

@@ -253,7 +253,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shard: [0, 1, 2, 3]
+        shard: [0, 1, 2, 3, 4, 5, 6, 7]
     env:
       FUZZ_SEED: '0'
     steps:
@@ -263,7 +263,7 @@ jobs:
       run: scons
     - name: Gate every assigned lifecycle module serially
       timeout-minutes: 10
-      run: python3 tools/ci/run_vehicle_tests.py --suite aol-lifecycle --shard-index "${{ matrix.shard }}" --shard-count 4 --output "ci-results/aol-lifecycle-${{ matrix.shard }}"
+      run: python3 tools/ci/run_vehicle_tests.py --suite aol-lifecycle --shard-index "${{ matrix.shard }}" --shard-count 8 --output "ci-results/aol-lifecycle-${{ matrix.shard }}"
     - name: Save every lifecycle outcome and native build
       if: always()
       uses: actions/upload-artifact@v4
@@ -286,7 +286,7 @@ jobs:
         pattern: aol-lifecycle-*
         path: ci-results/aol-lifecycles
     - name: Require every full-catalog case exactly once and passed
-      run: python3 tools/ci/lifecycle_shards.py ci-results/aol-lifecycles --count 4 --output ci-results/aol-lifecycle-complete.json
+      run: python3 tools/ci/lifecycle_shards.py ci-results/aol-lifecycles --count 8 --output ci-results/aol-lifecycle-complete.json
     - name: Save complete catalog verdict
       if: always()
       uses: actions/upload-artifact@v4

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "3296de2a8c3e7de490d9f1d3dc96668c0484c20c7aa3e7098a7d5528f1327d7c", "schema_version": 1}
+{"protected_inventory_sha256": "a4d381b8657230b38fe1a23ee4fc01717dc84be82e6f136daa88f24e88d851e5", "schema_version": 1}

tools/ci/lifecycle_shards.py

@@ -1,8 +1,9 @@
-"""Partition whole lifecycle modules and prove complete shard outcomes."""
+"""Partition lifecycle workloads and prove complete shard outcomes."""
 import argparse
 import hashlib
 from collections import Counter
 import json
+import math
 import os
 import subprocess
 from pathlib import Path
@@ -74,24 +75,91 @@ def same_build_inputs(first, second):
   return first['inputs'] == second['inputs']
 
 
+def matches(identity, target):
+  """A whole module, or one declared heavy module's whole pytest function."""
+  module, *functions = target.split('::', 1)
+  prefix = module.removesuffix('.py').replace('/', '.') + '.'
+  if not functions:
+    return identity.startswith(prefix)
+  function = functions[0].split('[', 1)[0]
+  if not function.isidentifier() or not function.startswith('test_'):
+    raise ValueError('Invalid lifecycle function selector')
+  selected = prefix + functions[0]
+  return identity == selected if '[' in functions[0] else identity == selected or identity.startswith(selected + '[')
+
+
+def workload_policy():
+  policy = json.loads(Path(__file__).with_name('lifecycle_workloads.json').read_text())
+  if policy['version'] != 1 or set(policy['function_groups']) != {
+      'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py',
+      'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py'}:
+    raise ValueError('Unreviewed lifecycle function grouping policy')
+  if type(policy['independent_function_max_cases']) is not int or not 1 <= policy['independent_function_max_cases'] <= 60:
+    raise ValueError('Invalid independent function group size')
+  if any(row.get('independent_parameters') is not True for row in policy['function_groups'].values()):
+    raise ValueError('Function parameter independence was not reviewed')
+  values = [policy['independent_function_max_seconds'], policy['unknown_seconds_per_case']]
+  values.extend(row['estimated_seconds_per_case'] for row in policy['function_groups'].values())
+  if any(not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0 for value in values):
+    raise ValueError('Invalid lifecycle workload estimate')
+  for row in policy['modules'].values():
+    if type(row['cases']) is not int or row['cases'] < 1 or not math.isfinite(row['seconds']) or row['seconds'] <= 0:
+      raise ValueError('Invalid measured lifecycle workload')
+  return policy
+
+
 def partition(targets, identities, count):
-  if count < 1 or len(set(targets)) != len(targets) or len(set(identities)) != len(identities):
+  if (not 1 <= count <= 8 or len(set(targets)) != len(targets) or len(set(identities)) != len(identities) or
+      any(not target.endswith('.py') or '::' in target or Path(target).is_absolute() or '..' in Path(target).parts for target in targets)):
     raise ValueError('Invalid lifecycle catalog')
   owners = {}
   for identity in identities:
-    matches = [target for target in targets if identity.startswith(target.removesuffix('.py').replace('/', '.') + '.')]
-    if len(matches) != 1:
+    modules = [target for target in targets if matches(identity, target)]
+    if len(modules) != 1:
       raise ValueError(f'Lifecycle identity needs exactly one module: {identity}')
-    owners[identity] = matches[0]
-  weights = Counter(owners.values())
-  if set(weights) != set(targets):
+    owners[identity] = modules[0]
+  if set(owners.values()) != set(targets):
     raise ValueError('Lifecycle module collected no cases')
-  shards = [[] for _ in range(count)]
-  loads = [0] * count
-  for target in sorted(targets, key=lambda target: (-weights[target], target)):
+  policy = workload_policy()
+  units = {}
+  for identity, target in owners.items():
+    unit = target
+    if target in policy['function_groups']:
+      function = identity[len(target.removesuffix('.py').replace('/', '.') + '.'):].split('[', 1)[0]
+      if not function.isidentifier() or not function.startswith('test_'):
+        raise ValueError('Heavy lifecycle grouping requires top-level pytest functions')
+      unit += '::' + function
+    units.setdefault(unit, []).append(identity)
+  bounded = {}
+  for unit, selected in units.items():
+    target = unit.split('::', 1)[0]
+    if (target in policy['function_groups'] and
+        (len(selected) > policy['independent_function_max_cases'] or
+         len(selected) * policy['function_groups'][target]['estimated_seconds_per_case'] > policy['independent_function_max_seconds'])):
+      prefix = target.removesuffix('.py').replace('/', '.') + '.'
+      # Only reviewed independent parameter cases become exact leaf selectors.
+      # Actual collected node IDs are retained verbatim, including parameter punctuation.
+      for identity in selected:
+        bounded[target + '::' + identity[len(prefix):]] = [identity]
+    else:
+      bounded[unit] = selected
+  units = bounded
+  weights = {}
+  for unit, selected in units.items():
+    target = unit.split('::', 1)[0]
+    if target in policy['function_groups']:
+      per_case = policy['function_groups'][target]['estimated_seconds_per_case']
+    elif target in policy['modules']:
+      measured = policy['modules'][target]
+      per_case = measured['seconds'] / measured['cases']
+    else:
+      per_case = policy['unknown_seconds_per_case']
+    weights[unit] = per_case * len(selected)
+  shards, loads = [[] for _ in range(count)], [0.0] * count
+  for unit in sorted(units, key=lambda unit: (-weights[unit], unit)):
     index = min(range(count), key=lambda index: (loads[index], index))
-    shards[index].append(target)
-    loads[index] += weights[target]
+    shards[index].append(unit)
+    loads[index] += weights[unit]
   return shards
 
 
@@ -160,8 +228,7 @@ def aggregate(directory, count):
       raise ValueError('Lifecycle outcome report failed')
     if result['selected_test_ids'] != ids or Counter(row['id'] for row in records) != Counter(ids) or any(row['status'] != 'passed' for row in records):
       raise ValueError('Lifecycle missing, duplicate or unsuccessful outcomes')
-    prefixes = tuple(target.removesuffix('.py').replace('/', '.') + '.' for target in expected[index])
-    if ids != [identity for identity in full if identity.startswith(prefixes)]:
+    if ids != [identity for identity in full if any(matches(identity, target) for target in expected[index])]:
       raise ValueError('Lifecycle shard cases differ from the full catalog')
     seen.extend(ids)
   if Counter(seen) != Counter(full) or len(set(full)) != len(full):

tools/ci/lifecycle_workloads.json

@@ -0,0 +1,463 @@
+{
+  "function_groups": {
+    "openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py": {
+      "basis": "Owner-reported164cases274s; optional future module, never adds cases to catalog",
+      "estimated_seconds_per_case": 2.0,
+      "independent_parameters": true
+    },
+    "openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py": {
+      "basis": "Scheduling estimate: observed72cases213s in6ebshard3; incomplete module, not a full measured runtime",
+      "estimated_seconds_per_case": 4.0,
+      "independent_parameters": true
+    }
+  },
+  "independent_function_max_cases": 60,
+  "independent_function_max_seconds": 240.0,
+  "modules": {
+    "opendbc_repo/opendbc/car/honda/tests/test_classic_alpha_aol_parity.py": {
+      "cases": 33,
+      "seconds": 0.6483215049998989,
+      "source_sha256": "2f907a1ba0e43f5c310748678a0b233b9c656a21043094e1329a6491458c4386"
+    },
+    "opendbc_repo/opendbc/car/honda/tests/test_manual_honda_port_aol.py": {
+      "cases": 8,
+      "seconds": 0.10920116499994492,
+      "source_sha256": "8634678921c2f2919d01945e9f71b01ca025861e56bbdcfa7b5b0b35899cc6ab"
+    },
+    "opendbc_repo/opendbc/car/mazda/tests/test_dashboard_stop_sign.py": {
+      "cases": 52,
+      "seconds": 1.8563495159999093,
+      "source_sha256": "ae178c63d9cdc55845036f8d062ddc16d323a518ae1f91b5a3e11417a432ceda"
+    },
+    "opendbc_repo/opendbc/car/volkswagen/tests/test_meb_discovery.py": {
+      "cases": 8,
+      "seconds": 0.22158582399970328,
+      "source_sha256": "0183f94bc70f7d73769540f75147656445671b699ce797af2fb88de5532875a9"
+    },
+    "opendbc_repo/opendbc/car/volkswagen/tests/test_meb_preferences.py": {
+      "cases": 171,
+      "seconds": 0.21106671699919843,
+      "source_sha256": "7bb06ba1f1e307719a53cf24af29c2cd3141dda35daa417cb535fd8d4f771040"
+    },
+    "opendbc_repo/opendbc/car/volkswagen/tests/test_mqb_aol_discovery.py": {
+      "cases": 6,
+      "seconds": 0.11721130700016147,
+      "source_sha256": "246e9c0c5962bd6a9c2f1f31459f9e06a5709bd4dc8791883909edff6c03b638"
+    },
+    "opendbc_repo/opendbc/car/volkswagen/tests/test_mqb_topology_discovery.py": {
+      "cases": 18,
+      "seconds": 0.3372017629998254,
+      "source_sha256": "29734b924f186ba545a568273c0dfd27aa75627f5166d31e078da4b84a5b9505"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_ford_configuration_parity.py": {
+      "cases": 85,
+      "seconds": 2.086589082000387,
+      "source_sha256": "278d3f825224427ebddde99373b0811d136456c7caec54842f0bea8b02ad67dc"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_ford_full_engagement.py": {
+      "cases": 169,
+      "seconds": 1.0015241779998973,
+      "source_sha256": "f7fd6b4cbdbae72797277b5a039928b4d7406451ca14839142118e27f3e2473a"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_hkg_mode_parity.py": {
+      "cases": 75,
+      "seconds": 3.885427439000523,
+      "source_sha256": "97711068cdc3ebab22165b2ddab417f0eab13134da452c189dc1af7da1deb521"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_hyundai_angle_tx_recovery.py": {
+      "cases": 9,
+      "seconds": 0.24554885200001308,
+      "source_sha256": "3e3559792d8daef5fc09c017f742060f93c737d76904ae1822c0b12d4c7b0a91"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_mazda_stock_aol.py": {
+      "cases": 13,
+      "seconds": 0.23759958699997696,
+      "source_sha256": "16eeb9c2fcf0f945eb1196b99651bdfad7407ef4de2b0cf00dd22def471dbd22"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_meb_aol.py": {
+      "cases": 63,
+      "seconds": 0.8211498970007369,
+      "source_sha256": "06d099bdaea9462d9038f9f2f06f48b3ce3c5a18ede4d174fc55ffddf97f2b7a"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_pe_combined_callers.py": {
+      "cases": 2,
+      "seconds": 20.573897311999986,
+      "source_sha256": "bb8f4ae38158db387f9e1f39ff7acd4f505cf6bdf44e79a1fd45f80a88a229ae"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_pe_set_release_callers.py": {
+      "cases": 1,
+      "seconds": 13.006026775999999,
+      "source_sha256": "0db205adee85085ceb5af97a7195844484072083ade34ab5453e036ad21c7734"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_subaru_avh_startup.py": {
+      "cases": 25,
+      "seconds": 6.132081978999622,
+      "source_sha256": "eb3a1c05e0d90f6b30e58c53a1cffb0bf490ec1e57a3311cb68ea3e7e7487e1e"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_subaru_outback_angle.py": {
+      "cases": 8,
+      "seconds": 0.07942780600001242,
+      "source_sha256": "d8bdce43be7b9e62b0e71dfadf4d18b629b5f6eed0e257f57a897145fdb091a4"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_subaru_sng.py": {
+      "cases": 6,
+      "seconds": 0.0969043210000109,
+      "source_sha256": "3bd08c34e9db4a056ae99a72b321aa72a0d619ea178b0b88e047b75f55c719ec"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_subaru_software_cruise.py": {
+      "cases": 4,
+      "seconds": 0.030385844000022644,
+      "source_sha256": "a66efee40870deea1e4d5620b93b0c9fcde9676cb4d80ab23a9cef37f725b343"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_subaru_stock_aol.py": {
+      "cases": 10,
+      "seconds": 0.04827245700005278,
+      "source_sha256": "f59adcbd502ec880148dcc0b8fd7c1a21b3207b31bc1e3db23909973d87a21a3"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_subaru_stop_start.py": {
+      "cases": 5,
+      "seconds": 0.058483079000041016,
+      "source_sha256": "6ea85a318bb77f3a808c38dc9017629d6391c09ac2845536dbf381a2c829e912"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_volkswagen_direct_aol.py": {
+      "cases": 22,
+      "seconds": 0.11070563900011621,
+      "source_sha256": "ce7e6e3ffc1d448261d820784b6257c49810a7c83dc578ab09ea30af0240b77d"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_volkswagen_engagement_arm.py": {
+      "cases": 2,
+      "seconds": 0.007902616000137641,
+      "source_sha256": "f1430a7891ef36c4058b9c3c592e20f542b278129fdec000d17a0772a744ff8e"
+    },
+    "opendbc_repo/opendbc/safety/tests/test_volkswagen_stock_aol.py": {
+      "cases": 6,
+      "seconds": 0.49320137799998065,
+      "source_sha256": "15a1f6ba1c55022bf93a13180629a5685e7f119f9203ae5e4d02e916e4fdea78"
+    },
+    "openpilot/selfdrive/car/tests/test_registered_ford_discovery.py": {
+      "cases": 27,
+      "seconds": 1.3744605619999106,
+      "source_sha256": "94cc207f618e17d41d5bcba27374b4e21a14597829535520a8a2acb78a5492dd"
+    },
+    "openpilot/starpilot/aol/tests/test_chrysler.py": {
+      "cases": 14,
+      "seconds": 0.15553418699983013,
+      "source_sha256": "85ecac582401263d06ed6aa43e6cf9348513db9086c8b17d91968e2cefcde7f4"
+    },
+    "openpilot/starpilot/aol/tests/test_ev6_source_deadline.py": {
+      "cases": 39,
+      "seconds": 4.952025227999798,
+      "source_sha256": "be85a97dac7787acd813b406969120ef833927c9f36b7493e9577ad102e755db"
+    },
+    "openpilot/starpilot/aol/tests/test_ev9_main_withdrawal_guards.py": {
+      "cases": 37,
+      "seconds": 0.08945876099949146,
+      "source_sha256": "94d19de8f8a786f6a6ca3fe24f6509f8304b8ed8952cb87dd06e3365fc2f7559"
+    },
+    "openpilot/starpilot/aol/tests/test_gm.py": {
+      "cases": 29,
+      "seconds": 24.629748084000198,
+      "source_sha256": "dec162ff70628d3c924780d6beaae2f5da999a557a675970f58d7adc9166f354"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_ascm_receive_order.py": {
+      "cases": 340,
+      "seconds": 6.830010596001102,
+      "source_sha256": "77a2183fbdc3fecb8e8072246d750138f3a9340a33629cb5dbd36309500393cf"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_cancel_remap.py": {
+      "cases": 48,
+      "seconds": 3.7847172010006034,
+      "source_sha256": "63580c71e8716520e8156ff1b8031edfad898e37d332378f2d7d40acd9b1d639"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_held_brake_successor.py": {
+      "cases": 42,
+      "seconds": 141.64848118599923,
+      "source_sha256": "750bfc0681699792f2964db7d4410e83a2ae2c994d0056ab9e6a3ce848e3bc5c"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_held_native.py": {
+      "cases": 140,
+      "seconds": 3.158628827999678,
+      "source_sha256": "d32a0d27f69c016a626c58bcfd09d68bb25ffdff52b33ceab1e57b3f48fdae5b"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_ordinary_warning.py": {
+      "cases": 24,
+      "seconds": 0.41067921500018656,
+      "source_sha256": "ea11350b3d2ddbb3a950ac11fa3ba2aa511567fa8d6a2a70960b446a39840ddf"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_pedal_order.py": {
+      "cases": 34,
+      "seconds": 0.6718514639992463,
+      "source_sha256": "a3aa9eda612888f1d3631097d2341080e6282b4ca96fb35d33577cbf91477a78"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_route1c_receive_order.py": {
+      "cases": 54,
+      "seconds": 1.547176897000213,
+      "source_sha256": "279ac2d8dc2defcf6c14b30ba9e0efedf2d56163ad279e814457b5438560ded9"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_startup_main.py": {
+      "cases": 67,
+      "seconds": 8.541517943999565,
+      "source_sha256": "0686d37199979cd48c92694e771ce0b0d3b47372bc3dfed587fb075ea1bf3d0d"
+    },
+    "openpilot/starpilot/aol/tests/test_gm_steering_receive_order.py": {
+      "cases": 45,
+      "seconds": 1.5989013689995772,
+      "source_sha256": "0264749678d62a9e83dde063ee8e66291c13801bd61b899db5e7f438d87aed10"
+    },
+    "openpilot/starpilot/aol/tests/test_honda_alt_radar_lifecycle.py": {
+      "cases": 14,
+      "seconds": 15.806861794000156,
+      "source_sha256": "be3bd9bcb84542e191743968312987288197f028bfc19f8cad11a1adff9a469a"
+    },
+    "openpilot/starpilot/aol/tests/test_honda_classic_alpha_lifecycle.py": {
+      "cases": 8,
+      "seconds": 12.389079092999964,
+      "source_sha256": "a41940cdd820382dc247c4d002bdd239d2a542f3751fd8d5463d230c8415c442"
+    },
+    "openpilot/starpilot/aol/tests/test_honda_pedal_lifecycle.py": {
+      "cases": 8,
+      "seconds": 11.878174505999993,
+      "source_sha256": "482ef127e7c3eb91c97dab3645ecc6125b1d0fd28b283f6e4a19ae010965dda6"
+    },
+    "openpilot/starpilot/aol/tests/test_honda_recovery_lifecycle.py": {
+      "cases": 5,
+      "seconds": 7.382482789999983,
+      "source_sha256": "b645f18383c37ed393a40b9db88d9680d8e3eb2d4ef384b25ade5b50bf135e1d"
+    },
+    "openpilot/starpilot/aol/tests/test_honda_reverse_join.py": {
+      "cases": 342,
+      "seconds": 0.7359273099999655,
+      "source_sha256": "46a9ddd6f5b48e5571ae1df5dfb5e392dc78ea1a8889086485f488f886940ae7"
+    },
+    "openpilot/starpilot/aol/tests/test_independent_lateral_soft_takeover.py": {
+      "cases": 4,
+      "seconds": 0.36067755299984583,
+      "source_sha256": "ca04ccb1aa702c299151fb0f4a76a22afb5923411c6655a768c6576cd2817d8b"
+    },
+    "openpilot/starpilot/aol/tests/test_meb_actual_callers.py": {
+      "cases": 20,
+      "seconds": 18.64853539600017,
+      "source_sha256": "a87812c36d931e77b41785114f55a4d95d926c45322a5aacb9271070175ebffc"
+    },
+    "openpilot/starpilot/aol/tests/test_mqb_topology_callers.py": {
+      "cases": 24,
+      "seconds": 24.303543111000295,
+      "source_sha256": "8300c3eef7ee075ed11a40d99726d3de9e3757d34856b3921191da675106170c"
+    },
+    "openpilot/starpilot/aol/tests/test_mqb_topology_contract.py": {
+      "cases": 277,
+      "seconds": 0.18643848999931834,
+      "source_sha256": "6af1c00cb307c79fb4b024ba8bde72da98eaa16a0f9ba94c9bd46dace44c97b0"
+    },
+    "openpilot/starpilot/aol/tests/test_native_inventory_control_cadence.py": {
+      "cases": 85,
+      "seconds": 0.3153113310008848,
+      "source_sha256": "5f9e0f4d8a07efbf8c7313f06c1d6ffa270bd781ccfb6119981c627058e7ed75"
+    },
+    "openpilot/starpilot/aol/tests/test_pe_combined_profile_contract.py": {
+      "cases": 31,
+      "seconds": 0.040698814000279526,
+      "source_sha256": "9dc11b2ee02f9a609cc1fcf4f1546dc15694d0ef9557f291beedfbf567481d8c"
+    },
+    "openpilot/starpilot/aol/tests/test_stock_angle_physical_recovery.py": {
+      "cases": 43,
+      "seconds": 9.417894639000508,
+      "source_sha256": "c1e1f05ee6ce661dcf364a33069a5ba029a1bf113c31b0f07646148dd101842a"
+    },
+    "openpilot/starpilot/aol/tests/test_stock_lateral_lag_gap_sequence.py": {
+      "cases": 4,
+      "seconds": 0.3693457449999187,
+      "source_sha256": "c9d26307eb74153898f2ad241f3ce36d3940e19d02aaa456a9111224d021e922"
+    },
+    "openpilot/starpilot/aol/tests/test_stock_location_recovery.py": {
+      "cases": 140,
+      "seconds": 0.34873870500052817,
+      "source_sha256": "c6eb1344d1b59e2d2a18d6c8406b358091f274e524ab17686e67b96897bf7841"
+    },
+    "openpilot/starpilot/aol/tests/test_toyota.py": {
+      "cases": 9,
+      "seconds": 4.275880477000101,
+      "source_sha256": "92b92b46c457c30c7d95949ef2f24f92cf79043b2b5ece635e10c9923a4eea93"
+    },
+    "openpilot/starpilot/aol/tests/test_volkswagen.py": {
+      "cases": 96,
+      "seconds": 0.17290022300085184,
+      "source_sha256": "f0662719d00fc77e664176cf9c087eb3ce321772de456c585ecb406d7bd7031c"
+    },
+    "openpilot/starpilot/aol/tests/test_volkswagen_main_boot.py": {
+      "cases": 12,
+      "seconds": 0.01782609900033094,
+      "source_sha256": "fed6af0445f73970bea2d07313f4117c55650c7b525a07e567c800577ee5a114"
+    },
+    "openpilot/starpilot/aol/tests/test_volkswagen_saved_intent.py": {
+      "cases": 10,
+      "seconds": 0.012575867000123253,
+      "source_sha256": "f627232c0ed786c316bfbebd4852e4c9c10d41847fbae14957d737e4babf39df"
+    },
+    "openpilot/starpilot/car/nissan/tests/test_aol.py": {
+      "cases": 55,
+      "seconds": 1.4633806500006585,
+      "source_sha256": "4303ab761aa4df46f7a959634c9b7355e76a324dafae97fd95d64fb1ec527a3e"
+    },
+    "openpilot/starpilot/car/rivian/tests/test_aol_lifecycle.py": {
+      "cases": 2,
+      "seconds": 4.649988322000013,
+      "source_sha256": "7bd28d138aeabebe49afd89a4d20c3fcb7a6f8665d04d510a30f7cfb735cedc0"
+    },
+    "openpilot/starpilot/car/rivian/tests/test_aol_native.py": {
+      "cases": 14,
+      "seconds": 0.09512363700025617,
+      "source_sha256": "563b4c3d4681a31efcd617345a418417cbe37beb5993bbbfdb7779890bbbd43c"
+    },
+    "openpilot/starpilot/car/rivian/tests/test_aol_policy.py": {
+      "cases": 26,
+      "seconds": 0.007779541000672907,
+      "source_sha256": "d2e97250674ed5377bc82ebc27fb4b74228a49a2fc3bc4ca0b1932f846525ff6"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_avh.py": {
+      "cases": 4,
+      "seconds": 0.08652710500007288,
+      "source_sha256": "7825da497c3baadde9ced2fbaffccfb4fac49e5f3bde3649bc003ca207bf6baf"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_avh_callers.py": {
+      "cases": 11,
+      "seconds": 115.07622235099996,
+      "source_sha256": "e571b425c49b1b639a1937669e2541bccf49b96eb01aec96ca34dd825ed7b662"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_forester_angle_callers.py": {
+      "cases": 4,
+      "seconds": 18.46506426399992,
+      "source_sha256": "e79313e7858606218b2fc4e0ddc065512e6be190e250587e91570d5f5a9bba6c"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_outback_angle_callers.py": {
+      "cases": 4,
+      "seconds": 21.32710642299992,
+      "source_sha256": "5e5e750b7f44bb75f0e278ef394a77499338a11a64079804bee8bbfc43aa7fca"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_outback_startup_callers.py": {
+      "cases": 8,
+      "seconds": 18.163427912999964,
+      "source_sha256": "769da5685e87cd3203f0575fb38c89e33d979829aa50e149c5b4980e27d986d5"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_smoothing.py": {
+      "cases": 33,
+      "seconds": 0.04882807000080902,
+      "source_sha256": "8be1d4efb6b2aa0da3ac1654e91451a8587fcda21ede55bedb3b5cc180dc0689"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_smoothing_factory.py": {
+      "cases": 48,
+      "seconds": 1.392584342999271,
+      "source_sha256": "261c026576da773c8eaad7afab05e8ac5753ab7af1e82640823f17531ffd56b6"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_smoothing_settings.py": {
+      "cases": 7,
+      "seconds": 0.04594962300006955,
+      "source_sha256": "9cace4ea79dfc3ca3bbd5b43698a1fcbfa656e28666460f54285790615a8524f"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_sng_callers.py": {
+      "cases": 10,
+      "seconds": 25.291504849000148,
+      "source_sha256": "5f7c462a9861cd0e2934eb17caef340f45372efb975c4a782fb2652a3078ce4c"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_software_cruise.py": {
+      "cases": 26,
+      "seconds": 0.0727703799993833,
+      "source_sha256": "a76e993baec5a8fa9007649c65fda5767df6fe0e8d7da696a595f031e3329f9c"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_software_cruise_callers.py": {
+      "cases": 2,
+      "seconds": 12.736984069999835,
+      "source_sha256": "25fb3caade53ed424333daf4883fe4bdbf7b44d36286c2839d95473ca3937d48"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_stock_aol_callers.py": {
+      "cases": 5,
+      "seconds": 21.631809591000092,
+      "source_sha256": "1ed4f9d45907b2c41a6cd8344e1623f9e1c74825f6e2395ef2459589f1a7ca9e"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_stop_start.py": {
+      "cases": 5,
+      "seconds": 0.07591541000010693,
+      "source_sha256": "bf1e420f7387e99da55c667ed40a3d0c2d6535469bd91d6d94be196d2fdddf83"
+    },
+    "openpilot/starpilot/car/subaru/tests/test_stop_start_callers.py": {
+      "cases": 11,
+      "seconds": 26.80400199300027,
+      "source_sha256": "42c394bd27c29917cc0876606560bb55ad96c97d5b2b5ad0b34a3876c3a9c1ec"
+    },
+    "openpilot/starpilot/car/toyota/tests/test_legacy_advert_filter.py": {
+      "cases": 182,
+      "seconds": 0.1225578689983422,
+      "source_sha256": "84002911a55b0381b23369c9d84cad93fced558cb451076e9eb8902830166606"
+    },
+    "openpilot/starpilot/car/toyota/tests/test_legacy_advert_lateral.py": {
+      "cases": 78,
+      "seconds": 0.0679267290001917,
+      "source_sha256": "b91f61aacb0fc2cfafae115d36a50cc1c46f7952a2ee27c71c93292b65f5dc24"
+    },
+    "openpilot/starpilot/car/toyota/tests/test_legacy_advert_native.py": {
+      "cases": 48,
+      "seconds": 6.749222234999706,
+      "source_sha256": "242f2d1670fe5e25224fa25c24983dc609479871ff16dfcc274aebac7d42d611"
+    },
+    "openpilot/starpilot/car/toyota/tests/test_legacy_stop_timer.py": {
+      "cases": 2,
+      "seconds": 0.04418585500013705,
+      "source_sha256": "d642a6c2c139295e1a4a79985af4525e6850a9073fbb4a6de06eb004840f442c"
+    },
+    "openpilot/starpilot/car/toyota/tests/test_legacy_stop_timer_callers.py": {
+      "cases": 38,
+      "seconds": 70.52549054199937,
+      "source_sha256": "bcd8f46d4e0520181d43c19975c6b9b454199f944bcb4db034ace3a7cf7b18e0"
+    },
+    "openpilot/starpilot/car/toyota/tests/test_lexus_is_caller_native.py": {
+      "cases": 12,
+      "seconds": 42.31224894199954,
+      "source_sha256": "dd8fb8e2e6a7d45739fe956d7f9d6d162c3d84e75c8cca25b26819ea3ba0ace5"
+    },
+    "openpilot/starpilot/longitudinal/tests/test_volt_mode_transition.py": {
+      "cases": 66,
+      "seconds": 0.11916299399990748,
+      "source_sha256": "c4ffd66c53f5d92f57d6b743245ecaa137e9c144d687bbf6f26da8a2148d18ea"
+    },
+    "openpilot/starpilot/tests/test_ford_ci_registration.py": {
+      "cases": 2,
+      "seconds": 0.007034268999973392,
+      "source_sha256": "33a1cbd07b36f471c13a40f896202616ffa287140949014d1bd9ece31ddbdc12"
+    },
+    "openpilot/starpilot/tests/test_ford_configuration_admission.py": {
+      "cases": 57,
+      "seconds": 0.058055562000333794,
+      "source_sha256": "1d3bd128236cf789b37a6162018dfb719e3e82a0955882f2109a3e23663825f3"
+    },
+    "openpilot/starpilot/tests/test_ford_configuration_parity.py": {
+      "cases": 112,
+      "seconds": 217.39153231700044,
+      "source_sha256": "95b173ace6c7d4f3a258618b688936414145f4e6d253e8eb5fdbe1e4cd0a28ba"
+    },
+    "openpilot/starpilot/ui/tests/test_meb_preferences.py": {
+      "cases": 18,
+      "seconds": 0.02584481600001709,
+      "source_sha256": "e05af87c2c0707d126ca1eb1e6209b656a34854b619d095cd33436ce89ae922c"
+    }
+  },
+  "reference": {
+    "artifacts": {
+      "0": {
+        "duration_seconds": 319.92404296200004,
+        "results_sha256": "068997f0cbfd42a825bfda8a9ce821bfa61ce8b9dc903a85a31bf907fd199f06"
+      },
+      "1": {
+        "duration_seconds": 512.3838762299999,
+        "results_sha256": "bdd55736da9bbcc8d06fdffb3502d120ccf3dc4144e4dd9bc58cbfd0064ad98e"
+      },
+      "2": {
+        "duration_seconds": 234.18615906500003,
+        "results_sha256": "e6632a5af8ed8f3347bead25261208c062565ec3628011a1b46fd75f7b77cc05"
+      }
+    },
+    "revision": "6eb9df96425a661afcbf1edfa27894d6e49b3706",
+    "run_id": 39923284827
+  },
+  "unknown_seconds_per_case": 1.0,
+  "version": 1
+}

tools/ci/run_vehicle_tests.py

@@ -680,10 +680,12 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
       shards = lifecycle_shards.partition(full_targets, ids, shard_count)
       plan.update(full_catalog_targets=full_targets, full_catalog_test_ids=list(ids),
                   shard={"index": shard_index, "count": shard_count})
-      plan["targets"] = shards[shard_index]
-      prefixes = tuple(target.removesuffix(".py").replace("/", ".") + "." for target in plan["targets"])
-      tests = [test for test in tests if test.id().startswith(prefixes)]
-      pytest_nodeids = [nodeid for nodeid in pytest_nodeids if vehicle_pytest.test_id(nodeid).startswith(prefixes)]
+      assigned_targets = shards[shard_index]
+      plan["targets"] = assigned_targets
+      def assigned(identity):
+        return any(lifecycle_shards.matches(identity, target) for target in assigned_targets)
+      tests = [test for test in tests if assigned(test.id())]
+      pytest_nodeids = [nodeid for nodeid in pytest_nodeids if assigned(vehicle_pytest.test_id(nodeid))]
       ids = [test.id() for test in tests] + [vehicle_pytest.test_id(nodeid) for nodeid in pytest_nodeids]
     if len(set(ids)) != len(ids):
       errors.append("Vehicle runners selected duplicate test IDs")
@@ -724,8 +726,7 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
     gate_errors.extend(f"Required lifecycle case did not pass: {record['id']} ({record['status']})"
                        for record in records if record["status"] != "passed")
     for target in plan["targets"]:
-      prefix = Path(target).with_suffix("").as_posix().replace("/", ".") + "."
-      if not any(identity.startswith(prefix) for identity in ids):
+      if not any(lifecycle_shards.matches(identity, target) for identity in ids):
         gate_errors.append(f"Required lifecycle module collected no cases: {target}")
   elapsed = time.monotonic() - started
   exit_code = test_runner.report(records, errors + gate_errors, 10, elapsed)

tools/ci/tests/test_lifecycle_shards.py

@@ -5,6 +5,7 @@ import json
 from pathlib import Path
 import tempfile
 import unittest
+from unittest.mock import patch
 
 spec = importlib.util.spec_from_file_location('private_lifecycle_shards', Path(__file__).parents[1] / 'lifecycle_shards.py')
 assert spec is not None and spec.loader is not None
@@ -33,6 +34,46 @@ class LifecycleShardTests(unittest.TestCase):
       with self.subTest(targets=bad_targets, count=count), self.assertRaises(ValueError):
         helper.partition(bad_targets, bad_ids, count)
 
+  def test_only_large_reviewed_heavy_functions_split_into_exact_parameter_leaves(self):
+    heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'
+    prefix = heavy.removesuffix('.py').replace('/', '.') + '.'
+    full = [prefix + f'test_nominal[{index}]' for index in range(126)]
+    full += [prefix + f'test_veto[{index}]' for index in range(32)]
+    full += ['other.Test.test_one', 'other.Test.test_two']
+    targets = [heavy, 'other.py']
+    shards = helper.partition(targets, full, 4)
+    selectors = [target for shard in shards for target in shard]
+    self.assertIn(heavy + '::test_veto', selectors)
+    self.assertIn('other.py', selectors)
+    self.assertEqual(len(selectors), 128)
+    self.assertTrue(all(heavy + f'::test_nominal[{index}]' in selectors for index in range(126)))
+    selected = [[identity for identity in full if any(helper.matches(identity, target) for target in shard)] for shard in shards]
+    self.assertEqual(sorted(identity for shard in selected for identity in shard), sorted(full))
+    self.assertEqual(shards, helper.partition(list(reversed(targets)), list(reversed(full)), 4))
+    self.assertFalse(helper.matches(prefix + 'test_nominal_extra[1]', heavy + '::test_nominal'))
+    self.assertFalse(helper.matches('foreign.test_nominal[1]', heavy + '::test_nominal'))
+    self.assertTrue(helper.matches(prefix + 'test_nominal[1]', heavy + '::test_nominal[1]'))
+    self.assertFalse(helper.matches(prefix + 'test_nominal[10]', heavy + '::test_nominal[1]'))
+    self.assertTrue(helper.matches(prefix + 'test_nominal[::complex/parameter]', heavy + '::test_nominal[::complex/parameter]'))
+
+  def test_heavy_function_rule_never_splits_classes_or_accepts_forged_targets(self):
+    heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'
+    prefix = heavy.removesuffix('.py').replace('/', '.') + '.'
+    for targets, ids in (([heavy], [prefix + 'TestClass.test_case']),
+                         ([heavy + '::test_case'], [prefix + 'test_case']),
+                         ([heavy], [prefix + 'test_case', prefix + 'test_case'])):
+      with self.subTest(targets=targets), self.assertRaises(ValueError):
+        helper.partition(targets, ids, 4)
+    with self.assertRaises(ValueError):
+      helper.partition(['other.py'], ['other.test_case'], 9)
+
+  def test_future_declared_module_does_not_add_any_cases(self):
+    targets, identities = self.catalog()
+    shards = helper.partition(targets, identities, 8)
+    self.assertEqual(sorted(target for shard in shards for target in shard), targets)
+    camera = 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py'
+    self.assertFalse(any(camera in target for shard in shards for target in shard))
+
   def documents(self):
     targets, full = self.catalog()
     partition = helper.partition(targets, full, 4)
@@ -49,7 +90,7 @@ class LifecycleShardTests(unittest.TestCase):
                                                                for name, checksum in can_sources.items()})
     docs = []
     for index, assigned in enumerate(partition):
-      ids = [identity for identity in full if identity.startswith(assigned[0].removesuffix('.py') + '.')]
+      ids = [identity for identity in full if any(helper.matches(identity, target) for target in assigned)]
       doc = {'suite': 'aol-lifecycle', 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4},
              'targets': assigned, 'selected_test_ids': ids, 'full_catalog_targets': targets, 'full_catalog_test_ids': full,
              'collection_errors': [], 'gate_errors': [], 'source': source, 'source_after': source,
@@ -83,6 +124,28 @@ class LifecycleShardTests(unittest.TestCase):
   def test_complete_catalog_passes(self):
     self.assertEqual(self.check(self.documents())['cases_passed'], 12)
 
+  def test_parameter_leaf_aggregate_requires_exact_complete_passed_outcomes(self):
+    heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py'
+    prefix = heavy.removesuffix('.py').replace('/', '.') + '.'
+    full = [prefix + f'test_nominal[{index}]' for index in range(126)] + ['other.Test.test_case']
+    with patch.object(self, 'catalog', return_value=([heavy, 'other.py'], full)):
+      docs = self.documents()
+    self.assertEqual(self.check(copy.deepcopy(docs))['cases_passed'], 127)
+    for change in ('missing', 'duplicate', 'skip', 'forged_catalog', 'forged_leaf'):
+      bad = json.loads(json.dumps(docs))
+      if change == 'missing':
+        bad[0][1]['results'].pop()
+      elif change == 'duplicate':
+        bad[0][1]['results'].append(bad[0][1]['results'][0])
+      elif change == 'skip':
+        bad[0][1]['results'][0]['status'] = 'skipped'
+      elif change == 'forged_catalog':
+        bad[0][0]['full_catalog_test_ids'].pop()
+      else:
+        bad[0][0]['targets'][0] += '-forged'
+      with self.subTest(change=change), self.assertRaises((ValueError, KeyError, StopIteration)):
+        self.check(bad)
+
   def test_missing_modified_or_forged_native_artifacts_are_denied(self):
     def remove(root):
       (root / '0/libsafety-debug.so').unlink()