diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 89172892f6..45914f3c88 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -203,13 +203,6 @@ jobs: RAYLIB_BACKEND: headless STARPILOT_TEST_DIAGNOSTICS: ci-results/host-test-diagnostics/unittest run: tools/op.sh test --json-output unit-test-results.json - - name: Test custom module-level pytest cases - if: ${{ !cancelled() && steps.build.outcome == 'success' }} - timeout-minutes: 15 - env: - RAYLIB_BACKEND: headless - STARPILOT_TEST_DIAGNOSTICS: ci-results/host-test-diagnostics/pytest - run: python3 tools/ci/run_host_tests.py --pytest - name: Test GM vehicle configurations and CAN behavior if: ${{ !cancelled() && steps.build.outcome == 'success' }} timeout-minutes: 5 @@ -244,6 +237,80 @@ jobs: ci-results/host-test-diagnostics/ if-no-files-found: error + custom_pytest: + name: custom pytest (${{ matrix.shard }}) + runs-on: ubuntu-24.04 + timeout-minutes: 35 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] + steps: + - uses: actions/checkout@v7 + - name: Provide JavaScript runtime for Galaxy contract tests + uses: actions/setup-node@v7 + with: + node-version: '24' + package-manager-cache: false + - name: Provide pinned map provider toolchain + uses: actions/setup-go@v7 + with: + go-version: '1.25.1' + cache-dependency-path: mapd_repo/go.sum + - name: Fetch pinned map provider dependencies + working-directory: mapd_repo + env: + GOTOOLCHAIN: local + run: go mod download + - run: ./tools/op.sh setup + id: setup + - name: Build openpilot + id: build + run: scons + - name: Provide H.264 encoder for Android Auto integration tests + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: | + uv pip install --python .venv/bin/python --only-binary av av==19.0.1 + .venv/bin/python -c "import av; assert 'libx264' in av.codecs_available" + - name: Test every assigned custom pytest node serially + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + timeout-minutes: 15 + env: + RAYLIB_BACKEND: headless + run: .venv/bin/python -m tools.ci.custom_pytest_shards run --index "${{ matrix.shard }}" --count 24 --output "ci-results/custom-pytest-${{ matrix.shard }}" + - name: Save collection, every outcome, JUnit and durable worker diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: custom-pytest-${{ matrix.shard }} + path: ci-results/custom-pytest-${{ matrix.shard }}/ + if-no-files-found: error + + custom_pytest_complete: + name: complete custom pytest catalog + needs: custom_pytest + if: always() + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v4 + with: + pattern: custom-pytest-* + path: ci-results/custom-pytest-shards + - name: Require every case passed or audited capability-unavailable + run: python3 -m tools.ci.custom_pytest_shards aggregate --directory ci-results/custom-pytest-shards --count 24 --output ci-results/custom-pytest-complete.json + - name: Save complete custom catalog verdict + if: always() + uses: actions/upload-artifact@v4 + with: + name: custom-pytest-complete + path: ci-results/custom-pytest-complete.json + if-no-files-found: error + aol_lifecycles: name: AOL lifecycles (${{ matrix.shard }}) runs-on: ubuntu-24.04 diff --git a/prebuilt.json b/prebuilt.json index b8d9fbd594..eb5034667b 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "08228676cd7e8748025e8485f6a751f470d936973d5bf72540a5df5aff3371fc", "schema_version": 1} +{"protected_inventory_sha256": "dd1da18910ae99b2a3568583cf26b0cf205fc07012c08c24d59e63cddc9e3838", "schema_version": 1} diff --git a/tools/ci/custom_pytest_shards.py b/tools/ci/custom_pytest_shards.py new file mode 100644 index 0000000000..75cc556a7b --- /dev/null +++ b/tools/ci/custom_pytest_shards.py @@ -0,0 +1,421 @@ +"""Run serial custom pytest shards and require exact passed or audited-unavailable outcomes.""" +import argparse +from collections import Counter, defaultdict +import hashlib +import importlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +from xml.etree import ElementTree + +from tools.ci.lifecycle_shards import build_identity, same_build_inputs + +ROOT = Path(__file__).resolve().parents[2] +HEAVY_MODULES = { + 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py', + 'openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py', +} + +CAPABILITY_NODES = { + 'openpilot/starpilot/galaxy_bluetooth/vendor/ifaddr/test_ifaddr.py::test_netifaces_compatibility', + 'openpilot/starpilot/system/android_auto/tests/test_apk_identity.py::test_real_android_auto_17_6_xapk', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-32-18-0-0]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-32-18-4-2]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-32-20-3-5]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-40-18-0-6]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-32-18-0-0]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-32-18-4-2]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-32-20-3-5]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-40-18-0-6]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[False-32-18-0-0]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[False-32-18-4-2]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[False-32-20-3-5]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-18-0-0]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-18-4-2]', + 'openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-20-3-5]', +} + + +def checksum(data): + return hashlib.sha256(data).hexdigest() + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + '\n') + temporary.replace(path) + + +def policy(): + value = json.loads(Path(__file__).with_name('custom_pytest_workloads.json').read_text()) + if (value['version'] != 1 or set(value['independent_functions']) != HEAVY_MODULES or + set(value['capability_skips']) != CAPABILITY_NODES): + raise ValueError('Unreviewed custom pytest grouping policy') + if type(value['independent_max_cases']) is not int or not 1 <= value['independent_max_cases'] <= 60: + raise ValueError('Invalid independent parameter group bound') + numbers = [value['unknown_seconds_per_case'], value['independent_max_seconds']] + numbers.extend(row['seconds_per_case'] for row in value['independent_functions'].values()) + numbers.extend(row['seconds'] for row in value['modules'].values()) + if any(type(n) not in (int, float) or not math.isfinite(n) or n <= 0 for n in numbers): + raise ValueError('Invalid custom pytest workload estimate') + if any(type(row['cases']) is not int or row['cases'] < 1 or len(row['blob']) != 40 for row in value['modules'].values()): + raise ValueError('Invalid measured custom pytest workload') + return value + + +def blob_id(path): + data = path.read_bytes() + return hashlib.sha1(f'blob {len(data)}\0'.encode() + data).hexdigest() + + +def partition(catalog, count, root=ROOT): + """Whole modules, except explicitly reviewed independent heavy parameter cases.""" + if type(count) is not int or not 1 <= count <= 24 or not catalog or len(catalog) != len(set(catalog)): + raise ValueError('Empty, duplicate or invalid custom pytest catalog') + grouped = defaultdict(list) + for node in catalog: + if not isinstance(node, str) or '::' not in node: + raise ValueError('Invalid custom pytest node ID') + module = node.split('::', 1)[0] + if not module.endswith('.py') or Path(module).is_absolute() or '..' in Path(module).parts: + raise ValueError('Invalid custom pytest module path') + grouped[module].append(node) + config = policy() + units = [] + for module, nodes in sorted(grouped.items()): + if module in config['independent_functions']: + rate = config['independent_functions'][module]['seconds_per_case'] + functions = defaultdict(list) + for node in nodes: + function = node.split('::', 1)[1].split('[', 1)[0] + if not function.isidentifier() or not function.startswith('test_'): + raise ValueError('Heavy grouping requires reviewed top-level pytest functions') + functions[function].append(node) + limit = min(config['independent_max_cases'], max(1, int(config['independent_max_seconds'] // rate))) + for function, function_nodes in sorted(functions.items()): + for start in range(0, len(function_nodes), limit): + selected = sorted(function_nodes)[start:start + limit] + units.append((len(selected) * rate, module + '::' + function + ':' + str(start), selected)) + else: + rate = config['unknown_seconds_per_case'] + measured = config['modules'].get(module) + if measured and measured['cases'] == len(nodes) and blob_id(Path(root) / module) == measured['blob']: + rate = measured['seconds'] / measured['cases'] + units.append((rate * len(nodes), module, nodes)) + selected, loads = [set() for _ in range(count)], [0.0] * count + for cost, _name, nodes in sorted(units, key=lambda unit: (-unit[0], unit[1])): + index = min(range(count), key=lambda i: (loads[i], i)) + selected[index].update(nodes) + loads[index] += cost + return [[node for node in catalog if node in shard] for shard in selected], loads + + +def source_snapshot(root=ROOT): + root = Path(root) + return {'build': build_identity(root), + 'runners': {name: checksum((root / 'tools/ci' / name).read_bytes()) for name in ( + 'run_host_tests.py', 'custom_pytest_shards.py', 'custom_pytest_workloads.json', + 'pytest_progress.py', 'worker_diagnostics.py', 'lifecycle_shards.py', 'lifecycle_build_outputs.json')}} + + +REAL_XAPK = 'openpilot/.cache/android_auto/android-auto-17-6-663454-release.xapk' + + +def guard_observation(guard, root): + if guard == 'gpu_env': + return {'STARPILOT_AUTO_GL_TEST': os.environ.get('STARPILOT_AUTO_GL_TEST')} + if guard == 'real_xapk': + return {'path': REAL_XAPK, 'exists': (Path(root) / REAL_XAPK).exists()} + if guard == 'netifaces': + try: + importlib.import_module('netifaces') + except ImportError: + return {'importable': False} + return {'importable': True} + raise ValueError('Unreviewed capability guard') + + +def guard_unavailable(guard, observation): + if guard == 'gpu_env' and set(observation) == {'STARPILOT_AUTO_GL_TEST'}: + value = observation['STARPILOT_AUTO_GL_TEST'] + if value is None or isinstance(value, str): + return value != '1' + if (guard == 'real_xapk' and set(observation) == {'path', 'exists'} and + observation['path'] == REAL_XAPK and type(observation['exists']) is bool): + return not observation['exists'] + if guard == 'netifaces' and set(observation) == {'importable'} and type(observation['importable']) is bool: + return not observation['importable'] + raise ValueError('Forged capability condition evidence') + + +def capability_state(selected, root=ROOT): + inventory = policy()['capability_skips'] + state = {} + for node in selected: + if node in inventory: + row = inventory[node] + state[node] = {'source_blob': blob_id(Path(root) / node.split('::')[0]), + 'guard': row['guard'], 'observation': guard_observation(row['guard'], root)} + return state + + +def validate_capabilities(selected, state, root=ROOT): + inventory = policy()['capability_skips'] + if set(state) != set(selected) & set(inventory): + raise ValueError('Missing or forged audited capability nodes') + for node, record in state.items(): + row = inventory[node] + if set(record) != {'source_blob', 'guard', 'observation'} or record['guard'] != row['guard']: + raise ValueError('Forged audited capability guard') + if record['source_blob'] != blob_id(Path(root) / node.split('::')[0]): + raise ValueError('Audited capability source differs from checkout') + guard_unavailable(record['guard'], record['observation']) + + +def validate_outcomes(selected, reports, capabilities=None): + capabilities = {} if capabilities is None else capabilities + inventory = policy()['capability_skips'] + if not selected or set(reports) != set(selected): + raise ValueError('Missing or unexpected custom pytest outcomes') + skipped = [] + for node in selected: + phases = reports[node] + if (Counter(row['phase'] for row in phases) == Counter({'setup': 1, 'call': 1, 'teardown': 1}) and + all(row['status'] == 'passed' for row in phases)): + continue + row = inventory.get(node) + record = capabilities.get(node) + if (row is None or record is None or record['source_blob'] != row['source_blob'] or + record['guard'] != row['guard'] or not guard_unavailable(record['guard'], record['observation'])): + raise ValueError('Failed, skipped or erroneous nonexempt custom pytest node') + if Counter(report['phase'] for report in phases) != Counter({'setup': 1, 'teardown': 1}): + raise ValueError('Audited capability skip has wrong phases') + setup = next(report for report in phases if report['phase'] == 'setup') + teardown = next(report for report in phases if report['phase'] == 'teardown') + if setup['status'] != 'skipped' or setup.get('reason') != row['reason'] or teardown['status'] != 'passed': + raise ValueError('Audited capability skip has wrong reason or outcome') + skipped.append({'id': node, 'status': 'skipped', 'disposition': 'capability-unavailable', + 'reason': row['reason'], 'source_blob': row['source_blob'], 'guard': row['guard'], + 'observation': record['observation']}) + return skipped + + + +def junit_address(node): + path, bracket, parameters = node.partition('[') + names = path.split('::') + names[0] = names[0].replace('/', '.').removesuffix('.py') + names[-1] += bracket + parameters + return '.'.join(names[:-1]), names[-1] + + +def validate_junit(path, selected, skipped): + cases = ElementTree.parse(path).getroot().findall('.//testcase') + expected = {junit_address(node): node for node in selected} + if len(expected) != len(selected) or Counter((row.get('classname'), row.get('name')) for row in cases) != Counter(expected.keys()): + raise ValueError('JUnit must contain every selected node exactly once') + unavailable = {row['id']: row for row in skipped} + for case in cases: + node = expected[(case.get('classname'), case.get('name'))] + if case.find('failure') is not None or case.find('error') is not None: + raise ValueError('JUnit contains failed or erroneous custom pytest case') + skips = case.findall('skipped') + if node not in unavailable: + if skips: + raise ValueError('JUnit contains an unaudited skip') + elif (len(skips) != 1 or skips[0].get('type') != 'pytest.skip' or + skips[0].get('message') != unavailable[node]['reason'].removeprefix('Skipped: ')): + raise ValueError('JUnit audited capability reason differs from actual phase reports') + + +# Explicit pytest plugin. Selection happens after real collection, never from an invented node list. +def pytest_addoption(parser): + group = parser.getgroup('custom-pytest-shards') + group.addoption('--custom-shard-index', type=int) + group.addoption('--custom-shard-count', type=int) + group.addoption('--custom-shard-output') + + +def pytest_configure(config): + output = config.getoption('--custom-shard-output') + if output: + plugin = ShardPlugin(config) + config.pluginmanager.register(plugin, 'custom-pytest-shard-evidence') + + +class ShardPlugin: + def __init__(self, config): + self.index = config.getoption('--custom-shard-index') + self.count = config.getoption('--custom-shard-count') + self.output = Path(config.getoption('--custom-shard-output')) + self.errors = [] + self.catalog = [] + self.selected = [] + self.reports = defaultdict(list) + self.capabilities = {} + + def pytest_collectreport(self, report): + if report.failed or report.skipped: + self.errors.append({'id': report.nodeid, 'failure': str(report.longrepr)[:4096]}) + + def pytest_collection_modifyitems(self, session, config, items): + self.catalog = [item.nodeid for item in items] + assignment, loads = partition(self.catalog, self.count) + if type(self.index) is not int or not 0 <= self.index < self.count: + raise ValueError('Invalid custom pytest shard index') + self.selected = assignment[self.index] + self.capabilities = capability_state(self.selected) + selected = set(self.selected) + rejected = [item for item in items if item.nodeid not in selected] + items[:] = [item for item in items if item.nodeid in selected] + config.hook.pytest_deselected(items=rejected) + write_json(self.output / 'plan.json', {'catalog': self.catalog, 'selected': self.selected, + 'index': self.index, 'count': self.count, 'estimated_seconds': loads[self.index], + 'estimate_only': True}) + + def pytest_runtest_logreport(self, report): + row = {'phase': report.when, 'status': report.outcome, 'duration': report.duration} + if report.skipped: + row['reason'] = str(report.longrepr[2]) if isinstance(report.longrepr, tuple) and len(report.longrepr) == 3 else str(report.longrepr) + self.reports[report.nodeid].append(row) + + def pytest_sessionfinish(self, session, exitstatus): + errors = list(self.errors) + skipped = [] + try: + validate_capabilities(self.selected, self.capabilities) + skipped = validate_outcomes(self.selected, self.reports, self.capabilities) + except ValueError as exc: + errors.append({'outcomes': str(exc)}) + value = {'catalog': self.catalog, 'selected': self.selected, 'index': self.index, 'count': self.count, + 'reports': dict(self.reports), 'capabilities': self.capabilities, 'audited_skips': skipped, + 'collection_errors': self.errors, 'gate_errors': errors, + 'pytest_exit': int(exitstatus), 'status': 'completed' if not errors and int(exitstatus) == 0 else 'failed'} + write_json(self.output / 'outcomes.json', value) + if errors and int(exitstatus) == 0: + session.exitstatus = 1 + + +def run(index, count, output): + output = Path(output).resolve() + output.mkdir(parents=True, exist_ok=False) + before = source_snapshot() + write_json(output / 'source-before.json', before) + env = dict(os.environ, STARPILOT_TEST_DIAGNOSTICS=str(output / 'diagnostics')) + command = [sys.executable, str(ROOT / 'tools/ci/run_host_tests.py'), '--pytest', '-p', + 'tools.ci.custom_pytest_shards', f'--custom-shard-index={index}', f'--custom-shard-count={count}', + f'--custom-shard-output={output}'] + child = subprocess.run(command, cwd=ROOT, env=env, check=False) + after = source_snapshot() + write_json(output / 'source-after.json', after) + plan_path = output / 'plan.json' + selected = json.loads(plan_path.read_text())['selected'] if plan_path.is_file() else [] + result = {'capabilities': capability_state(selected), 'index': index, 'count': count, 'child_exit': child.returncode, 'command': command, + 'source_before': before, 'source_after': after, + 'outcomes_sha256': checksum((output / 'outcomes.json').read_bytes()) if (output / 'outcomes.json').is_file() else None, + 'junit_sha256': checksum((output / 'diagnostics/pytest.xml').read_bytes()) if (output / 'diagnostics/pytest.xml').is_file() else None, + 'plan_sha256': checksum((output / 'plan.json').read_bytes()) if (output / 'plan.json').is_file() else None, + 'selected_collection_sha256': checksum((output / 'diagnostics/pytest-collection.json').read_bytes()) + if (output / 'diagnostics/pytest-collection.json').is_file() else None} + success = child.returncode == 0 and before == after and all(result[key] for key in ( + 'outcomes_sha256', 'junit_sha256', 'plan_sha256', 'selected_collection_sha256')) + result['status'] = 'completed' if success else 'failed' + write_json(output / 'receipt.json', result) + return 0 if success else 1 + + +def aggregate(directory, count, root=ROOT): + paths = sorted(Path(directory).rglob('receipt.json')) + if len(paths) != count: + raise ValueError('Missing or duplicate custom pytest shard jobs') + expected_source = source_snapshot(root) + full = None + seen = [] + audited_skips = [] + indices = set() + for path in paths: + doc = json.loads(path.read_text()) + index = doc['index'] + if type(index) is not int or not 0 <= index < count or index in indices or doc['count'] != count: + raise ValueError('Invalid or duplicate custom pytest shard index') + indices.add(index) + if doc['status'] != 'completed' or doc['child_exit'] != 0: + raise ValueError('Failed, cancelled or incomplete custom pytest job') + before, after = doc['source_before'], doc['source_after'] + if before != after or before['runners'] != expected_source['runners']: + raise ValueError('Source or runner changed during execution') + if not same_build_inputs(before['build'], expected_source['build']): + raise ValueError('Custom pytest source HEAD, input closure or build recipe mismatch') + outcome_path = path.with_name('outcomes.json') + junit = path.parent / 'diagnostics/pytest.xml' + if not outcome_path.is_file() or checksum(outcome_path.read_bytes()) != doc['outcomes_sha256']: + raise ValueError('Missing or changed custom pytest outcomes artifact') + if not junit.is_file() or checksum(junit.read_bytes()) != doc['junit_sha256']: + raise ValueError('Missing or changed custom pytest JUnit artifact') + for name, key in (('plan.json', 'plan_sha256'), ('diagnostics/pytest-collection.json', 'selected_collection_sha256')): + artifact = path.parent / name + if not artifact.is_file() or checksum(artifact.read_bytes()) != doc[key]: + raise ValueError('Missing or changed actual collection witness') + plan = json.loads(path.with_name('plan.json').read_text()) + outcome = json.loads(outcome_path.read_text()) + if any(plan[key] != outcome[key] for key in ('index', 'count', 'catalog', 'selected')): + raise ValueError('Actual collection plan and outcomes disagree') + actual_selected = json.loads((path.parent / 'diagnostics/pytest-collection.json').read_text()) + if actual_selected != outcome['selected']: + raise ValueError('Actual pytest progress collection differs from selected outcomes') + if outcome['status'] != 'completed' or outcome['pytest_exit'] != 0 or outcome['collection_errors'] or outcome['gate_errors']: + raise ValueError('Custom pytest collection, setup or execution failed') + if outcome['index'] != index or outcome['count'] != count: + raise ValueError('Custom pytest outcome job identity differs') + catalog = outcome['catalog'] + if full is None: + full = catalog + if catalog != full: + raise ValueError('Custom pytest full catalogs disagree') + assignment, _ = partition(catalog, count, root) + if outcome['selected'] != assignment[index]: + raise ValueError('Forged custom pytest case assignment') + validate_capabilities(outcome['selected'], outcome['capabilities'], root) + if doc['capabilities'] != outcome['capabilities']: + raise ValueError('Wrapper and pytest capability observations disagree') + skipped = validate_outcomes(outcome['selected'], outcome['reports'], outcome['capabilities']) + if skipped != outcome['audited_skips']: + raise ValueError('Forged audited capability skip disposition') + validate_junit(junit, outcome['selected'], skipped) + audited_skips.extend(skipped) + seen.extend(outcome['selected']) + if not full or len(full) != len(set(full)) or Counter(seen) != Counter(full): + raise ValueError('Custom pytest aggregate must cover every collected case exactly once') + return {'status': 'completed', 'shards': count, 'cases_passed': len(full) - len(audited_skips), + 'cases_collected': len(full), 'cases_audited_skipped': len(audited_skips), 'audited_skips': audited_skips, + 'modules': len({node.split('::', 1)[0] for node in full}), + 'revision': expected_source['build']['inputs']['revision']} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('run', 'aggregate')) + parser.add_argument('--index', type=int) + parser.add_argument('--count', type=int, default=24) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--directory', type=Path) + args = parser.parse_args() + if args.action == 'run': + if args.index is None: + parser.error('--index is required') + return run(args.index, args.count, args.output) + if args.directory is None: + parser.error('--directory is required') + result = aggregate(args.directory, args.count) + write_json(args.output, result) + print(json.dumps(result)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/ci/custom_pytest_workloads.json b/tools/ci/custom_pytest_workloads.json new file mode 100644 index 0000000000..435234b8ca --- /dev/null +++ b/tools/ci/custom_pytest_workloads.json @@ -0,0 +1,1353 @@ +{ + "capability_skips": { + "openpilot/starpilot/galaxy_bluetooth/vendor/ifaddr/test_ifaddr.py::test_netifaces_compatibility": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "netifaces", + "reason": "Skipped: netifaces not installed", + "source_blob": "e0f6abb458d61bfedc6df4aa7d1983e385d941d3" + }, + "openpilot/starpilot/system/android_auto/tests/test_apk_identity.py::test_real_android_auto_17_6_xapk": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "real_xapk", + "reason": "Skipped: the user's own Android Auto XAPK is not in .cache/", + "source_blob": "a58afe41442cdbd4fa42f40da6b7596b4bf0f900" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-32-18-0-0]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-32-18-4-2]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-32-20-3-5]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[False-40-18-0-6]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-32-18-0-0]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-32-18-4-2]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-32-20-3-5]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_composed_car_frame_converts_to_the_same_picture[True-40-18-0-6]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[False-32-18-0-0]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[False-32-18-4-2]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[False-32-20-3-5]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-18-0-0]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-18-4-2]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + }, + "openpilot/starpilot/system/android_auto/tests/test_gpu_nv12.py::test_fused_projection_conversion_matches_rgba_composition[True-32-20-3-5]": { + "basis": "Existing exact capability guard, preserved from8b9338a0; skipped remains unavailable, never passed", + "guard": "gpu_env", + "reason": "Skipped: STARPILOT_AUTO_GL_TEST=1 enables tests requiring a real graphics context", + "source_blob": "d9d6f940a5c752a14e861c954c765f450a2953d6" + } + }, + "independent_functions": { + "openpilot/starpilot/car/toyota/tests/test_camera_2aa_callers.py": { + "basis": "Owner estimate164cases274s; only actual collected IDs when present", + "seconds_per_case": 2.0 + }, + "openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py": { + "basis": "Estimate from incomplete72cases213s; independent parameter cases reviewed", + "seconds_per_case": 4.0 + } + }, + "independent_max_cases": 60, + "independent_max_seconds": 240.0, + "modules": { + "openpilot/starpilot/analytics/tests/test_report.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "bbc00d21d5534d8b13d4742535d197104e1c7770", + "cases": 11, + "seconds": 0.01570105300004343 + }, + "openpilot/starpilot/analytics/tests/test_runtime.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a71636128edb525ef1a3b0f3d1737c7f903aafe0", + "cases": 16, + "seconds": 0.018713170000069113 + }, + "openpilot/starpilot/aol/tests/test_angle_brake_deadline.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "bd8707bbc3ddb9211e2b6b0ed61724c650938cab", + "cases": 72, + "seconds": 4.696376356000201 + }, + "openpilot/starpilot/aol/tests/test_angle_cancel.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "531b0a22f8f946cd0cd3c26f3dc008b62137b6a0", + "cases": 204, + "seconds": 2.1877240810023295 + }, + "openpilot/starpilot/aol/tests/test_angle_cancel_batches.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "89357c9517a304b362cdf1d9ff2b99c4dc09e90f", + "cases": 48, + "seconds": 1.7485119020011552 + }, + "openpilot/starpilot/aol/tests/test_angle_stock_cruise_transition.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "2752226cbb396e52b6d25be70d37a1fbf5db964e", + "cases": 48, + "seconds": 4.166145991999542 + }, + "openpilot/starpilot/aol/tests/test_angle_stop_fault_guards.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c89eac6a36db4ebc264dd2fe9d060f44cd25108f", + "cases": 10, + "seconds": 0.5556368480000629 + }, + "openpilot/starpilot/aol/tests/test_angle_stop_native_feedback.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "36b6f18520fc1105aba370286b280ae24b8d2170", + "cases": 48, + "seconds": 11.807243329999324 + }, + "openpilot/starpilot/aol/tests/test_availability_intent_continuation.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ba55a335e3a1954b602b5b0be20669a24e35a5c0", + "cases": 9, + "seconds": 1.3481809210001074 + }, + "openpilot/starpilot/aol/tests/test_availability_recovery_pair.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "02e486f58ae1e37ccc544998d6c609d5d22b0a56", + "cases": 24, + "seconds": 1.0712147810000943 + }, + "openpilot/starpilot/aol/tests/test_availability_timer_deferral.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "85f37d780db35f24b4262ddf776b4bdde25ddcdd", + "cases": 29, + "seconds": 1.9850080119998665 + }, + "openpilot/starpilot/aol/tests/test_canfd_location_native_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6bc748accaf305f35bff52566476202fca3d3960", + "cases": 24, + "seconds": 31.686034698999038 + }, + "openpilot/starpilot/aol/tests/test_carstate_queue.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9d7b6b7a91977e73b70d6354157b005248736b6b", + "cases": 19, + "seconds": 0.9538941670003851 + }, + "openpilot/starpilot/aol/tests/test_classic_cancel_guards.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a8ff703d759b7d46d0e6b11422bae0751447e758", + "cases": 43, + "seconds": 0.09101203100124167 + }, + "openpilot/starpilot/aol/tests/test_classic_family_button_scope.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "0e1c179df9ef42c4aba9f48cf03c6f7fe228544c", + "cases": 256, + "seconds": 0.19877483400182427 + }, + "openpilot/starpilot/aol/tests/test_classic_location_stock_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "453a6db4c6b29620ef83179f124078ec7f004cc8", + "cases": 4, + "seconds": 10.159973965000063 + }, + "openpilot/starpilot/aol/tests/test_classic_long_profiles.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b5a3f88e08d0a92ceee0a2a0fa17957252b6655c", + "cases": 125, + "seconds": 15.338653122000778 + }, + "openpilot/starpilot/aol/tests/test_classic_scc_mapped_buttons.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "7927212aae6c9ad77472ee4ab2425647ddb5377c", + "cases": 6, + "seconds": 0.003333182999881501 + }, + "openpilot/starpilot/aol/tests/test_classic_startup_neutral_replacement.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e8e15ae28d68077eb0eae58391786a7e26db3099", + "cases": 72, + "seconds": 97.60630237199962 + }, + "openpilot/starpilot/aol/tests/test_classic_stock_full_engagement.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f57801b5d0137fd4217965ba27e94dbd08131e99", + "cases": 17, + "seconds": 0.005143071000020427 + }, + "openpilot/starpilot/aol/tests/test_distracted_lateral.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c5de09266d2dc79fb4cb7321fbc70054fd87cdcc", + "cases": 15, + "seconds": 0.8913972130000047 + }, + "openpilot/starpilot/aol/tests/test_dm_lazy.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "408c57a0a47160623eeea6d3c4608cf14f6518b8", + "cases": 12, + "seconds": 0.5704981669998688 + }, + "openpilot/starpilot/aol/tests/test_early_source_handoff.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "aafbe349814dfeaf4e395ab31ca1beaefa10da6b", + "cases": 28, + "seconds": 1.7409567749991766 + }, + "openpilot/starpilot/aol/tests/test_ev6_source_deadline.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "3056884d4d819dc5b3bf571087eca65a1b7b6cf1", + "cases": 39, + "seconds": 4.475186652000957 + }, + "openpilot/starpilot/aol/tests/test_ev6_stock_parity.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "25a77d3ab3361ff51ff4f9d735cd1435b6b8b53b", + "cases": 13, + "seconds": 9.546168218000162 + }, + "openpilot/starpilot/aol/tests/test_ev9_main_withdrawal_guards.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "7ae5aa30208fb5360b23dd0408e49550ffb92c1b", + "cases": 37, + "seconds": 0.06747250999956123 + }, + "openpilot/starpilot/aol/tests/test_event_input_boundary.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "23371e98f3f7ca195d0e8a1dd42e4ac69d656997", + "cases": 16, + "seconds": 0.7582186850003154 + }, + "openpilot/starpilot/aol/tests/test_event_input_reconciliation.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "771e1320818f446400ce123f431738834e79530c", + "cases": 35, + "seconds": 2.2763796870003716 + }, + "openpilot/starpilot/aol/tests/test_expired_axis_zero_request.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "1c1eaa731198aadddac7b37e220915c521476cca", + "cases": 168, + "seconds": 0.2653669170000512 + }, + "openpilot/starpilot/aol/tests/test_final_source_handoff.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "255a3747d3a75a8363c5bdfdefe77d3c30041e2d", + "cases": 79, + "seconds": 6.026702479999585 + }, + "openpilot/starpilot/aol/tests/test_fresh_axis_engagement.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "80066741a3af77dce33239ed4b8b552af11f1f40", + "cases": 39, + "seconds": 3.0711735680001766 + }, + "openpilot/starpilot/aol/tests/test_fresh_inactive_transport.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c4b17547e8e44f40c80e748bcfe33a62a379a861", + "cases": 110, + "seconds": 0.40340294599991466 + }, + "openpilot/starpilot/aol/tests/test_fresh_lag_card_joins.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "03a53d6878c324d525aa05241b398c774055e355", + "cases": 17, + "seconds": 1.141698840999993 + }, + "openpilot/starpilot/aol/tests/test_fresh_lag_continuity.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f0a73f7a595eb4fd03161e2677b338eb55d15d80", + "cases": 19, + "seconds": 4.4917773239995995 + }, + "openpilot/starpilot/aol/tests/test_fresh_lag_renewal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e578971cab7171b943a137c1f48ab73884426027", + "cases": 8, + "seconds": 0.41426587900025424 + }, + "openpilot/starpilot/aol/tests/test_full_engagement.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "1d6e455f99987244ab0c9b4df4914c9b189561c3", + "cases": 31, + "seconds": 0.010724543000037556 + }, + "openpilot/starpilot/aol/tests/test_gm_ascm_receive_order.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "0dbb9239a035e4cbf077bbc020f5d2f7c4abb772", + "cases": 340, + "seconds": 7.9761687649995565 + }, + "openpilot/starpilot/aol/tests/test_gm_ascm_stock_low_speed.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b883bb70bafe914dc03c246597b201529d375598", + "cases": 4, + "seconds": 0.18662649299994882 + }, + "openpilot/starpilot/aol/tests/test_gm_cancel_remap.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "91eb62b6a2ca08b25fb2dfb746043dda61142d6a", + "cases": 77, + "seconds": 26.224995480999155 + }, + "openpilot/starpilot/aol/tests/test_gm_cc_lateral_minimum.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "975b3942abf8b926cc8b73a9fceae870addafe01", + "cases": 34, + "seconds": 4.086111377999828 + }, + "openpilot/starpilot/aol/tests/test_gm_diagnostic_fast_path.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "768f2c56053142efe575280d08e4253f7f7a4b9f", + "cases": 94, + "seconds": 2.142265815999849 + }, + "openpilot/starpilot/aol/tests/test_gm_diagnostic_faults.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "58fb6c01030dc61df277ac24f986260cd5b00436", + "cases": 73, + "seconds": 0.09446410099997138 + }, + "openpilot/starpilot/aol/tests/test_gm_held_brake_successor.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9eda6d81d788579577ac6c929fa7c496b69a4fbd", + "cases": 42, + "seconds": 105.58976007000115 + }, + "openpilot/starpilot/aol/tests/test_gm_held_native.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "40ca7135f297f39f65f25e18cf8c4b46e57a5b33", + "cases": 140, + "seconds": 2.2376366669986965 + }, + "openpilot/starpilot/aol/tests/test_gm_lag_episode_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "743dac21874e519b53f1f68d2fd3c0eb52284d40", + "cases": 10, + "seconds": 1.750253536999935 + }, + "openpilot/starpilot/aol/tests/test_gm_main_response.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "75a2b37b1ea3b6dd48bac26e615fbcfdcf31938f", + "cases": 26, + "seconds": 0.9047342160006338 + }, + "openpilot/starpilot/aol/tests/test_gm_native_long_withdrawal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "3377908ca76eaadbcbb09664b4966ea6dadb6e9e", + "cases": 16, + "seconds": 2.378323437998688 + }, + "openpilot/starpilot/aol/tests/test_gm_ordinary_warning.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "4f533b1fdc964f0fe0a94bfae5e6a356d682fa7b", + "cases": 24, + "seconds": 0.4101906449998296 + }, + "openpilot/starpilot/aol/tests/test_gm_parked_startup.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "1095af5a3d3087e01bf9b84b9be50916ced2e7c8", + "cases": 16, + "seconds": 32.477997968001546 + }, + "openpilot/starpilot/aol/tests/test_gm_parked_startup_guards.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "0e9b5f48ae9153036a2e4625b509c85864f5bc3e", + "cases": 51, + "seconds": 0.05308719599793221 + }, + "openpilot/starpilot/aol/tests/test_gm_pedal_order.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a00c16e5c086d516ff90555b121024a4381f269b", + "cases": 34, + "seconds": 0.4906818480003494 + }, + "openpilot/starpilot/aol/tests/test_gm_qualification_cache.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "d9009b8820e6badbf367f610d5968cd64d256d32", + "cases": 24, + "seconds": 0.07107740700030263 + }, + "openpilot/starpilot/aol/tests/test_gm_receipt_clock.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "aaa9d3571263667d1e847339fb281c1a9a9a31ac", + "cases": 30, + "seconds": 0.4403899619994718 + }, + "openpilot/starpilot/aol/tests/test_gm_receipt_clock_ipc.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a969ffd163aa34260ac1dfc09e47231cf7a896de", + "cases": 4, + "seconds": 0.08923392199972113 + }, + "openpilot/starpilot/aol/tests/test_gm_route18e_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "39f0b5cae6c29d52f4ad97d7ba0cf449c9f837a0", + "cases": 20, + "seconds": 0.96458438900072 + }, + "openpilot/starpilot/aol/tests/test_gm_route1c_receive_order.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c7e122294fe6e931d0a916d514277c6583babcde", + "cases": 54, + "seconds": 1.2040031370024735 + }, + "openpilot/starpilot/aol/tests/test_gm_source_deadline.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "5ec2f4180786d781149eb9a7032f969c726ccbbf", + "cases": 96, + "seconds": 4.565723292001849 + }, + "openpilot/starpilot/aol/tests/test_gm_startup_main.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ea89907e03507256fd9988e53a416c55b295b09e", + "cases": 67, + "seconds": 6.266599991999328 + }, + "openpilot/starpilot/aol/tests/test_gm_steering_receive_order.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "373f959893a71dfefb9269c0dd06057aa32c4030", + "cases": 45, + "seconds": 0.9887229489997935 + }, + "openpilot/starpilot/aol/tests/test_gm_successful_rearm.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "be45cef65df949a39f7a107faa8a618e11f4538c", + "cases": 16, + "seconds": 0.49868501099967943 + }, + "openpilot/starpilot/aol/tests/test_gm_volt_f1_admission.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "bf30297b79ad469c1bf4cb66674c9df6e2dad3a7", + "cases": 69, + "seconds": 4.8613181069995335 + }, + "openpilot/starpilot/aol/tests/test_gm_volt_f1_lifecycle.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "bdc9bf3a78e4fd80c3976b797b6ec89fc8866fea", + "cases": 6, + "seconds": 21.712717287000487 + }, + "openpilot/starpilot/aol/tests/test_held_override_renewal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "74519102608c73038b4afc09ca2f7c017786f847", + "cases": 28, + "seconds": 2.1453604770008496 + }, + "openpilot/starpilot/aol/tests/test_hkg_cancel_caller.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "815dc9a1ebdb7ffc49a24d992310ef39d2fc583f", + "cases": 180, + "seconds": 0.13312947200120107 + }, + "openpilot/starpilot/aol/tests/test_hkg_cancel_joined.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "74ca4d6161fa0bb1bcbd0a03ab0e1e2842b0ba10", + "cases": 15, + "seconds": 10.425348328999462 + }, + "openpilot/starpilot/aol/tests/test_hkg_independent_cancel_contract.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "60f0b91dfc3067a7ff15090e6ca192475536be75", + "cases": 38, + "seconds": 0.0246681020005326 + }, + "openpilot/starpilot/aol/tests/test_hkg_paired_cruise_buttons.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "edab544e5d92289eecca7d1b7c5d6b57c38e68f2", + "cases": 16, + "seconds": 0.20186797999917871 + }, + "openpilot/starpilot/aol/tests/test_hkg_shared_cancel_guard.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "27d0704373708c0772b0562cb3d8cb3ebb5567f0", + "cases": 35, + "seconds": 0.07043804600039039 + }, + "openpilot/starpilot/aol/tests/test_honda_brake_intent_health.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "afdf10ec5242ccefa7502ced82b5c2017a2d2387", + "cases": 55, + "seconds": 0.03712579200032451 + }, + "openpilot/starpilot/aol/tests/test_honda_panda_faults.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9c3be454c247fac6fcac8a148c3bdc1e24fe2f5c", + "cases": 135, + "seconds": 0.06450449499948263 + }, + "openpilot/starpilot/aol/tests/test_honda_published_hud.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a65621632288bb18bcd9467eddc42ce99bac86b8", + "cases": 1, + "seconds": 0.08897456799991232 + }, + "openpilot/starpilot/aol/tests/test_honda_reverse_join.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "71aa4598e2f10b0a5f05188983410ae9421a7f25", + "cases": 342, + "seconds": 0.5564633279993814 + }, + "openpilot/starpilot/aol/tests/test_host_source_lag.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "20c9c51987c259c5fbc37036dd683c0bb98105ca", + "cases": 10, + "seconds": 0.9829136180003388 + }, + "openpilot/starpilot/aol/tests/test_host_source_lag_axes.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "3be83f9a6bd509f98c492f6db2b212577ecb265d", + "cases": 6, + "seconds": 0.3131175190005706 + }, + "openpilot/starpilot/aol/tests/test_host_source_lag_history.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b2300685b09303b09d0c72b898cd56cda8494eee", + "cases": 23, + "seconds": 3.8427703780007505 + }, + "openpilot/starpilot/aol/tests/test_host_source_lag_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "98f3c0215d4d7c5af0766d61eb686bab0ac986a2", + "cases": 3, + "seconds": 0.16929447699976663 + }, + "openpilot/starpilot/aol/tests/test_host_source_lag_vetoes.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f019a665fe2ef7cafd36259dafd9462d5eef175f", + "cases": 18, + "seconds": 1.547271879000391 + }, + "openpilot/starpilot/aol/tests/test_independent_lateral_soft_takeover.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b4c26a6bb7d5c8ce4f703acb5acc34517869b118", + "cases": 4, + "seconds": 0.42748875999996017 + }, + "openpilot/starpilot/aol/tests/test_input_deadline_continuity.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e37879142c8a626fcbebe8a0a68f435ea963372a", + "cases": 24, + "seconds": 2.546252389000074 + }, + "openpilot/starpilot/aol/tests/test_input_wait_clock.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6e324008156f355209c2b8e7cd36cdbce7c69cca", + "cases": 21, + "seconds": 1.0255810019991713 + }, + "openpilot/starpilot/aol/tests/test_intent_carstate_reconciliation.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "5a7557a0f13038ff2416d33491cb7a08a9b7b27b", + "cases": 75, + "seconds": 5.005243934999044 + }, + "openpilot/starpilot/aol/tests/test_intent_drain_budget.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "0dc6bf754e8cab0c53eefa06398baa4e941ae32a", + "cases": 17, + "seconds": 0.018439479999415198 + }, + "openpilot/starpilot/aol/tests/test_intent_projection.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "931fb3da4cef1e1210db48e077221dd78c1dd663", + "cases": 34, + "seconds": 1.1918240979991879 + }, + "openpilot/starpilot/aol/tests/test_intent_publication.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "157e0f97ef8790dc4700fccc03923ccb3ee8ea37", + "cases": 68, + "seconds": 0.11740127800294431 + }, + "openpilot/starpilot/aol/tests/test_ioniq6_stock_repair.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "31dc5759907aeb7f95b80dbfeb45292302b2944e", + "cases": 5, + "seconds": 0.0826883969998562 + }, + "openpilot/starpilot/aol/tests/test_lag_takeover_alert.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ba5f367f0ae0e80e05963a8bf225966a4bb072b3", + "cases": 22, + "seconds": 0.7384297040002821 + }, + "openpilot/starpilot/aol/tests/test_late_lateral_inputs.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ec9913da3fb8c6cb57ffb0dae4841a65ce310ab1", + "cases": 42, + "seconds": 2.761263056001553 + }, + "openpilot/starpilot/aol/tests/test_location_input_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c264d8ed06c991314ed9bceadef5d94bfa9cf5a3", + "cases": 92, + "seconds": 8.268085852003423 + }, + "openpilot/starpilot/aol/tests/test_long_only_input_wait.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c8bf275ceaafa9b5c57a8748984627f9c41e6b2b", + "cases": 31, + "seconds": 2.14428290899923 + }, + "openpilot/starpilot/aol/tests/test_meb_actual_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "df93b6302bd67982eb979425514f0f8af62de2d2", + "cases": 20, + "seconds": 18.60317318799912 + }, + "openpilot/starpilot/aol/tests/test_mqb_actual_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f81e8aee26307bc3e53741cbc583805826c64571", + "cases": 8, + "seconds": 5.915249016999496 + }, + "openpilot/starpilot/aol/tests/test_mqb_direct_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "75b64914940a2de4f05d212d703676feed2767a7", + "cases": 16, + "seconds": 13.081402123999396 + }, + "openpilot/starpilot/aol/tests/test_mqb_topology_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "223cd2f9e589d33789a5f97078d0eee52ea7b882", + "cases": 24, + "seconds": 17.933131918998924 + }, + "openpilot/starpilot/aol/tests/test_mqb_topology_contract.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6bc140401de639cf1c7a3d78608645b549f4f25c", + "cases": 277, + "seconds": 0.15566354300153762 + }, + "openpilot/starpilot/aol/tests/test_native_inventory_control_cadence.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "de1d9cc81c88bdcc79e579dabb3dfc7a5831ff49", + "cases": 85, + "seconds": 0.2197536180010502 + }, + "openpilot/starpilot/aol/tests/test_native_long_withdrawal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "51e5fd0b04dceea871ecb2cfeed276c1b8636f6a", + "cases": 7, + "seconds": 1.2938219959996786 + }, + "openpilot/starpilot/aol/tests/test_native_long_withdrawal_guards.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6d9142116b4d611a825472ea748411063ff64c2f", + "cases": 57, + "seconds": 4.2747545060005905 + }, + "openpilot/starpilot/aol/tests/test_native_observation_admission.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e1ac8cac86402960aef58ee1ae2b49c0b2045253", + "cases": 50, + "seconds": 0.050205153999058894 + }, + "openpilot/starpilot/aol/tests/test_native_observation_reuse.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "00da11ebfb6f1ab2307e7162f5c7d0eb24d9b65d", + "cases": 51, + "seconds": 0.048841543999515125 + }, + "openpilot/starpilot/aol/tests/test_new_expired_combined_feedback.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "2a09b22a8f5c4cdd86e047f874fd177954b98706", + "cases": 54, + "seconds": 3.5224649109991333 + }, + "openpilot/starpilot/aol/tests/test_nexo_stock_aol.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9eff44df7be1a2e4ddda04f0ce3aeccbcbadd60f", + "cases": 6, + "seconds": 1.8618471709999085 + }, + "openpilot/starpilot/aol/tests/test_ordinary_source_availability.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "8a12742e3f35ca4ce8d775ed9a3e70cbf433c0f0", + "cases": 27, + "seconds": 5.010743771999614 + }, + "openpilot/starpilot/aol/tests/test_panda_inventory.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b65648b766871c643b2d01ebf180b2587c77eaa7", + "cases": 58, + "seconds": 1.8637142869988566 + }, + "openpilot/starpilot/aol/tests/test_panda_siren.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "4bdf9543d0d782a6d7e9c6f6823c00d08dedf387", + "cases": 124, + "seconds": 4.202911927999139 + }, + "openpilot/starpilot/aol/tests/test_pe_combined_profile_contract.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b644afb3b670a9e4cd507cbca5d02cadc184a520", + "cases": 31, + "seconds": 0.027042498999662712 + }, + "openpilot/starpilot/aol/tests/test_post_deadline_input_reconciliation.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "19f5437d1f0ec630eb4742e11ce28d08c28c4cf6", + "cases": 72, + "seconds": 4.646644738998248 + }, + "openpilot/starpilot/aol/tests/test_post_drain_source_cutoff.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "5a9ae2e3e7587c1da17f55730afe478123ff77ce", + "cases": 20, + "seconds": 1.5153750820002188 + }, + "openpilot/starpilot/aol/tests/test_recorded_temporary_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "eefe477f6b0f260e2cb68162fe0aa734a6d43064", + "cases": 33, + "seconds": 7.630824317999441 + }, + "openpilot/starpilot/aol/tests/test_resume_lateral.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a89041ec3bad0484b0738f9dd100746a5d0278fa", + "cases": 20, + "seconds": 6.926444755999228 + }, + "openpilot/starpilot/aol/tests/test_seatbelt_cold_start.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "0579fe5ae802d8ed07185c880a18515c3a587399", + "cases": 29, + "seconds": 2.045757378000644 + }, + "openpilot/starpilot/aol/tests/test_seatbelt_lateral.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "2e40805d513de28523044b61f7e7c3c049a3103f", + "cases": 50, + "seconds": 9.221335891998933 + }, + "openpilot/starpilot/aol/tests/test_shared_source_availability.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "02701ee9dda135b39cbdab27ce23279ed328ec51", + "cases": 6, + "seconds": 0.20850461499981066 + }, + "openpilot/starpilot/aol/tests/test_single_input_wait.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "d7572c43a9477b982ba30986bc24c5699f3b4efe", + "cases": 6, + "seconds": 0.18110381500059702 + }, + "openpilot/starpilot/aol/tests/test_sonata_cruise_only_withdrawal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9470e1b7b707d7b0f09add96559286a629add838", + "cases": 4, + "seconds": 5.220821042999887 + }, + "openpilot/starpilot/aol/tests/test_sonata_hybrid_cancel_scope.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "254a3fba092367386d1fc77506940403f698a518", + "cases": 40, + "seconds": 0.032421490000388076 + }, + "openpilot/starpilot/aol/tests/test_sonata_irq_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "2f1b6cc4b61fddfef9ff96d32af319d82698b1b7", + "cases": 8, + "seconds": 12.252140282000255 + }, + "openpilot/starpilot/aol/tests/test_sonata_irq_policy.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "55150bcaff43a1636812bf8127c250a7f1ec0bd9", + "cases": 39, + "seconds": 0.016377474000137227 + }, + "openpilot/starpilot/aol/tests/test_sonata_mapped_button_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "1e2d5a981c619c8502fe6da376f4a001a0405c3b", + "cases": 4, + "seconds": 6.003875502000028 + }, + "openpilot/starpilot/aol/tests/test_sonata_stock_transition.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "7ea222ad14d35e4bcd8604d7cbcedc3ea69aa015", + "cases": 4, + "seconds": 5.7069376249999095 + }, + "openpilot/starpilot/aol/tests/test_sonata_temporary_withdrawals.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a581abfcdc76df49ee9da8fec13ec95f23feddbc", + "cases": 12, + "seconds": 27.36669733299982 + }, + "openpilot/starpilot/aol/tests/test_source_pause_backlog_ageout.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "606c072265077417fe30690c52910c7feda20b5e", + "cases": 2, + "seconds": 0.36891262199992525 + }, + "openpilot/starpilot/aol/tests/test_source_pause_driver_monitoring.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9ae9a94133f450fd137814c0c631bef72c1fe49e", + "cases": 2, + "seconds": 0.1477047380001295 + }, + "openpilot/starpilot/aol/tests/test_source_pause_intent_bursts.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "afe31f78b1c6bb5efab9c2b22839e7d89b8a5361", + "cases": 7, + "seconds": 0.864470963000258 + }, + "openpilot/starpilot/aol/tests/test_source_pause_native_regrant.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e1d2f108c394c4eb0a5033bb9456f153708bc544", + "cases": 3, + "seconds": 0.15551505100029317 + }, + "openpilot/starpilot/aol/tests/test_stock_angle_pedals.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "79a3c0221ba4fab52ba4535aa81f39851b92ee71", + "cases": 36, + "seconds": 6.7943038529990645 + }, + "openpilot/starpilot/aol/tests/test_stock_angle_physical_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c4e4ed8a8347370046fe5cf7864c8d5786dde249", + "cases": 43, + "seconds": 9.202019890000429 + }, + "openpilot/starpilot/aol/tests/test_stock_eps_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a35cdc121636f4125193fb737805d108d3f4cb25", + "cases": 145, + "seconds": 0.3297414050000498 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_family.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "70f35bcdab7fe801e7518e47e085c02ea8f382d3", + "cases": 16, + "seconds": 1.375165628999639 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_fault_provenance.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "3970d1be11e7c24dfe1be4639f9d63e1e9100bb5", + "cases": 21, + "seconds": 1.0761112740003682 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_gap_delivery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a0ff4523726707ffc357b7bac20e4478fae3af36", + "cases": 12, + "seconds": 0.5945382560005328 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_hkg.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "35823ae9b379a9f3108c253597f1c52b0a355fe3", + "cases": 16, + "seconds": 0.6161071760000141 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_honda.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "21d57a569a9f8c6b2404c06d81dd8cf2d06270f1", + "cases": 9, + "seconds": 0.3575195220007572 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_honda_vetoes.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e07c3ed9b3045713f81cb369e0a9d2d8c886f8e8", + "cases": 9, + "seconds": 0.40931337699930737 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_lag_gap_sequence.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "1b745a05ad78b4d5eada1c7c3e1e6bf7c4991605", + "cases": 4, + "seconds": 0.28225010499977543 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_pcm.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "948ec9d48671ed9b7b9b4da208cc65d842f8dce6", + "cases": 33, + "seconds": 2.5134013959986987 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_pending_refusals.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "92074a88724dffbc654ec837ddda8ca794f6e637", + "cases": 9, + "seconds": 0.4949968290000015 + }, + "openpilot/starpilot/aol/tests/test_stock_lateral_receipt_refusals.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "115ecba4e337a40e5cc3f19fde440248fd1a3709", + "cases": 12, + "seconds": 1.2602012689985713 + }, + "openpilot/starpilot/aol/tests/test_stock_location_family_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "0c8d853a6692b67980ff89a0852f4113e411195b", + "cases": 125, + "seconds": 10.222362870998722 + }, + "openpilot/starpilot/aol/tests/test_stock_location_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "29a9e98a170c7e3bb9cd8583cbe861f5f3be9060", + "cases": 140, + "seconds": 0.2624793409981976 + }, + "openpilot/starpilot/aol/tests/test_temporary_lag_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "bd28a2f1f7cdf913aaf56709888b5650b4644bda", + "cases": 41, + "seconds": 4.47272300899931 + }, + "openpilot/starpilot/aol/tests/test_temporary_steering_retention.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "283920b9c09f2162a8aa285f4d7571e00b3ae22b", + "cases": 5, + "seconds": 2.45121389899964 + }, + "openpilot/starpilot/aol/tests/test_temporary_steering_sequence.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c6dd77036a44b18156586f37b708753016c8eec5", + "cases": 42, + "seconds": 32.113955324999324 + }, + "openpilot/starpilot/aol/tests/test_torque_stock_brake_guards.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c563c27f50e73c9c73cb9de90f761bc37bcdc50d", + "cases": 115, + "seconds": 7.248188423999409 + }, + "openpilot/starpilot/aol/tests/test_torque_stock_cruise_transition.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "445c9f29298160172f6bd09ce2549d647006b5cf", + "cases": 12, + "seconds": 0.8534802440003659 + }, + "openpilot/starpilot/aol/tests/test_toyota_native_long_withdrawal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "d4dca4df61385a4c8759113f296258f0fd153d11", + "cases": 11, + "seconds": 1.4291361090004102 + }, + "openpilot/starpilot/aol/tests/test_toyota_parking_brake_deadline.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f0e4cd563c562e7a48a5c8ea46e7706de7c99cc8", + "cases": 17, + "seconds": 2.0830971540010523 + }, + "openpilot/starpilot/aol/tests/test_toyota_secoc.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f8608c708a0d0a796d602b070d4549757c7d74d0", + "cases": 19, + "seconds": 0.33589995599982103 + }, + "openpilot/starpilot/aol/tests/test_toyota_secoc_joined.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "f28b340d2b670888e4be3743fd10d8de9ff36c47", + "cases": 10, + "seconds": 7.131905708999966 + }, + "openpilot/starpilot/aol/tests/test_vehicle_policy_cache.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "09f13ca27dbbba08419cc52e5d0ec562f772d1fe", + "cases": 30, + "seconds": 0.02487205699844708 + }, + "openpilot/starpilot/aol/tests/test_volkswagen.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "3578fd02305a9775cf9526615a0ff15bcbf4f79c", + "cases": 96, + "seconds": 0.13271008700030507 + }, + "openpilot/starpilot/aol/tests/test_volkswagen_main_boot.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "89a76d7e0b53ae9d45934664fe50ebc30e2b283e", + "cases": 12, + "seconds": 0.030175071999337888 + }, + "openpilot/starpilot/aol/tests/test_volkswagen_saved_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6572f91b62654d5173ab0a054ab2c38bf89caf05", + "cases": 10, + "seconds": 0.00974568899937367 + }, + "openpilot/starpilot/aol/tests/test_volt_lag_pedal_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ccde2f25dd95faca8b2846ddee0bf23f5e0ed668", + "cases": 81, + "seconds": 7.6747092149998934 + }, + "openpilot/starpilot/aol/tests/test_zero_brake_pause.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a8af56594d74be445a14283e2d8c0277b88f93ad", + "cases": 12, + "seconds": 0.003759734998766362 + }, + "openpilot/starpilot/audio/tests/test_axis_alerts.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a3ba8c8fdcb6cc5daf800c9cced6d965da6355d2", + "cases": 26, + "seconds": 0.05584578899970438 + }, + "openpilot/starpilot/audio/tests/test_builtin_sounds.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b215c70f9fad9e291e0434c9edda13452cba8242", + "cases": 11, + "seconds": 0.14225007299978643 + }, + "openpilot/starpilot/audio/tests/test_complete_alert_assets.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ae9cb58fe5f40206c91dd209e319c83cf73a71f8", + "cases": 4, + "seconds": 0.015294718999939505 + }, + "openpilot/starpilot/audio/tests/test_soundd_loop_dismissal.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "9fef3d0bab262a06a08515f72c276524e185555c", + "cases": 12, + "seconds": 0.0055887870003061835 + }, + "openpilot/starpilot/audio/tests/test_soundd_output.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "e213cdf781dc8e16e7163100eaf7aa83337258bb", + "cases": 14, + "seconds": 0.17394949000072302 + }, + "openpilot/starpilot/audio/tests/test_stock_event_parity.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "aa40f19480d54cb9efb7b3557277775ac7edf183", + "cases": 8, + "seconds": 0.017907940999521088 + }, + "openpilot/starpilot/audio/tests/test_stream_diagnostics.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a3a6b6906e5d93c4c105c4c799ef9ed89020f521", + "cases": 6, + "seconds": 0.004547201999685058 + }, + "openpilot/starpilot/bluetooth/tests/test_native_owner.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "94c2b2b8d9d8d1fd84bc5e9d272869acf09ec311", + "cases": 12, + "seconds": 0.00834470499944473 + }, + "openpilot/starpilot/bluetooth/tests/test_onroad_management.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6c12df826a66d29a462de4fee4cf01dd86abf26e", + "cases": 1, + "seconds": 0.001093809999701989 + }, + "openpilot/starpilot/bluetooth/tests/test_startup_admission.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b7cb93ece0b0fc5caba0eccaa29c891dce2edda7", + "cases": 2, + "seconds": 0.8956470970001646 + }, + "openpilot/starpilot/car/chrysler/tests/test_location_intent.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "7467e59366178f42db555a32898c1275de72c9f3", + "cases": 48, + "seconds": 5.843426561001252 + }, + "openpilot/starpilot/car/gm/tests/test_gear_events.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "09b852c14a71d07d21cbc04feaec0d2aa429498a", + "cases": 86, + "seconds": 0.05770317200131103 + }, + "openpilot/starpilot/car/gm/tests/test_radar_recovery.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "21605746e8b483dd5e24af126138c864e6b1786f", + "cases": 34, + "seconds": 0.035843196997802806 + }, + "openpilot/starpilot/car/honda/tests/test_radar_pipeline.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "16a249ded94e0639d75ab987b89735c23dc6d84d", + "cases": 4, + "seconds": 0.01250701900016793 + }, + "openpilot/starpilot/car/hyundai/tests/test_button_migration.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "6ecc45e30faae67d5a6d921040a3a45c70db73fb", + "cases": 3, + "seconds": 0.0008924179999212356 + }, + "openpilot/starpilot/car/mazda/tests/test_cancel_lateral.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "11ee4ebdf24f38ae28fb8b7f41d91480a5e46a8d", + "cases": 37, + "seconds": 0.014431757999773254 + }, + "openpilot/starpilot/car/mazda/tests/test_main_startup.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "ea3302881f26b8105f10078f5f98abde32595e85", + "cases": 25, + "seconds": 0.028806589999476273 + }, + "openpilot/starpilot/car/mazda/tests/test_reverse_join.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "69520d7ee589514e9b0859fd317b84e104fa7ef9", + "cases": 118, + "seconds": 0.11664806899830182 + }, + "openpilot/starpilot/car/nissan/tests/test_aol.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "5d3ac402c5a727b4e575fefaeec079f6b989901d", + "cases": 55, + "seconds": 2.547297542000024 + }, + "openpilot/starpilot/car/nissan/tests/test_joined.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "a37ead422ce3cf8056047e88880ca3a6377d207f", + "cases": 5, + "seconds": 3.863408480999624 + }, + "openpilot/starpilot/car/rivian/tests/test_aol_lifecycle.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "580632ad5917316082f1a452f4f893302555dd3c", + "cases": 2, + "seconds": 3.350176654000734 + }, + "openpilot/starpilot/car/rivian/tests/test_aol_native.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "c972d55c696c2179228f030649b6c6f3eed5619a", + "cases": 14, + "seconds": 0.064205106999907 + }, + "openpilot/starpilot/car/rivian/tests/test_aol_policy.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "47110f05017c16a51063d47425c1668ac8fa9de7", + "cases": 26, + "seconds": 0.00923967999983688 + }, + "openpilot/starpilot/car/subaru/tests/test_angle_tune.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b7e3f8abb2a2ab0a1ccf3c0f8e126ce49317d695", + "cases": 4, + "seconds": 12.103728422999438 + }, + "openpilot/starpilot/car/subaru/tests/test_avh.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "8bc5ddcebcb64b4c7c00b7fe273c486a1b0ae36f", + "cases": 4, + "seconds": 0.12005528000031518 + }, + "openpilot/starpilot/car/subaru/tests/test_avh_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "5485e094cb17eb5af5c696ff77f436e3ba0797c6", + "cases": 11, + "seconds": 100.80441129400083 + }, + "openpilot/starpilot/car/subaru/tests/test_forester_angle_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "74fe799b8b296cb539d0a4ea94101ac531ee04ea", + "cases": 4, + "seconds": 36.30036697600053 + }, + "openpilot/starpilot/car/subaru/tests/test_outback_angle_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "2f10a6fb2a8b2d7c1bf4a945f7a81fb40337f6b6", + "cases": 4, + "seconds": 24.017425771000035 + }, + "openpilot/starpilot/car/subaru/tests/test_outback_startup_callers.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "96a4e790193c6a2e9d3e630ab07fbcf78b118b4a", + "cases": 8, + "seconds": 30.039832143000467 + }, + "openpilot/starpilot/car/subaru/tests/test_reverse_join.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "7f85698805fea556d241fe345a8d83f1126c8d0c", + "cases": 228, + "seconds": 1.1123749099972429 + }, + "openpilot/starpilot/car/subaru/tests/test_smoothing.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "511c4ff7d97ed614f60fd3b69f63174bae0d19dd", + "cases": 33, + "seconds": 0.0886940450009206 + }, + "openpilot/starpilot/car/subaru/tests/test_smoothing_factory.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b018b04909205f6518e896e18bde26ed354d4b76", + "cases": 48, + "seconds": 3.83445302599921 + }, + "openpilot/starpilot/car/subaru/tests/test_smoothing_model_join.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "48f84ff25978cf9fdb655d5df43e1f6114bd40b0", + "cases": 6, + "seconds": 0.1519034060006561 + }, + "openpilot/starpilot/car/subaru/tests/test_smoothing_settings.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "4bd31df430f9e3653dcc804b2292fd8d47687512", + "cases": 7, + "seconds": 0.04106036899997889 + }, + "openpilot/starpilot/car/subaru/tests/test_sng.py": { + "basis": "Complete pytest phase timings in 6eb push run39923284827", + "blob": "b09fac7de5ba95c095551646f161dfaa775b4822", + "cases": 32, + "seconds": 0.03719692899835536 + }, + "openpilot/starpilot/car/subaru/tests/test_sng_callers.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "9b35ece0fa5377f4d25b7ea6ad34f21abbc6641a", + "cases": 10, + "seconds": 25.291504849000148 + }, + "openpilot/starpilot/car/subaru/tests/test_software_cruise.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "53915fc1208cb1e8eaae593f3aa11c3b1016fadd", + "cases": 26, + "seconds": 0.0727703799993833 + }, + "openpilot/starpilot/car/subaru/tests/test_software_cruise_callers.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "abe65b7992fbb50b8756506b25004a8fb8e93b36", + "cases": 2, + "seconds": 12.736984069999835 + }, + "openpilot/starpilot/car/subaru/tests/test_stock_aol_callers.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "89a4a6ebd867be8a7fbdd392f547f0e1de6c3805", + "cases": 5, + "seconds": 21.631809591000092 + }, + "openpilot/starpilot/car/subaru/tests/test_stop_start.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "474c4d5e3224404485400734349d50df2fe1b427", + "cases": 5, + "seconds": 0.07591541000010693 + }, + "openpilot/starpilot/car/subaru/tests/test_stop_start_callers.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "e0c70db67552c565e53c37da5f47cb6491dfda51", + "cases": 11, + "seconds": 26.80400199300027 + }, + "openpilot/starpilot/car/toyota/tests/test_legacy_advert_filter.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "d44351a816a5da1c40ff688f36da0ab7d7dd4efe", + "cases": 182, + "seconds": 0.1225578689983422 + }, + "openpilot/starpilot/car/toyota/tests/test_legacy_advert_lateral.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "fccb9876418b2a6c7f14583575923d7660c49d9b", + "cases": 78, + "seconds": 0.0679267290001917 + }, + "openpilot/starpilot/car/toyota/tests/test_legacy_advert_native.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "88c7294efccc57a55a65906eb7f43c1dfed196de", + "cases": 48, + "seconds": 6.749222234999706 + }, + "openpilot/starpilot/car/toyota/tests/test_legacy_stop_timer.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "14b84fe7df86965d60834fb0cb41c3ad8cca1c83", + "cases": 2, + "seconds": 0.04418585500013705 + }, + "openpilot/starpilot/car/toyota/tests/test_legacy_stop_timer_callers.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "8b5ce754e6a88fdb09e5abb6638d0e6f81e3f266", + "cases": 38, + "seconds": 70.52549054199937 + }, + "openpilot/starpilot/car/toyota/tests/test_lexus_is_caller_native.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "f85cce9c35e46610769bfcb3bac6427a22eab1a3", + "cases": 12, + "seconds": 42.31224894199954 + }, + "openpilot/starpilot/longitudinal/tests/test_volt_mode_transition.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "c4fd0b22dec68978f8ac77d2e68d2bbe3596461c", + "cases": 66, + "seconds": 0.11916299399990748 + }, + "openpilot/starpilot/tests/test_ford_ci_registration.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "8065ac29f007fc2d14ce32fe81e8e5ff456ddd6a", + "cases": 2, + "seconds": 0.007034268999973392 + }, + "openpilot/starpilot/tests/test_ford_configuration_admission.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "6cb2072d0cbfba6f852af87a5c4d9b5b26e42599", + "cases": 57, + "seconds": 0.058055562000333794 + }, + "openpilot/starpilot/tests/test_ford_configuration_parity.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "9d0a84a2918787cc7b4d4764f1d863b01e145223", + "cases": 112, + "seconds": 217.39153231700044 + }, + "openpilot/starpilot/ui/tests/test_meb_preferences.py": { + "basis": "Separate complete AOL reference from same6eb source; not custom wall-time measurement", + "blob": "58eaefa7a396f875ddf7df6aca712dfdcf4a476c", + "cases": 18, + "seconds": 0.02584481600001709 + } + }, + "reference_sha": "6eb9df96425a661afcbf1edfa27894d6e49b3706", + "unknown_seconds_per_case": 1.0, + "version": 1 +} diff --git a/tools/ci/tests/test_custom_pytest_shards.py b/tools/ci/tests/test_custom_pytest_shards.py new file mode 100644 index 0000000000..efdf7f4b84 --- /dev/null +++ b/tools/ci/tests/test_custom_pytest_shards.py @@ -0,0 +1,554 @@ +"""Pure tests for custom pytest selection, durable outcomes and mandatory completion.""" +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +from xml.etree import ElementTree + +import pytest + +from tools.ci import custom_pytest_shards as shards + + +def source(): + return {'build': {'inputs': {'revision': 'commit'}, 'declared_outputs': {}}, 'runners': {'runner': 'a' * 64}} + + +def reports(nodes): + return {node: [{'phase': phase, 'status': 'passed', 'duration': .01} for phase in ('setup', 'call', 'teardown')] for node in nodes} + + + +def save_junit(path, selected, skipped): + root = ElementTree.Element('testsuites') + suite = ElementTree.SubElement(root, 'testsuite') + by_node = {row['id']: row for row in skipped} + for node in selected: + classname, name = shards.junit_address(node) + case = ElementTree.SubElement(suite, 'testcase', classname=classname, name=name) + if node in by_node: + ElementTree.SubElement(case, 'skipped', type='pytest.skip', message=by_node[node]['reason'].removeprefix('Skipped: ')) + path.parent.mkdir(parents=True, exist_ok=True) + ElementTree.ElementTree(root).write(path) + + +def save_job(directory, index, count, catalog, selected): + root = directory / str(index) + root.mkdir(parents=True) + outcome = {'index': index, 'count': count, 'catalog': catalog, 'selected': selected, 'reports': reports(selected), + 'status': 'completed', 'pytest_exit': 0, 'collection_errors': [], 'gate_errors': [], 'capabilities': {}, 'audited_skips': []} + shards.write_json(root / 'outcomes.json', outcome) + shards.write_json(root / 'plan.json', {key: outcome[key] for key in ('index', 'count', 'catalog', 'selected')}) + shards.write_json(root / 'diagnostics/pytest-collection.json', selected) + save_junit(root / 'diagnostics/pytest.xml', selected, []) + receipt = {'capabilities': {}, 'index': index, 'count': count, 'status': 'completed', 'child_exit': 0, + 'source_before': source(), 'source_after': source(), + 'outcomes_sha256': shards.checksum((root / 'outcomes.json').read_bytes()), + 'junit_sha256': shards.checksum((root / 'diagnostics/pytest.xml').read_bytes()), + 'plan_sha256': shards.checksum((root / 'plan.json').read_bytes()), + 'selected_collection_sha256': shards.checksum((root / 'diagnostics/pytest-collection.json').read_bytes())} + shards.write_json(root / 'receipt.json', receipt) + return root + + +@pytest.fixture +def jobs(tmp_path, monkeypatch): + monkeypatch.setattr(shards, 'source_snapshot', lambda root=shards.ROOT: source()) + monkeypatch.setattr(shards, 'same_build_inputs', lambda a, b: a == b) + catalog = ['a.py::test_a[1]', 'a.py::test_a[10]', 'b.py::test_b'] + assigned, _ = shards.partition(catalog, 2) + paths = [save_job(tmp_path, index, 2, catalog, selected) for index, selected in enumerate(assigned)] + return tmp_path, paths, catalog + + +def rewrite(path, name, change): + target = path / name + value = json.loads(target.read_text()) + change(value) + shards.write_json(target, value) + if name == 'outcomes.json': + receipt = json.loads((path / 'receipt.json').read_text()) + receipt['outcomes_sha256'] = shards.checksum(target.read_bytes()) + shards.write_json(path / 'receipt.json', receipt) + + +def test_complete_exact_catalog(jobs): + directory, _, catalog = jobs + result = shards.aggregate(directory, 2) + assert result['cases_passed'] == len(catalog) + assert result['shards'] == 2 + + +@pytest.mark.parametrize('mutation', [ + lambda d: d.update(status='failed'), + lambda d: d.update(status='cancelled'), + lambda d: d.update(child_exit=1), + lambda d: d.update(count=3), + lambda d: d.update(index=1), + lambda d: d['source_after']['runners'].update(runner='b' * 64), + lambda d: d['source_before']['runners'].update(runner='b' * 64), + lambda d: d['source_before']['build']['inputs'].update(revision='other'), + lambda d: d.update(outcomes_sha256='0' * 64), + lambda d: d.update(junit_sha256='0' * 64), +]) +def test_failed_cancelled_forged_or_changed_job_denied(jobs, mutation): + directory, paths, _ = jobs + rewrite(paths[0], 'receipt.json', mutation) + with pytest.raises(ValueError): + shards.aggregate(directory, 2) + + +@pytest.mark.parametrize('name', ['receipt.json', 'outcomes.json', 'diagnostics/pytest.xml', 'plan.json', 'diagnostics/pytest-collection.json']) +def test_missing_job_or_artifact_denied(jobs, name): + directory, paths, _ = jobs + (paths[0] / name).unlink() + with pytest.raises(ValueError): + shards.aggregate(directory, 2) + + +@pytest.mark.parametrize('mutation', [ + lambda d: d.update(status='failed'), + lambda d: d.update(pytest_exit=2), + lambda d: d['collection_errors'].append({'id': 'bad.py'}), + lambda d: d['gate_errors'].append({'error': 'setup'}), + lambda d: d['catalog'].append('invented.py::test_fake'), + lambda d: d['catalog'].append(d['catalog'][0]), + lambda d: d['selected'].append(d['selected'][0]), + lambda d: d.update(selected=[]), + lambda d: d['reports'].clear(), + lambda d: next(iter(d['reports'].values())).pop(), + lambda d: next(iter(d['reports'].values())).append({'phase': 'call', 'status': 'passed'}), + lambda d: next(iter(d['reports'].values()))[1].update(status='skipped'), + lambda d: next(iter(d['reports'].values()))[1].update(status='failed'), + lambda d: next(iter(d['reports'].values()))[0].update(status='failed'), + lambda d: next(iter(d['reports'].values()))[2].update(status='failed'), +]) +def test_collection_assignment_outcome_errors_denied(jobs, mutation): + directory, paths, _ = jobs + rewrite(paths[0], 'outcomes.json', mutation) + with pytest.raises(ValueError): + shards.aggregate(directory, 2) + + +def test_duplicate_job_denied(jobs): + directory, paths, _ = jobs + import shutil + shutil.copytree(paths[0], directory / 'duplicate') + with pytest.raises(ValueError): + shards.aggregate(directory, 2) + + +def test_changed_source_in_all_jobs_denied_against_checkout(jobs): + directory, paths, _ = jobs + for path in paths: + def change(d): + for name in ('source_before', 'source_after'): + d[name]['build']['inputs']['revision'] = 'forged' + rewrite(path, 'receipt.json', change) + with pytest.raises(ValueError, match='source HEAD'): + shards.aggregate(directory, 2) + + +def test_exact_parameter_nodes_whole_other_modules_and_bounded_heavy_groups(): + heavy = 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py' + catalog = [f'{heavy}::test_independent[{i}]' for i in range(126)] + ['other.py::test_case[1]', 'other.py::test_case[10]'] + assigned, loads = shards.partition(catalog, 24) + assert sorted(node for group in assigned for node in group) == sorted(catalog) + assert max(loads) <= 240 + assert any('other.py::test_case[1]' in group and 'other.py::test_case[10]' in group for group in assigned) + assert [sum(node.startswith(heavy) for node in group) for group in assigned].count(60) == 2 + assert shards.partition(catalog, 24) == (assigned, loads) + + +@pytest.mark.parametrize('catalog', [[], ['a.py::test_a', 'a.py::test_a'], ['../a.py::test_a'], ['/a.py::test_a'], ['a.py']]) +def test_invalid_catalog_denied(catalog): + with pytest.raises(ValueError): + shards.partition(catalog, 24) + + +def test_unreviewed_heavy_class_not_split(): + with pytest.raises(ValueError, match='top-level'): + shards.partition(['openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py::TestShared::test_x'], 24) + + +def test_changed_source_invalidates_measured_duration(tmp_path, monkeypatch): + p = tmp_path / 'a.py' + p.write_text('original') + config = copy.deepcopy(shards.policy()) + config['modules'] = {'a.py': {'cases': 1, 'seconds': 99., 'blob': shards.blob_id(p)}} + monkeypatch.setattr(shards, 'policy', lambda: config) + assert shards.partition(['a.py::test_a'], 1, tmp_path)[1] == [99.] + p.write_text('changed') + assert shards.partition(['a.py::test_a'], 1, tmp_path)[1] == [1.] + + +@pytest.mark.parametrize('body,expected', [ + ('def test_a():\n assert True\ndef test_b():\n assert True\n', 'completed'), + ('def test_a():\n assert False\ndef test_b():\n assert True\n', 'failed'), + ('import pytest\n@pytest.mark.skip(reason="visible skip")\ndef test_a():\n pass\ndef test_b():\n assert True\n', 'failed'), + ('raise RuntimeError("collection failure")\n', 'failed'), +]) +def test_real_pytest_plugin_durable_results(tmp_path, body, expected): + (tmp_path / 'test_cases.py').write_text(body) + output = tmp_path / 'evidence' + env = dict(os.environ, PYTHONPATH=str(shards.ROOT), PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', + STARPILOT_TEST_DIAGNOSTICS=str(output / 'diagnostics')) + command = [sys.executable, '-m', 'pytest', '-q', '-p', 'tools.ci.custom_pytest_shards', + '-p', 'tools.ci.pytest_progress', '--custom-shard-index=0', '--custom-shard-count=1', + f'--custom-shard-output={output}', f'--junitxml={output}/diagnostics/pytest.xml', 'test_cases.py'] + child = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, timeout=20, check=False) + outcome = json.loads((output / 'outcomes.json').read_text()) + assert outcome['status'] == expected, child.stdout.decode() + child.stderr.decode() + assert (child.returncode == 0) == (expected == 'completed') + assert (output / 'diagnostics/pytest.xml').is_file() + if expected == 'completed': + assert len(outcome['catalog']) == len(outcome['selected']) == 2 + assert list((output / 'diagnostics').glob('worker-*.jsonl')) + + +def test_real_child_death_leaves_failed_receipt_and_durable_node(tmp_path, monkeypatch): + wrapper = tmp_path / 'tools/ci/run_host_tests.py' + wrapper.parent.mkdir(parents=True) + wrapper.write_bytes((Path(__file__).parents[1] / 'run_host_tests.py').read_bytes()) + module = tmp_path / 'openpilot/starpilot/tests/test_signal.py' + module.parent.mkdir(parents=True) + module.write_text('import os, signal\ndef test_child_signal():\n os.kill(os.getpid(), signal.SIGTERM)\n') + monkeypatch.setattr(shards, 'ROOT', tmp_path) + monkeypatch.setattr(shards, 'source_snapshot', lambda: source()) + monkeypatch.setenv('PYTHONPATH', str(Path(__file__).parents[3])) + output = tmp_path / 'evidence' + assert shards.run(0, 1, output) == 1 + receipt = json.loads((output / 'receipt.json').read_text()) + assert receipt['status'] == 'failed' + assert receipt['child_exit'] == 143 + assert receipt['outcomes_sha256'] is None + workers = list((output / 'diagnostics').glob('worker-*.json')) + assert workers + assert any(json.loads(path.read_text())['active_test'].endswith('::test_child_signal') for path in workers) + + +def test_real_plugin_exact_parameter_assignment_across_shards(tmp_path): + path = tmp_path / 'openpilot/starpilot/car/toyota/tests/test_legacy_advert_callers.py' + path.parent.mkdir(parents=True) + path.write_text('import pytest\n@pytest.mark.parametrize("value", range(126))\ndef test_independent(value):\n assert 0 <= value < 126\n') + outcomes = [] + for index in range(3): + output = tmp_path / f'shard-{index}' + env = dict(os.environ, PYTHONPATH=str(shards.ROOT), PYTEST_DISABLE_PLUGIN_AUTOLOAD='1') + command = [sys.executable, '-m', 'pytest', '-q', '-p', 'tools.ci.custom_pytest_shards', + f'--custom-shard-index={index}', '--custom-shard-count=3', f'--custom-shard-output={output}', str(path)] + child = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, timeout=20, check=False) + assert child.returncode == 0, child.stdout.decode() + child.stderr.decode() + outcomes.append(json.loads((output / 'outcomes.json').read_text())) + seen = [node for outcome in outcomes for node in outcome['selected']] + assert len(seen) == len(set(seen)) == 126 + assert set(seen) == set(outcomes[0]['catalog']) + for outcome in outcomes: + shards.validate_outcomes(outcome['selected'], outcome['reports']) + + +def test_forged_catalog_in_all_outcomes_denied_by_actual_collection_witness(jobs): + directory, paths, _ = jobs + for path in paths: + rewrite(path, 'outcomes.json', lambda d: d['catalog'].append('invented.py::test_fake')) + with pytest.raises(ValueError, match='collection plan'): + shards.aggregate(directory, 2) + + +def test_changed_progress_collection_denied(jobs): + directory, paths, _ = jobs + shards.write_json(paths[0] / 'diagnostics/pytest-collection.json', ['invented.py::test_fake']) + with pytest.raises(ValueError, match='collection witness'): + shards.aggregate(directory, 2) + + +def test_real_wrapper_shards_and_aggregate_keep_every_case(tmp_path, monkeypatch): + wrapper = tmp_path / 'tools/ci/run_host_tests.py' + wrapper.parent.mkdir(parents=True) + wrapper.write_bytes((Path(__file__).parents[1] / 'run_host_tests.py').read_bytes()) + tests = tmp_path / 'openpilot/starpilot/tests' + tests.mkdir(parents=True) + for name in ('a', 'b'): + (tests / f'test_{name}.py').write_text(f'def test_{name}():\n assert True\n') + monkeypatch.setattr(shards, 'ROOT', tmp_path) + monkeypatch.setattr(shards, 'source_snapshot', lambda root=tmp_path: source()) + monkeypatch.setattr(shards, 'same_build_inputs', lambda a, b: a == b) + monkeypatch.setenv('PYTHONPATH', str(Path(__file__).parents[3])) + for index in range(2): + assert shards.run(index, 2, tmp_path / 'jobs' / str(index)) == 0 + result = shards.aggregate(tmp_path / 'jobs', 2) + assert result['cases_passed'] == 2 + assert result['status'] == 'completed' + + +def test_real_module_collection_skip_denied_even_when_other_cases_pass(tmp_path): + (tmp_path / 'test_skip.py').write_text('import pytest\npytest.skip("module skip must remain failure", allow_module_level=True)\n') + (tmp_path / 'test_pass.py').write_text('def test_pass():\n assert True\n') + output = tmp_path / 'evidence' + env = dict(os.environ, PYTHONPATH=str(shards.ROOT), PYTEST_DISABLE_PLUGIN_AUTOLOAD='1') + child = subprocess.run([sys.executable, '-m', 'pytest', '-q', '-p', 'tools.ci.custom_pytest_shards', + '--custom-shard-index=0', '--custom-shard-count=1', f'--custom-shard-output={output}', + 'test_skip.py', 'test_pass.py'], cwd=tmp_path, env=env, capture_output=True, timeout=20, check=False) + outcome = json.loads((output / 'outcomes.json').read_text()) + assert child.returncode != 0 + assert outcome['status'] == 'failed' + assert outcome['collection_errors'][0]['id'] == 'test_skip.py' + assert len(outcome['selected']) == 1 + shards.validate_outcomes(outcome['selected'], outcome['reports']) + + + +def unavailable_observation(guard): + return {'gpu_env': {'STARPILOT_AUTO_GL_TEST': None}, 'netifaces': {'importable': False}, + 'real_xapk': {'path': shards.REAL_XAPK, 'exists': False}}[guard] + + +def skipped_reports(node, reason): + return {node: [{'phase': 'setup', 'status': 'skipped', 'duration': .01, 'reason': reason}, + {'phase': 'teardown', 'status': 'passed', 'duration': .01}]} + + +@pytest.fixture +def capability_guards(monkeypatch): + inventory = shards.policy()['capability_skips'] + blobs = {node.split('::')[0]: row['source_blob'] for node, row in inventory.items()} + monkeypatch.setattr(shards, 'blob_id', lambda path: blobs[path.relative_to(shards.ROOT).as_posix()]) + monkeypatch.setattr(shards, 'guard_observation', lambda guard, root: unavailable_observation(guard)) + return inventory + + +@pytest.mark.parametrize('node', sorted(shards.CAPABILITY_NODES)) +def test_each_original_capability_skip_remains_unavailable_never_passed(capability_guards, node): + row = capability_guards[node] + state = shards.capability_state([node]) + shards.validate_capabilities([node], state) + result = shards.validate_outcomes([node], skipped_reports(node, row['reason']), state) + assert result == [{'id': node, 'status': 'skipped', 'disposition': 'capability-unavailable', + 'reason': row['reason'], 'source_blob': row['source_blob'], 'guard': row['guard'], + 'observation': unavailable_observation(row['guard'])}] + assert shards.validate_outcomes([node], reports([node]), state) == [] + + +@pytest.mark.parametrize('node', sorted(shards.CAPABILITY_NODES)) +def test_each_capability_wrong_reason_or_source_denied(capability_guards, node): + row = capability_guards[node] + state = shards.capability_state([node]) + with pytest.raises(ValueError): + shards.validate_outcomes([node], skipped_reports(node, row['reason'] + ' changed'), state) + state[node]['source_blob'] = '0' * 40 + with pytest.raises(ValueError): + shards.validate_outcomes([node], skipped_reports(node, row['reason']), state) + with pytest.raises(ValueError): + shards.validate_capabilities([node], state) + + +@pytest.mark.parametrize('node', sorted(shards.CAPABILITY_NODES)) +def test_each_available_capability_cannot_be_skipped(capability_guards, node): + row = capability_guards[node] + state = shards.capability_state([node]) + observation = {'gpu_env': {'STARPILOT_AUTO_GL_TEST': '1'}, 'netifaces': {'importable': True}, + 'real_xapk': {'path': shards.REAL_XAPK, 'exists': True}}[row['guard']] + state[node]['observation'] = observation + with pytest.raises(ValueError): + shards.validate_outcomes([node], skipped_reports(node, row['reason']), state) + + +def test_forged_capability_node_and_condition_denied(capability_guards): + node = sorted(capability_guards)[0] + state = shards.capability_state([node]) + with pytest.raises(ValueError): + shards.validate_outcomes([node + '-forged'], skipped_reports(node + '-forged', capability_guards[node]['reason']), state) + state[node]['observation'] = {'unavailable': True} + with pytest.raises(ValueError): + shards.validate_capabilities([node], state) + + +def test_audited16_counts_are_skipped_not_passed_in_exact_aggregate(tmp_path, monkeypatch, capability_guards): + monkeypatch.setattr(shards, 'source_snapshot', lambda root=shards.ROOT: source()) + monkeypatch.setattr(shards, 'same_build_inputs', lambda a, b: a == b) + catalog = sorted(capability_guards) + ['required.py::test_required'] + assigned, _ = shards.partition(catalog, 2) + for index, selected in enumerate(assigned): + path = save_job(tmp_path, index, 2, catalog, selected) + state = shards.capability_state(selected) + phase_reports = reports(selected) + for node in state: + phase_reports.update(skipped_reports(node, capability_guards[node]['reason'])) + skipped = shards.validate_outcomes(selected, phase_reports, state) + rewrite(path, 'outcomes.json', lambda d, phases=phase_reports, caps=state, skips=skipped: + d.update(reports=phases, capabilities=caps, audited_skips=skips)) + save_junit(path / 'diagnostics/pytest.xml', selected, skipped) + junit_hash = shards.checksum((path / 'diagnostics/pytest.xml').read_bytes()) + rewrite(path, 'receipt.json', lambda d, caps=state, digest=junit_hash: d.update(capabilities=caps, junit_sha256=digest)) + result = shards.aggregate(tmp_path, 2) + assert result['cases_collected'] == 17 + assert result['cases_passed'] == 1 + assert result['cases_audited_skipped'] == 16 + assert len(result['audited_skips']) == 16 + assert all(row['status'] == 'skipped' for row in result['audited_skips']) + + +def test_actual_capability_guard_conditions(tmp_path, monkeypatch): + monkeypatch.delenv('STARPILOT_AUTO_GL_TEST', raising=False) + assert shards.guard_unavailable('gpu_env', shards.guard_observation('gpu_env', tmp_path)) + monkeypatch.setenv('STARPILOT_AUTO_GL_TEST', '1') + assert not shards.guard_unavailable('gpu_env', shards.guard_observation('gpu_env', tmp_path)) + assert shards.guard_unavailable('real_xapk', shards.guard_observation('real_xapk', tmp_path)) + apk = tmp_path / shards.REAL_XAPK + apk.parent.mkdir(parents=True) + apk.write_bytes(b'presence-only guard witness; not a real APK') + assert not shards.guard_unavailable('real_xapk', shards.guard_observation('real_xapk', tmp_path)) + def unavailable(name): + assert name == 'netifaces' + raise ImportError('optional dependency absent') + monkeypatch.setattr(shards.importlib, 'import_module', unavailable) + assert shards.guard_unavailable('netifaces', shards.guard_observation('netifaces', tmp_path)) + monkeypatch.setattr(shards.importlib, 'import_module', lambda name: object()) + assert not shards.guard_unavailable('netifaces', shards.guard_observation('netifaces', tmp_path)) + + + +@pytest.mark.parametrize('change', ['missing', 'duplicate', 'skip', 'failure', 'error']) +def test_junit_false_success_and_unaudited_skips_denied(jobs, change): + directory, paths, _ = jobs + path = paths[0] / 'diagnostics/pytest.xml' + tree = ElementTree.parse(path) + suite = tree.getroot().find('testsuite') + assert suite is not None + case = suite.find('testcase') + assert case is not None + if change == 'missing': + suite.remove(case) + elif change == 'duplicate': + suite.append(copy.deepcopy(case)) + else: + ElementTree.SubElement(case, 'skipped' if change == 'skip' else change) + tree.write(path) + rewrite(paths[0], 'receipt.json', lambda d: d.update(junit_sha256=shards.checksum(path.read_bytes()))) + with pytest.raises(ValueError, match='JUnit'): + shards.aggregate(directory, 2) + + +def test_real_pytest_all16_audited_guards_have_skipped_junit_not_passes(tmp_path): + """Real pytest phases/JUnit with controlled guard witnesses; no graphics or private APK is tested.""" + import shutil + package = tmp_path / 'tools/ci' + package.mkdir(parents=True) + (tmp_path / 'tools/__init__.py').touch() + (package / '__init__.py').touch() + for name in ('custom_pytest_shards.py', 'lifecycle_shards.py', 'pytest_progress.py', 'worker_diagnostics.py'): + shutil.copyfile(shards.ROOT / 'tools/ci' / name, package / name) + config = shards.policy() + by_module = {} + for node in sorted(shards.CAPABILITY_NODES): + by_module.setdefault(node.split('::')[0], []).append(node) + for module, nodes in by_module.items(): + source_path = tmp_path / module + source_path.parent.mkdir(parents=True, exist_ok=True) + lines = ['import pytest', 'from unittest import SkipTest'] + groups = {} + for node in nodes: + tail = node.split('::', 1)[1] + function, bracket, parameter = tail.partition('[') + groups.setdefault(function, []).append(parameter.removesuffix(']') if bracket else None) + for function, parameters in groups.items(): + reason = config['capability_skips'][nodes[0]]['reason'].removeprefix('Skipped: ') + if parameters != [None]: + lines.append(f'@pytest.mark.parametrize("case", range({len(parameters)}), ids={parameters!r})') + signature = 'case' + else: + signature = '' + lines.extend([f'def {function}({signature}):', f' raise SkipTest({reason!r})']) + # The original guards skip during setup; a fixture models that exact phase. + lines = ['import pytest', 'from unittest import SkipTest', '@pytest.fixture(autouse=True)', + 'def original_capability_guard():', f' raise SkipTest({reason!r})'] + [ + line.replace(f' raise SkipTest({reason!r})', ' assert False, "body must not run when original guard is unavailable"') + for line in lines[2:]] + source_path.write_text('\n'.join(lines) + '\n') + for node in nodes: + config['capability_skips'][node]['source_blob'] = shards.blob_id(source_path) + shards.write_json(package / 'custom_pytest_workloads.json', config) + (tmp_path / 'guard_witness.py').write_text('''def pytest_configure(config): + from tools.ci import custom_pytest_shards as s + def observe(guard, root): + return {'gpu_env': {'STARPILOT_AUTO_GL_TEST': None}, 'netifaces': {'importable': False}, + 'real_xapk': {'path': s.REAL_XAPK, 'exists': False}}[guard] + s.guard_observation = observe +''') + output = tmp_path / 'evidence' + env = dict(os.environ, PYTHONPATH=str(tmp_path), PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', + STARPILOT_TEST_DIAGNOSTICS=str(output / 'diagnostics')) + child = subprocess.run([sys.executable, '-m', 'pytest', '-q', '-p', 'tools.ci.custom_pytest_shards', + '-p', 'tools.ci.pytest_progress', '-p', 'guard_witness', '--custom-shard-index=0', + '--custom-shard-count=1', f'--custom-shard-output={output}', + f'--junitxml={output}/diagnostics/pytest.xml', *by_module], cwd=tmp_path, + env=env, capture_output=True, timeout=20, check=False) + outcome = json.loads((output / 'outcomes.json').read_text()) + assert child.returncode == 0, child.stdout.decode() + child.stderr.decode() + assert outcome['status'] == 'completed' + assert len(outcome['catalog']) == len(outcome['audited_skips']) == 16 + assert all(row['status'] == 'skipped' for row in outcome['audited_skips']) + shards.validate_junit(output / 'diagnostics/pytest.xml', outcome['selected'], outcome['audited_skips']) + progress = json.loads((output / 'diagnostics/pytest-collection.json').read_text()) + assert progress == outcome['selected'] + + +def test_workflow_uses_test_interpreter_for_helper_and_actual_wrapper(tmp_path): + """Two real Python environments; optional module exists only in configured test environment.""" + import venv + project = tmp_path / 'project' + test_env = project / '.venv' + parent_env = tmp_path / 'parent-python' + builder = venv.EnvBuilder(with_pip=False, symlinks=True) + builder.create(test_env) + builder.create(parent_env) + if os.name == 'nt': + pytest.fail('The hosted custom shard workflow requires Linux Python paths') + test_python = test_env / 'bin/python' + parent_python = parent_env / 'bin/python' + probe_file = tmp_path / 'worker-observation.json' + for interpreter in (test_python, parent_python): + result = subprocess.check_output([str(interpreter), '-c', 'import sysconfig; print(sysconfig.get_path("purelib"))'], text=True) + site = Path(result.strip()) + site.mkdir(parents=True, exist_ok=True) + # This probe module records interpreter/capability only; it does not claim to execute pytest cases. + (site / 'pytest.py').write_text('''import json, os, sys +from pathlib import Path +from tools.ci import custom_pytest_shards as s +Path(os.environ['OPTIONAL_PROBE']).write_text(json.dumps({ + 'python': sys.executable, 'guard': s.guard_observation('netifaces', Path.cwd())})) +''') + if interpreter == test_python: + (site / 'netifaces.py').write_text('"""Pure optional-dependency witness; no native package imported."""\n') + wrapper = project / 'tools/ci/run_host_tests.py' + wrapper.parent.mkdir(parents=True) + wrapper.write_bytes((shards.ROOT / 'tools/ci/run_host_tests.py').read_bytes()) + test_file = project / 'openpilot/starpilot/tests/test_probe.py' + test_file.parent.mkdir(parents=True) + test_file.write_text('def test_probe():\n assert True\n') + probe = tmp_path / 'parent-probe.py' + probe.write_text('''import json, os, subprocess, sys +from pathlib import Path +from tools.ci import custom_pytest_shards as s +project = Path(sys.argv[1]) +parent = {'python': sys.executable, 'guard': s.guard_observation('netifaces', project)} +subprocess.run([sys.executable, str(project / 'tools/ci/run_host_tests.py'), '--pytest'], cwd=project, check=True) +worker = json.loads(Path(os.environ['OPTIONAL_PROBE']).read_text()) +print(json.dumps({'parent': parent, 'worker': worker})) +''') + environment = dict(os.environ, PYTHONPATH=str(shards.ROOT), OPTIONAL_PROBE=str(probe_file)) + environment.pop('PYTHONHOME', None) + wrong = json.loads(subprocess.check_output([str(parent_python), str(probe), str(project)], env=environment, text=True, timeout=20)) + assert wrong['parent']['guard'] == {'importable': False} + workflow = (shards.ROOT / '.github/workflows/tests.yaml').read_text() + run_line = next(line for line in workflow.splitlines() if 'tools.ci.custom_pytest_shards run ' in line) + interpreter = run_line.split('run: ', 1)[1].split()[0] + assert interpreter == '.venv/bin/python' + chosen = json.loads(subprocess.check_output([str(project / interpreter), str(probe), str(project)], env=environment, text=True, timeout=20)) + assert chosen['parent']['guard'] == chosen['worker']['guard'] == {'importable': True} + assert chosen['parent']['python'] == chosen['worker']['python'] == str(test_python) + aggregate_line = next(line for line in workflow.splitlines() if 'tools.ci.custom_pytest_shards aggregate ' in line) + assert aggregate_line.split('run: ', 1)[1].startswith('python3 -m ') + assert 'passed or audited capability-unavailable' in workflow