diff --git a/openpilot/selfdrive/modeld/modeld.py b/openpilot/selfdrive/modeld/modeld.py index 43af047f2a..daf7bae493 100755 --- a/openpilot/selfdrive/modeld/modeld.py +++ b/openpilot/selfdrive/modeld/modeld.py @@ -35,7 +35,7 @@ from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.hardware.usb import cable_connected from openpilot.common.filter_simple import FirstOrderFilter -from openpilot.common.realtime import config_realtime_process, DT_MDL +from openpilot.common.realtime import config_realtime_process, set_core_affinity, DT_MDL from openpilot.common.transformations.camera import DEVICE_CAMERAS from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.common.transformations.model import get_warp_matrix @@ -374,6 +374,7 @@ def main(demo=False): wait_for_chestnut() if not demo: wait_for_chestnut_power(CP, timeout=min(30., BIG_MODEL_TIMEOUT / 2)) + set_core_affinity([7]) if selection.big_path is not None: m, prepared = load_verified_model(vipc_client_main.width, vipc_client_main.height, big_path, selection.big_version, True, selection.big_id, selection.big_sha256) diff --git a/openpilot/starpilot/flm/producer_clock_equivalence.py b/openpilot/starpilot/flm/producer_clock_equivalence.py index 898718b5a6..f8970b3f30 100644 --- a/openpilot/starpilot/flm/producer_clock_equivalence.py +++ b/openpilot/starpilot/flm/producer_clock_equivalence.py @@ -18,7 +18,7 @@ PRODUCER_SOURCE_HASHES = ( ('openpilot', 'system/camerad/cameras/spectra.cc', '4fa26c17f0fb792a4ab6815657beccddee1d641faa2fd02e5aa7393f717a2f3a'), ('openpilot', 'system/camerad/cameras/camera_qcom2.cc', 'b34c24c9984a0ac086f7b961cb7ff09d0501231da50aaad0e676bb3dc3f2edd9'), ('openpilot', 'system/camerad/cameras/camera_common.cc', 'bd9a416a6ee21047b4deec6df04276d8717cd92fe14de43f3e9daff86b62bb8f'), - ('openpilot', 'selfdrive/modeld/modeld.py', '3cbd9a9843716cf44832d6dd1cf4bef7fbc1facd02b9583b9b7db7e9abeb4c7b'), + ('openpilot', 'selfdrive/modeld/modeld.py', '75f050328e073bdc2476da0172645093a8aa74179b7ec24990414fc4ec98fe69'), ('openpilot', 'selfdrive/modeld/fill_model_msg.py', 'b3916c34d1073d0c26a7a8adc51e7b148f63a1ac68b388226170fd0f8b8b56b4'), ('openpilot', 'system/sensord/sensord.py', '7c79ab07d775622a43001e5a9b9ccc9a7bafb065940185849b33eefc2fd26420'), ('openpilot', 'system/sensord/sensors/lsm6ds3_gyro.py', '3e561f3d8572e1477c9b4b810c7570837d51e81c5672fe21ec67a68cae00dd92'), diff --git a/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py b/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py index b662c0f196..9664372cb4 100644 --- a/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py +++ b/openpilot/starpilot/flm/tests/test_upstream_producer_transport.py @@ -8,7 +8,7 @@ from openpilot.starpilot.flm import producer_clock_equivalence as clock APPROVED = { - 'selfdrive/modeld/modeld.py': '3cbd9a9843716cf44832d6dd1cf4bef7fbc1facd02b9583b9b7db7e9abeb4c7b', + 'selfdrive/modeld/modeld.py': '75f050328e073bdc2476da0172645093a8aa74179b7ec24990414fc4ec98fe69', 'visionipc/visionipc_server.cc': 'a4325a5171eacb931eb271d06a992d2f27216040df113d92099f62cdfa67d81a', 'visionipc/visionipc_client.cc': 'd142cd60eedabfc581e9770fcd6f0ca81b762ba9dbc608b54dd20a27a317a52d', } diff --git a/openpilot/starpilot/models/tests/test_loader_cpu_affinity.py b/openpilot/starpilot/models/tests/test_loader_cpu_affinity.py new file mode 100644 index 0000000000..45e18f5afb --- /dev/null +++ b/openpilot/starpilot/models/tests/test_loader_cpu_affinity.py @@ -0,0 +1,88 @@ +"""Exercise the actual nested GPU loader without constructing a GPU context.""" +import ast +from pathlib import Path +from types import SimpleNamespace +import threading + +import pytest + + +SOURCE = Path(__file__).resolve().parents[3] / 'selfdrive/modeld/modeld.py' + + +def loader(source, *, demo=False, bundled=False, failure=None): + tree = ast.parse(source.read_text()) + main = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'main') + function = next(n for n in ast.walk(main) if isinstance(n, ast.FunctionDef) and n.name == 'load_big') + calls = [] + + def record(name, result=None): + def run(*args, **kwargs): + calls.append((name, args, threading.get_ident())) + if name == failure: + raise OSError(name) + return result + return run + + model = SimpleNamespace(warmup=record('warmup')) + namespace = {'demo': demo, 'CP': object(), 'BIG_MODEL_TIMEOUT': 120, 'wait_for_chestnut': record('usb'), + 'wait_for_chestnut_power': record('power'), 'set_core_affinity': record('affinity'), + 'selection': SimpleNamespace(big_path=None if bundled else Path('/verified.pkl'), big_version='v16', + big_id='mountaindewv3', big_sha256='a' * 64), + 'vipc_client_main': SimpleNamespace(width=1928, height=1208), 'big_path': Path('/verified.pkl'), + 'load_verified_model': record('verified', (model, 'prepared')), 'ModelState': record('bundled', model), + 'receipt_owner': SimpleNamespace(prepare=record('receipt', 'prepared')), + 'big_artifact_identity': object(), 'cloudlog': SimpleNamespace(exception=record('error'))} + outer = ast.parse('def outer():\n big_model = None\n big_prepared = None\n return load_big, lambda: (big_model, big_prepared)\n').body[0] + outer.body.insert(2, function) + module = ast.fix_missing_locations(ast.Module(body=[outer], type_ignores=[])) + exec(compile(module, str(source), 'exec'), namespace) + callback, state = namespace['outer']() + return callback, state, calls + + +@pytest.mark.parametrize('demo,expected', [(False, ['usb', 'power', 'affinity', 'verified']), + (True, ['usb', 'affinity', 'verified'])]) +def test_actual_gpu_loader_pins_after_readiness_before_verified_load(demo, expected): + callback, state, calls = loader(SOURCE, demo=demo) + callback() + assert [row[0] for row in calls] == expected + assert next(row[1] for row in calls if row[0] == 'affinity') == ([7],) + assert state()[1] == 'prepared' + + +@pytest.mark.parametrize('failure,expected', [('power', ['usb', 'power', 'error']), + ('affinity', ['usb', 'power', 'affinity', 'error'])]) +def test_failed_readiness_or_affinity_never_loads_a_model(failure, expected): + callback, state, calls = loader(SOURCE, failure=failure) + callback() + assert [row[0] for row in calls] == expected + assert state() == (None, None) + + +def test_bundled_gpu_loader_keeps_warmup_and_receipt(): + callback, state, calls = loader(SOURCE, bundled=True) + callback() + assert [row[0] for row in calls] == ['usb', 'power', 'affinity', 'bundled', 'warmup', 'receipt'] + assert state()[1] == 'prepared' + + +def test_affinity_is_applied_to_the_loader_thread_only(): + callback, state, calls = loader(SOURCE) + main_thread = threading.get_ident() + worker = threading.Thread(target=callback) + worker.start() + worker.join(timeout=2) + assert not worker.is_alive() + assert state()[1] == 'prepared' + assert all(row[2] == worker.ident and row[2] != main_thread for row in calls) + + +def test_no_fifo_priority_change_inside_gpu_loader(): + tree = ast.parse(SOURCE.read_text()) + main = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'main') + callback = next(n for n in ast.walk(main) if isinstance(n, ast.FunctionDef) and n.name == 'load_big') + assert not any(isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id == 'config_realtime_process' + for n in ast.walk(callback)) + assert sum(isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id == 'config_realtime_process' + for n in ast.walk(main)) == 1 diff --git a/prebuilt.json b/prebuilt.json index faaffc3fc8..184bf78ae1 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "0fb8cadb3c402848011c086cbf34c73c3325c804fa3de31309bc2cd0ae7b1b11", "schema_version": 1} +{"protected_inventory_sha256": "2783830363fab32346bf9a66afcf51e6840c7116f139bc41d07230ca52723763", "schema_version": 1}