From 503531ab384cd34a7114c7e0748d6c20232e0f88 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 11 Jul 2026 13:51:22 -0700 Subject: [PATCH] replace psutil with simple /proc helper (#38326) * replace psutil with simple /proc helper * mv to common --- openpilot/common/linux.py | 50 +++++++++++ openpilot/system/hardware/hardwared.py | 8 +- openpilot/system/updated/updated.py | 6 -- pyproject.toml | 1 - tools/scripts/cpu_usage_stat.py | 120 ------------------------- uv.lock | 18 ---- 6 files changed, 54 insertions(+), 149 deletions(-) create mode 100644 openpilot/common/linux.py delete mode 100755 tools/scripts/cpu_usage_stat.py diff --git a/openpilot/common/linux.py b/openpilot/common/linux.py new file mode 100644 index 000000000..73ea77b50 --- /dev/null +++ b/openpilot/common/linux.py @@ -0,0 +1,50 @@ +class LinuxSystemStats: + def __init__(self) -> None: + self._last_cpu_times = self._read_cpu_times() + + @staticmethod + def _read_cpu_times() -> dict[int, tuple[int, int]]: + cpu_times = {} + with open('/proc/stat') as f: + for line in f: + name, *values = line.split() + if not name.startswith('cpu') or not name[3:].isdigit(): + continue + + times = [int(value) for value in values] + idle = sum(times[3:5]) + total = sum(times[:8]) + cpu_times[int(name[3:])] = (idle, total) + return cpu_times + + def cpu_usage_percent(self) -> list[float]: + current_cpu_times = self._read_cpu_times() + usage = [] + for cpu, (idle, total) in sorted(current_cpu_times.items()): + last_times = self._last_cpu_times.get(cpu) + if last_times is None: + usage.append(0.) + continue + + last_idle, last_total = last_times + idle_delta = idle - last_idle + total_delta = total - last_total + if idle_delta < 0 or total_delta <= 0: + usage.append(0.) + else: + usage.append(max(0., min(100., 100. * (total_delta - idle_delta) / total_delta))) + + self._last_cpu_times = current_cpu_times + return usage + + @staticmethod + def memory_usage_percent() -> float: + memory = {} + with open('/proc/meminfo') as f: + for line in f: + key, value, *_ = line.split() + if key in ('MemTotal:', 'MemAvailable:'): + memory[key] = int(value) + + total = memory['MemTotal:'] + return max(0., min(100., 100. * (total - memory['MemAvailable:']) / total)) diff --git a/openpilot/system/hardware/hardwared.py b/openpilot/system/hardware/hardwared.py index 6b86ce552..581ac7f1e 100755 --- a/openpilot/system/hardware/hardwared.py +++ b/openpilot/system/hardware/hardwared.py @@ -7,8 +7,6 @@ import threading import time from collections import OrderedDict, namedtuple -import psutil - import openpilot.cereal.messaging as messaging from openpilot.cereal import log from openpilot.cereal.services import SERVICE_LIST @@ -19,6 +17,7 @@ from openpilot.common.realtime import DT_HW from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.common.hardware import HARDWARE, TICI, PC from openpilot.common.hardware.usb import get_usb_state, set_usb_state +from openpilot.common.linux import LinuxSystemStats from openpilot.system.loggerd.config import get_available_percent from openpilot.common.swaglog import cloudlog from openpilot.system.hardware.power_monitoring import PowerMonitoring @@ -142,6 +141,7 @@ def hw_state_thread(end_event, hw_queue): def hardware_thread(end_event, hw_queue) -> None: + system_stats = LinuxSystemStats() pm = messaging.PubMaster(['deviceState']) sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "selfdriveState", "pandaStates"], poll="pandaStates") @@ -232,9 +232,9 @@ def hardware_thread(end_event, hw_queue) -> None: pass msg.deviceState.freeSpacePercent = get_available_percent(default=100.0) - msg.deviceState.memoryUsagePercent = int(round(psutil.virtual_memory().percent)) + msg.deviceState.memoryUsagePercent = int(round(system_stats.memory_usage_percent())) msg.deviceState.gpuUsagePercent = int(round(HARDWARE.get_gpu_usage_percent())) - online_cpu_usage = [int(round(n)) for n in psutil.cpu_percent(percpu=True)] + online_cpu_usage = [int(round(n)) for n in system_stats.cpu_usage_percent()] offline_cpu_usage = [0., ] * (len(msg.deviceState.cpuTempC) - len(online_cpu_usage)) msg.deviceState.cpuUsagePercent = online_cpu_usage + offline_cpu_usage diff --git a/openpilot/system/updated/updated.py b/openpilot/system/updated/updated.py index c8f45fdb5..ac8111d5c 100755 --- a/openpilot/system/updated/updated.py +++ b/openpilot/system/updated/updated.py @@ -3,7 +3,6 @@ import os import re import datetime import subprocess -import psutil import shutil import signal import fcntl @@ -427,11 +426,6 @@ def main() -> None: except OSError as e: raise RuntimeError("couldn't get overlay lock; is another instance running?") from e - # Set low io priority - proc = psutil.Process() - if psutil.LINUX: - proc.ionice(psutil.IOPRIO_CLASS_BE, value=7) - # Check if we just performed an update if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") diff --git a/pyproject.toml b/pyproject.toml index c4df3aabe..9f1391719 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,6 @@ dependencies = [ "inputs", # these should be removed - "psutil", "setproctitle", # logreader diff --git a/tools/scripts/cpu_usage_stat.py b/tools/scripts/cpu_usage_stat.py deleted file mode 100755 index 902df1c77..000000000 --- a/tools/scripts/cpu_usage_stat.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -''' -System tools like top/htop can only show current cpu usage values, so I write this script to do statistics jobs. - Features: - Use psutil library to sample cpu usage(avergage for all cores) of openpilot processes, at a rate of 5 samples/sec. - Do cpu usage statistics periodically, 5 seconds as a cycle. - Calculate the average cpu usage within this cycle. - Calculate minumium/maximum/accumulated_average cpu usage as long term inspections. - Monitor multiple processes simuteneously. - Sample usage: - root@localhost:/data/openpilot$ python openpilot/tools/scripts/cpu_usage_stat.py pandad,ubloxd - ('Add monitored proc:', './pandad') - ('Add monitored proc:', 'python locationd/ubloxd.py') - pandad: 1.96%, min: 1.96%, max: 1.96%, acc: 1.96% - ubloxd.py: 0.39%, min: 0.39%, max: 0.39%, acc: 0.39% -''' -import psutil -import time -import os -import sys -import numpy as np -import argparse -import re -from collections import defaultdict - -from openpilot.system.manager.process_config import managed_processes - -# Do statistics every 5 seconds -PRINT_INTERVAL = 5 -SLEEP_INTERVAL = 0.2 - -monitored_proc_names = [ - # android procs - 'SurfaceFlinger', 'sensors.qcom' -] + list(managed_processes.keys()) - -cpu_time_names = ['user', 'system', 'children_user', 'children_system'] - - -def get_arg_parser(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument("proc_names", nargs="?", default='', - help="Process names to be monitored, comma separated") - parser.add_argument("--list_all", action='store_true', - help="Show all running processes' cmdline") - parser.add_argument("--detailed_times", action='store_true', - help="show cpu time details (split by user, system, child user, child system)") - return parser - - -if __name__ == "__main__": - args = get_arg_parser().parse_args(sys.argv[1:]) - if args.list_all: - for p in psutil.process_iter(): - print('cmdline', p.cmdline(), 'name', p.name()) - sys.exit(0) - - if len(args.proc_names) > 0: - monitored_proc_names = args.proc_names.split(',') - monitored_procs = [] - stats = {} - for p in psutil.process_iter(): - if p == psutil.Process(): - continue - matched = any(l for l in p.cmdline() if any(pn for pn in monitored_proc_names if re.match(fr'.*{pn}.*', l, re.M | re.I))) - if matched: - k = ' '.join(p.cmdline()) - print('Add monitored proc:', k) - stats[k] = {'cpu_samples': defaultdict(list), 'min': defaultdict(lambda: None), 'max': defaultdict(lambda: None), - 'avg': defaultdict(float), 'last_cpu_times': None, 'last_sys_time': None} - stats[k]['last_sys_time'] = time.monotonic() - stats[k]['last_cpu_times'] = p.cpu_times() - monitored_procs.append(p) - i = 0 - interval_int = int(PRINT_INTERVAL / SLEEP_INTERVAL) - while True: - for p in monitored_procs: - k = ' '.join(p.cmdline()) - cur_sys_time = time.monotonic() - cur_cpu_times = p.cpu_times() - cpu_times = np.subtract(cur_cpu_times, stats[k]['last_cpu_times']) / (cur_sys_time - stats[k]['last_sys_time']) - stats[k]['last_sys_time'] = cur_sys_time - stats[k]['last_cpu_times'] = cur_cpu_times - cpu_percent = 0 - for num, name in enumerate(cpu_time_names): - stats[k]['cpu_samples'][name].append(cpu_times[num]) - cpu_percent += cpu_times[num] - stats[k]['cpu_samples']['total'].append(cpu_percent) - time.sleep(SLEEP_INTERVAL) - i += 1 - if i % interval_int == 0: - l = [] - for k, stat in stats.items(): - if len(stat['cpu_samples']) <= 0: - continue - for name, samples in stat['cpu_samples'].items(): - samples = np.array(samples) - avg = samples.mean() - c = samples.size - min_cpu = np.amin(samples) - max_cpu = np.amax(samples) - if stat['min'][name] is None or min_cpu < stat['min'][name]: - stat['min'][name] = min_cpu - if stat['max'][name] is None or max_cpu > stat['max'][name]: - stat['max'][name] = max_cpu - stat['avg'][name] = (stat['avg'][name] * (i - c) + avg * c) / (i) - stat['cpu_samples'][name] = [] - - msg = f"avg: {stat['avg']['total']:.2%}, min: {stat['min']['total']:.2%}, max: {stat['max']['total']:.2%} {os.path.basename(k)}" - if args.detailed_times: - for stat_type in ['avg', 'min', 'max']: - msg += f"\n {stat_type}: {[(name + ':' + str(round(stat[stat_type][name] * 100, 2))) for name in cpu_time_names]}" - l.append((os.path.basename(k), stat['avg']['total'], msg)) - l.sort(key=lambda x: -x[1]) - for x in l: - print(x[2]) - print('avg sum: {:.2%} over {} samples {} seconds\n'.format( - sum(stat['avg']['total'] for k, stat in stats.items()), i, i * SLEEP_INTERVAL - )) diff --git a/uv.lock b/uv.lock index d5b000665..2284243b5 100644 --- a/uv.lock +++ b/uv.lock @@ -898,7 +898,6 @@ dependencies = [ { name = "jeepney" }, { name = "numpy" }, { name = "pillow" }, - { name = "psutil" }, { name = "pycapnp" }, { name = "pyjwt" }, { name = "pyzmq" }, @@ -981,7 +980,6 @@ requires-dist = [ { name = "numpy", specifier = ">=2.0" }, { name = "pillow" }, { name = "pre-commit-hooks", marker = "extra == 'testing'" }, - { name = "psutil" }, { name = "pycapnp", specifier = "==2.1.0" }, { name = "pyjwt" }, { name = "pytest", marker = "extra == 'testing'" }, @@ -1122,22 +1120,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] -[[package]] -name = "psutil" -version = "7.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, - { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, - { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, -] - [[package]] name = "pycapnp" version = "2.1.0"