mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-07-17 02:02:05 +08:00
Adeeb Shihadeh
62b97fabf7
* merge selfdrive/debug/ into tools/scripts/ * rm unused * single timings script * lil more * profiling is scripts quality
103 lines
4.0 KiB
Python
Executable File
103 lines
4.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import datetime
|
|
import time
|
|
from collections import deque
|
|
from dataclasses import dataclass, field
|
|
|
|
import numpy as np
|
|
|
|
import cereal.messaging as messaging
|
|
from cereal.services import SERVICE_LIST
|
|
|
|
|
|
@dataclass
|
|
class ServiceTiming:
|
|
times: list[float] = field(default_factory=list)
|
|
window: deque[float] = field(default_factory=lambda: deque(maxlen=100))
|
|
valids: deque[bool] = field(default_factory=lambda: deque(maxlen=100))
|
|
lag_events: list[tuple[float, float]] = field(default_factory=list)
|
|
|
|
def add(self, mono_time: float, valid: bool, expected_interval: float | None, lag_threshold: float) -> None:
|
|
if self.times:
|
|
dt = mono_time - self.times[-1]
|
|
self.window.append(dt)
|
|
if expected_interval is not None and dt > lag_threshold * expected_interval:
|
|
self.lag_events.append((mono_time, dt))
|
|
|
|
self.times.append(mono_time)
|
|
self.valids.append(valid)
|
|
|
|
def intervals(self, latest_only: bool) -> np.ndarray:
|
|
if latest_only:
|
|
return np.array(self.window)
|
|
return np.diff(self.times)
|
|
|
|
|
|
def format_row(name: str, timing: ServiceTiming, latest_only: bool) -> str:
|
|
dts = timing.intervals(latest_only)
|
|
if len(dts) == 0:
|
|
return f"{name:25} waiting for messages"
|
|
|
|
mean = np.mean(dts)
|
|
hz = 1.0 / mean if mean > 0 else 0.0
|
|
valid = all(timing.valids) if timing.valids else False
|
|
return f"{name:25} {hz:8.2f}Hz {mean * 1e3:8.2f}ms {np.std(dts) * 1e3:8.2f}ms {np.max(dts) * 1e3:8.2f}ms {np.min(dts) * 1e3:8.2f}ms valid={valid}"
|
|
|
|
|
|
def print_lag_events(name: str, timing: ServiceTiming, printed_lags: dict[str, int]) -> None:
|
|
start = printed_lags.get(name, 0)
|
|
for mono_time, dt in timing.lag_events[start:]:
|
|
print(f"{mono_time:.3f} {name} lag {dt:.3f}s", flush=True)
|
|
printed_lags[name] = len(timing.lag_events)
|
|
|
|
|
|
def monitor_services(socket_names: list[str], print_interval: float, lag_threshold: float, lag_only: bool) -> None:
|
|
sockets = {name: messaging.sub_sock(name, conflate=False) for name in socket_names}
|
|
timings = {name: ServiceTiming() for name in socket_names}
|
|
printed_lags: dict[str, int] = {}
|
|
|
|
start_time = time.monotonic()
|
|
last_print = start_time
|
|
|
|
try:
|
|
while True:
|
|
for name, sock in sockets.items():
|
|
for msg in messaging.drain_sock(sock):
|
|
expected_interval = 1.0 / SERVICE_LIST[name].frequency if name in SERVICE_LIST else None
|
|
timings[name].add(msg.logMonoTime / 1e9, msg.valid, expected_interval, lag_threshold)
|
|
|
|
now = time.monotonic()
|
|
if now - last_print < print_interval:
|
|
time.sleep(0.01)
|
|
continue
|
|
|
|
if not lag_only:
|
|
print(flush=True)
|
|
print(f"{'service':25} {'freq':>10} {'mean':>10} {'std':>10} {'max':>10} {'min':>10} valid", flush=True)
|
|
for name in socket_names:
|
|
print(format_row(name, timings[name], latest_only=True), flush=True)
|
|
|
|
for name in socket_names:
|
|
print_lag_events(name, timings[name], printed_lags)
|
|
|
|
last_print = now
|
|
except KeyboardInterrupt:
|
|
print("\n", flush=True)
|
|
print("=" * 5, "timing summary", "=" * 5, flush=True)
|
|
print(f"{'service':25} {'freq':>10} {'mean':>10} {'std':>10} {'max':>10} {'min':>10} valid", flush=True)
|
|
for name in socket_names:
|
|
print(format_row(name, timings[name], latest_only=False), flush=True)
|
|
print("=" * 5, datetime.timedelta(seconds=time.monotonic() - start_time), "=" * 5, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="Check live service timing, frequency, validity, and lag")
|
|
parser.add_argument("socket", nargs="*", default=["carState"], help="service/socket name")
|
|
parser.add_argument("--lag-threshold", type=float, default=10.0, help="report intervals above this multiple of the expected service interval")
|
|
parser.add_argument("--lag-only", action="store_true", help="only print lag events")
|
|
parser.add_argument("--print-interval", type=float, default=1.0, help="seconds between table updates")
|
|
args = parser.parse_args()
|
|
|
|
monitor_services(args.socket, args.print_interval, args.lag_threshold, args.lag_only)
|