#!/usr/bin/env python3 from __future__ import annotations import time from collections import deque from dataclasses import dataclass from typing import Any import cereal.messaging as messaging from cereal import custom from openpilot.common.swaglog import cloudlog TRACE_SERVICE = "iqPerfTrace" MAX_TRACE_SAMPLES = 16 _SHARED_PM: messaging.PubMaster | None = None @dataclass(slots=True) class PerfSample: frame_id: int = 0 loop_dt_us: int = 0 update_us: int = 0 state_control_us: int = 0 publish_us: int = 0 tail_work_us: int = 0 rk_remaining_us: int = 0 stale_carcontrol_us: int = 0 stale_carcontrol_frames: int = 0 sendcan_gap_us: int = 0 model_eval_us: int = 0 model_dropped_frames: int = 0 model_backlog: int = 0 texture_decode_us: int = 0 texture_upload_us: int = 0 texture_unload_us: int = 0 texture_prune_us: int = 0 texture_consume_us: int = 0 texture_batch_size: int = 0 texture_bytes: int = 0 texture_cache_before: int = 0 texture_cache_after: int = 0 texture_unloaded: int = 0 memory_usage_percent: int = 0 gpu_usage_percent: int = 0 cpu_usage_percent: int = 0 flags: int = 0 class PerfTraceRing: def __init__(self, size: int = MAX_TRACE_SAMPLES): self._samples: deque[PerfSample] = deque(maxlen=size) def push(self, sample: PerfSample) -> None: self._samples.append(sample) def snapshot(self) -> list[PerfSample]: return list(self._samples) class PerfTraceEmitter: _SEVERITY_MAP = { "info": custom.IQPerfTrace.Severity.info, "warning": custom.IQPerfTrace.Severity.warning, "error": custom.IQPerfTrace.Severity.error, "critical": custom.IQPerfTrace.Severity.critical, } def __init__(self, process_name: str, pubmaster: messaging.PubMaster | None = None): self.process_name = process_name self._pm: messaging.PubMaster | None = pubmaster self._last_emit_mono: dict[str, float] = {} self._disabled = False def _pubmaster(self) -> messaging.PubMaster: global _SHARED_PM if self._pm is not None: return self._pm if _SHARED_PM is None: _SHARED_PM = messaging.PubMaster([TRACE_SERVICE]) self._pm = _SHARED_PM return self._pm @staticmethod def _clamp_uint(value: int, bits: int) -> int: return max(0, min(value, (1 << bits) - 1)) @staticmethod def _clamp_int(value: int, bits: int) -> int: lo = -(1 << (bits - 1)) hi = (1 << (bits - 1)) - 1 return max(lo, min(value, hi)) def emit(self, event_class: str, *, severity: str = "warning", frame_id: int = 0, total_time_us: int = 0, rk_remaining_us: int = 0, batch_size: int = 0, dropped_frames: int = 0, backlog: int = 0, flags: int = 0, samples: list[PerfSample] | None = None, missing_services: list[str] | None = None, top_processes: list[str] | None = None, detail: str = "", min_interval_s: float = 0.0, mirror_cloudlog: bool = True) -> bool: if self._disabled: return False now = time.monotonic() last_emit = self._last_emit_mono.get(event_class, 0.0) if min_interval_s > 0.0 and (now - last_emit) < min_interval_s: return False self._last_emit_mono[event_class] = now msg = messaging.new_message(TRACE_SERVICE) trace = msg.iqPerfTrace trace.process = self.process_name trace.eventClass = event_class trace.severity = self._SEVERITY_MAP.get(severity, custom.IQPerfTrace.Severity.warning) trace.frameId = self._clamp_uint(int(frame_id), 32) trace.totalTimeUs = self._clamp_uint(int(total_time_us), 32) trace.rkRemainingUs = self._clamp_int(int(rk_remaining_us), 32) trace.batchSize = self._clamp_uint(int(batch_size), 16) trace.droppedFrames = self._clamp_uint(int(dropped_frames), 16) trace.backlog = self._clamp_uint(int(backlog), 16) trace.flags = self._clamp_uint(int(flags), 32) trace.missingServices = list(missing_services or []) trace.topProcesses = list(top_processes or []) trace.detail = detail trace_samples = samples or [] samples_builder = trace.init("samples", len(trace_samples)) for i, sample in enumerate(trace_samples): builder = samples_builder[i] builder.frameId = self._clamp_uint(int(sample.frame_id), 32) builder.loopDtUs = self._clamp_uint(int(sample.loop_dt_us), 32) builder.updateUs = self._clamp_uint(int(sample.update_us), 32) builder.stateControlUs = self._clamp_uint(int(sample.state_control_us), 32) builder.publishUs = self._clamp_uint(int(sample.publish_us), 32) builder.tailWorkUs = self._clamp_uint(int(sample.tail_work_us), 32) builder.rkRemainingUs = self._clamp_int(int(sample.rk_remaining_us), 32) builder.staleCarControlUs = self._clamp_uint(int(sample.stale_carcontrol_us), 32) builder.staleCarControlFrames = self._clamp_uint(int(sample.stale_carcontrol_frames), 16) builder.sendcanGapUs = self._clamp_uint(int(sample.sendcan_gap_us), 32) builder.modelEvalUs = self._clamp_uint(int(sample.model_eval_us), 32) builder.modelDroppedFrames = self._clamp_uint(int(sample.model_dropped_frames), 16) builder.modelBacklog = self._clamp_uint(int(sample.model_backlog), 16) builder.textureDecodeUs = self._clamp_uint(int(sample.texture_decode_us), 32) builder.textureUploadUs = self._clamp_uint(int(sample.texture_upload_us), 32) builder.textureUnloadUs = self._clamp_uint(int(sample.texture_unload_us), 32) builder.texturePruneUs = self._clamp_uint(int(sample.texture_prune_us), 32) builder.textureConsumeUs = self._clamp_uint(int(sample.texture_consume_us), 32) builder.textureBatchSize = self._clamp_uint(int(sample.texture_batch_size), 16) builder.textureBytes = self._clamp_uint(int(sample.texture_bytes), 32) builder.textureCacheBefore = self._clamp_uint(int(sample.texture_cache_before), 16) builder.textureCacheAfter = self._clamp_uint(int(sample.texture_cache_after), 16) builder.textureUnloaded = self._clamp_uint(int(sample.texture_unloaded), 16) builder.memoryUsagePercent = self._clamp_uint(int(sample.memory_usage_percent), 16) builder.gpuUsagePercent = self._clamp_uint(int(sample.gpu_usage_percent), 16) builder.cpuUsagePercent = self._clamp_uint(int(sample.cpu_usage_percent), 16) builder.flags = self._clamp_uint(int(sample.flags), 32) try: self._pubmaster().send(TRACE_SERVICE, msg) except messaging.MultiplePublishersError: self._disabled = True cloudlog.error(f"iq_perf_trace disabled for {self.process_name}: duplicate publisher for {TRACE_SERVICE}") return False except Exception: cloudlog.exception(f"iq_perf_trace publish failed for {self.process_name}") return False if mirror_cloudlog: cloudlog.event( "iq_perf_trace", process=self.process_name, event_class=event_class, severity=severity, frame_id=int(frame_id), total_time_us=int(total_time_us), dropped_frames=int(dropped_frames), flags=int(flags), detail=detail, ) return True