StarPilot/openpilot/starpilot/system/starport/render_profile.py

"""Always-on sampling profile of the car renderer, in a size-capped file.

A daemon thread looks at the render thread's Python stack every INTERVAL
seconds (sys._current_frames: no tracing, so the renderer runs at full speed)
and counts, per function, how often it was running (self) or on the stack
(inclusive), plus the hottest source lines. Samples taken while the renderer
is idle (pacing, or the car showing its own screen) are only counted as idle.

Every WINDOW seconds the busiest entries of that window are appended to the
report file together with the renderer's latest ``render_stats`` (see RenderSummary). When the
file would grow past ``max_bytes`` it becomes ``<name>.1.txt`` (replacing the
previous one) and a new file starts, so at most twice ``max_bytes`` is kept.
The file outlives renderer restarts, unlike car_ui.log.

The sampler can only look while the render thread lets go of the GIL, which it
does in every raylib/GL call, so pure-Python work is credited to the next such
call in the innermost-frame sections. The "on the stack" section is accurate
per function and is the one to read for which part of the layout costs time.
Sampling runs at 25 samples a second.
"""

from __future__ import annotations

import os
import sys
import threading
import time
from collections import Counter, deque
from pathlib import Path

INTERVAL = 0.04    # 25 samples a second
WINDOW = 60.0
TOP = 25
TOP_LINES = 15
SUMMARY_INTERVAL = 5.0


class RenderPipelineSummary:
  """Bounded measurements of published frames; no tracing or per-frame log I/O.

  Camera EOF is an exposure timestamp, not CameraState arrival. Keep it separate
  from renderer capture-to-publish latency so unlike clocks/stages aren't compared.
  ID discontinuities count observations, not proof of a renderer-caused drop.
  """

  def __init__(self, now_ns: int, interval_ns: int = 5_000_000_000):
    self.started_ns, self.interval_ns = now_ns, interval_ns
    self.samples = deque(maxlen=240)
    self.previous = None
    self.repeats = self.skipped = self.resets = self.frames = 0

  def published(self, captured_ns, submitted_ns, published_ns, camera=None):
    if not captured_ns <= submitted_ns <= published_ns:
      return None
    camera_age = None
    if camera is None:
      self.previous = None
    else:
      stream, frame_id, eof_ns = camera
      if 0 < eof_ns <= published_ns:
        camera_age = published_ns - eof_ns
      if self.previous is not None and self.previous[0] == stream:
        delta = frame_id - self.previous[1]
        self.repeats += delta == 0
        self.skipped += max(0, delta - 1)
        self.resets += delta < 0
      self.previous = (stream, frame_id)
    self.frames += 1
    self.samples.append((submitted_ns - captured_ns, published_ns - submitted_ns,
                         published_ns - captured_ns, camera_age))
    if published_ns - self.started_ns < self.interval_ns:
      return None
    report = {'published_frames': self.frames, 'camera_id_repeats': self.repeats,
              'camera_id_gaps': self.skipped, 'camera_id_resets': self.resets}
    for index, name in enumerate(('capture_to_submit', 'submit_to_publish', 'capture_to_publish', 'camera_eof_to_publish')):
      values = sorted(sample[index] for sample in self.samples if sample[index] is not None)
      report[name + '_p95_ms'] = round(values[max(0, (95 * len(values) + 99) // 100 - 1)] / 1e6, 2) if values else None
    self.started_ns = published_ns
    self.samples.clear()
    self.repeats = self.skipped = self.resets = self.frames = 0
    return report


class RenderSummary:
  """Frame rate and mean frame time of the renderer while it renders, for the profile header."""

  def __init__(self, now: float, interval: float = SUMMARY_INTERVAL):
    self.interval = interval
    self.reset(now)

  def reset(self, now: float) -> None:
    self.started, self.frames, self.frame_seconds = now, 0, 0.0

  def frame_done(self, began: float, done: float) -> dict | None:
    """Count one frame; every ``interval`` returns the window's render_stats and starts a new one."""
    self.frames += 1
    self.frame_seconds += done - began
    if done - self.started < self.interval:
      return None
    report = {"event": "render_stats", "fps": round(self.frames / (done - self.started), 1),
              "frame_ms": round(1000 * self.frame_seconds / self.frames, 1)}
    self.reset(done)
    return report


class RenderSampler:
  def __init__(self, path: Path, *, max_bytes: int = 256 * 1024, interval: float = INTERVAL, window: float = WINDOW,
               clock=time.monotonic):
    self.path, self.max_bytes = path, max_bytes
    self.interval, self.window, self.clock = interval, window, clock
    self.thread_id = threading.get_ident()  # created on the render thread, which it samples
    self.rendering = False                  # set by the render loop around each frame
    self.onroad = False
    self.summary: dict | None = None        # latest render_stats
    self._names: dict = {}
    self._lock = threading.Lock()
    self._stop = threading.Event()
    self._worker = threading.Thread(target=self._run, name="car_ui_sampler", daemon=True)
    self._reset()

  def _reset(self) -> None:
    self.started = self.clock()
    self.samples = self.idle = self.onroad_samples = 0
    self.self_counts: Counter = Counter()
    self.total_counts: Counter = Counter()
    self.line_counts: Counter = Counter()
    self.param_samples: Counter = Counter()

  def start(self) -> None:
    self._worker.start()

  def close(self) -> None:
    self._stop.set()
    if self._worker.is_alive():
      self._worker.join(timeout=1.0)
    self.flush()

  def _name(self, code) -> str:
    name = self._names.get(code)
    if name is None:
      name = self._names[code] = f"{Path(code.co_filename).name}:{code.co_firstlineno} {code.co_name}"
    return name

  def sample(self, frame) -> None:
    """Count one sample of the render thread's stack (``frame`` is its innermost frame)."""
    with self._lock:
      self.samples += 1
      if not self.rendering:
        self.idle += 1
        return
      self.onroad_samples += self.onroad
      code = frame.f_code
      self.line_counts[f"{Path(code.co_filename).name}:{frame.f_lineno} {code.co_name}"] += 1
      self.self_counts[self._name(code)] += 1
      seen = set()
      param_seen = False
      while frame is not None:
        code = frame.f_code
        # Attribute sampled waits to keys without wrapping every Params call.
        # Only key names are captured; values (including credentials) never are.
        if not param_seen and code.co_name in ("get", "get_bool") and code.co_filename.endswith("/common/params.py"):
          key = frame.f_locals.get("key")
          if isinstance(key, str):
            self.param_samples[f"{code.co_name}({key})"] += 1
            param_seen = True
        name = self._name(frame.f_code)
        if name not in seen:
          seen.add(name)
          self.total_counts[name] += 1
        frame = frame.f_back

  def _run(self) -> None:
    while not self._stop.wait(self.interval):
      frame = sys._current_frames().get(self.thread_id)
      if frame is not None:
        self.sample(frame)
      del frame
      if self.clock() - self.started >= self.window:
        self.flush()

  def report(self) -> str:
    with self._lock:
      busy = self.samples - self.idle
      if busy <= 0:
        return ""
      state = "onroad" if self.onroad_samples * 2 >= busy else "offroad"
      lines = [f"==== {time.strftime('%Y-%m-%d %H:%M:%S')}  {self.clock() - self.started:.0f} s, {state} " +
               f"({100 * self.onroad_samples // busy}% onroad), rendering {100 * busy // self.samples}% of {self.samples} samples ===="]
      if self.summary:
        lines.append("render_stats " + " ".join(f"{key}={value}" for key, value in self.summary.items() if key != "event"))
      for title, counts, top in (("on the stack (time inside each function)", self.total_counts, TOP),
                                 ("innermost function (pure Python shows up at the next raylib/GL call)", self.self_counts, TOP),
                                 ("innermost line", self.line_counts, TOP_LINES)):
        lines.append(f"-- {title}")
        # Frames on every sample (the render loop and interpreter startup) say nothing.
        shown = [(name, count) for name, count in counts.most_common() if count < busy or counts is not self.total_counts]
        lines += [f"{100 * count / busy:6.1f}%  {name}" for name, count in shown[:top]]
      if self.param_samples:
        lines.append("-- Params keys sampled on the render thread (% of busy samples, not read counts)")
        lines += [f"{100 * count / busy:6.1f}%  {name}" for name, count in self.param_samples.most_common(TOP)]
    return "\n".join(lines) + "\n\n"

  def flush(self) -> None:
    text = self.report()
    with self._lock:
      self._reset()
    if text:
      self._write(text)

  def _write(self, text: str) -> None:
    try:
      self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
      try:
        if self.path.stat().st_size + len(text) > self.max_bytes:
          os.replace(self.path, self.path.with_name(f"{self.path.stem}.1{self.path.suffix}"))
      except FileNotFoundError:
        pass
      with open(self.path, "a") as handle:
        handle.write(text)
    except OSError:
      pass