diff --git a/sunnypilot/tools/__init__.py b/sunnypilot/tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/tools/memory_profiler/__init__.py b/sunnypilot/tools/memory_profiler/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/tools/memory_profiler/mem_usage.py b/sunnypilot/tools/memory_profiler/mem_usage.py new file mode 100644 index 0000000000..20b4bb2d0f --- /dev/null +++ b/sunnypilot/tools/memory_profiler/mem_usage.py @@ -0,0 +1,164 @@ +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" +import matplotlib.pyplot as plt +import os +import sys +import argparse +import numpy as np +import base64 +import io + +from openpilot.tools.lib.logreader import LogReader, ReadMode + + +def extract_mem_cpu_data(lr): + times, mems, cpus = [], [], [] + start_time = None + + for msg in lr: + if msg.which() == 'procLog': + if start_time is None: + start_time = msg.logMonoTime + mem = msg.procLog.mem + mem_usage = (mem.total - mem.available) / mem.total * 100 + cpu_usages = [(total - cpu.idle) / total * 100 for cpu in msg.procLog.cpuTimes + if (total := cpu.idle + cpu.user + cpu.system + cpu.nice + cpu.iowait + cpu.irq + cpu.softirq) > 0] + avg_cpu = sum(cpu_usages) / len(cpu_usages) if cpu_usages else 0 + times.append((msg.logMonoTime - start_time) / 1e9) + mems.append(mem_usage) + cpus.append(avg_cpu) + return times, mems, cpus + + +def process_segment(lr): + return [extract_mem_cpu_data(lr)] + + +def calculate_r_squared(y_true, y_pred): + ss_res = np.sum((y_true - y_pred) ** 2) + ss_tot = np.sum((y_true - np.mean(y_true)) ** 2) + return 1 - (ss_res / ss_tot) if ss_tot != 0 else 0 + + +def plot_results(segments, segment_data, route_name): + valid_data = [d for d in segment_data if d and d[0]] + if not valid_data: + print("No valid data to plot") + return + + avg_mems = [np.mean(d[1]) for d in valid_data] + avg_cpus = [np.mean(d[2]) for d in valid_data] + valid_segments = [segments[i] for i, d in enumerate(segment_data) if d and d[0]] + + height = max(10, 5 + len(valid_segments) * 0.4) + fig1, ax1 = plt.subplots(1, 1, figsize=(12, height), dpi=150) + + y_pos = range(len(valid_segments)) + ax1.barh([y - 0.2 for y in y_pos], avg_mems, height=0.4, color="dodgerblue", alpha=0.8, label="Avg Mem %") + ax1.barh([y + 0.2 for y in y_pos], avg_cpus, height=0.4, color="green", alpha=0.8, label="Avg CPU %") + + for i, (mem, cpu) in enumerate(zip(avg_mems, avg_cpus, strict=True)): + ax1.text(mem, i - 0.2, f"{mem:.1f}%", va="center", fontsize=8, color="#005a9e", fontweight="bold") + ax1.text(cpu, i + 0.2, f"{cpu:.1f}%", va="center", fontsize=8, color="#005a9e", fontweight="bold") + + ax1.set_yticks(y_pos) + ax1.set_yticklabels([f"Seg {s}" for s in valid_segments]) + ax1.set_xlabel("Usage (%)") + ax1.set_title("Average Memory and CPU Usage by Segment") + ax1.legend() + ax1.grid(axis="x", linestyle="--", alpha=0.5) + ax1.invert_yaxis() + + fig2, ax2 = plt.subplots(1, 1, figsize=(12, 8), dpi=150) + combined_times, combined_mems, combined_cpus = [], [], [] + time_offset = 0.0 + for times, mems, cpus in valid_data: + if times: + combined_times.extend([t + time_offset for t in times]) + combined_mems.extend(mems) + combined_cpus.extend(cpus) + time_offset += max(times) + + ax2.plot(combined_times, combined_mems, color="red", label="Memory Usage", alpha=0.6) + ax2.plot(combined_times, combined_cpus, color="blue", label="CPU Usage", alpha=0.6) + + warmup_sec = 60 + if len(combined_times) > 1 and combined_times[-1] > warmup_sec: + mask = np.array(combined_times) > warmup_sec + x_reg = np.array(combined_times)[mask] + + y_mem_reg = np.array(combined_mems)[mask] + slope_mem, intercept_mem = np.polyfit(x_reg, y_mem_reg, 1) + trend_mem = slope_mem * x_reg + intercept_mem + r2_mem = calculate_r_squared(y_mem_reg, trend_mem) + ax2.plot(x_reg, trend_mem, color="darkred", linestyle="--", linewidth=2.5, + label=f"Mem Trend (Slope: {slope_mem:.4f} %/s, R²: {r2_mem:.2f})") + + y_cpu_reg = np.array(combined_cpus)[mask] + slope_cpu, intercept_cpu = np.polyfit(x_reg, y_cpu_reg, 1) + trend_cpu = slope_cpu * x_reg + intercept_cpu + r2_cpu = calculate_r_squared(y_cpu_reg, trend_cpu) + ax2.plot(x_reg, trend_cpu, color="navy", linestyle="--", linewidth=2.5, + label=f"CPU Trend (Slope: {slope_cpu:.4f} %/s, R²: {r2_cpu:.2f})") + + ax2.set_xlabel("Time (s)") + ax2.set_ylabel("Usage (%)") + ax2.set_title("Memory and CPU Usage Over Time") + ax2.legend(loc='lower left', fontsize='small', framealpha=0.9) + ax2.grid(True, linestyle="--", alpha=0.5) + + buffer1 = io.BytesIO() + fig1.savefig(buffer1, format='webp', bbox_inches='tight', pad_inches=1.0) + buffer1.seek(0) + img1 = base64.b64encode(buffer1.getvalue()).decode() + + buffer2 = io.BytesIO() + fig2.savefig(buffer2, format='webp', bbox_inches='tight', pad_inches=1.0) + buffer2.seek(0) + img2 = base64.b64encode(buffer2.getvalue()).decode() + + filename = f"memory_usage_{route_name}.html" + save_path = os.path.join(os.path.dirname(__file__), "plots", filename) + os.makedirs(os.path.dirname(save_path), exist_ok=True) + + html_template = ( + "" + + f"