Compare commits

..

15 Commits

Author SHA1 Message Date
discountchubbs 429a334d60 clean up some MORE 2026-09-09 08:58:19 -07:00
James Vecellio-Grant c0a3ca44b6 Update model_replay.py 2026-09-09 00:56:07 -07:00
discountchubbs a039d9bc27 rm 2026-09-09 00:24:09 -07:00
discountchubbs c634f6ed8e y no more nproc 2026-09-09 00:08:41 -07:00
discountchubbs fd075878dd eh use latest. it has m1 chip 2026-09-09 00:05:23 -07:00
discountchubbs 5a62f7b427 drop 2026-09-09 00:01:03 -07:00
discountchubbs 13ea924578 wrap both models 2026-09-08 23:39:36 -07:00
discountchubbs ab07b706a2 lil more 2026-09-08 23:28:40 -07:00
discountchubbs 50ea3a0388 clean 2026-09-08 23:25:29 -07:00
discountchubbs 1371618685 Update model_replay.yaml 2026-09-08 23:17:20 -07:00
discountchubbs 3f3e918a42 compile both at same time FULL SPEED AHEAD 2026-09-08 23:09:14 -07:00
discountchubbs c625b719c1 Update model_replay.yaml 2026-09-08 23:01:21 -07:00
discountchubbs 86f67d7aa9 Update model_replay.yaml 2026-09-08 22:59:21 -07:00
discountchubbs 857eb5e135 Update model_replay.yaml 2026-09-08 22:47:42 -07:00
discountchubbs c2d0b415af replay deez 🌰 2026-09-08 22:42:28 -07:00
7 changed files with 337 additions and 176 deletions
+148
View File
@@ -0,0 +1,148 @@
name: Test Stock vs Sunnypilot Model Equivalence
on:
workflow_dispatch:
inputs:
model_ref:
description: 'Upstream openpilot commit ref'
required: false
default: ''
pull_request:
paths:
- 'openpilot/selfdrive/modeld/**'
- 'openpilot/sunnypilot/modeld_v2/**'
jobs:
test_stock_parity:
name: Compare Stock vs Sunnypilot Model Replay
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- run: ./tools/op.sh setup
- run: scons -j$(nproc 2>/dev/null || sysctl -n hw.logicalcpu) openpilot/cereal msgq_repo openpilot/common
- name: Fetch Big Model ONNX
run: |
mkdir -p /tmp/onnx_models
if [ -n "${{ inputs.model_ref }}" ]; then
echo "Fetching ONNX from upstream openpilot ref ${{ inputs.model_ref }}..."
git clone --depth 1 https://github.com/commaai/openpilot.git /tmp/upstream_openpilot
cd /tmp/upstream_openpilot
git fetch --depth 1 origin ${{ inputs.model_ref }}
git checkout ${{ inputs.model_ref }}
git lfs pull -I "**/selfdrive/modeld/models/big_driving_supercombo.onnx"
find . -name "big_driving_supercombo.onnx" -exec cp {} /tmp/onnx_models/ \;
else
cp openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx /tmp/onnx_models/
fi
- name: Compile models
env:
DEV: "CPU"
JIT_BATCH_SIZE: "0"
run: |
BIG_ONNX="/tmp/onnx_models/big_driving_supercombo.onnx"
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
CAMERA_RESOLUTIONS=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
python3 openpilot/selfdrive/modeld/compile_modeld.py \
--onnx "$BIG_ONNX" \
--model-size "$MODEL_SIZE" \
--camera-resolutions $CAMERA_RESOLUTIONS \
--output /tmp/stock_model.pkl \
--frame-skip 4 \
--benchmark-runs 1 &
python3 openpilot/sunnypilot/modeld_v2/compile_modeld.py \
--model-type supercombo \
--supercombo-onnx "$BIG_ONNX" \
--model-size "$MODEL_SIZE" \
--camera-resolutions $CAMERA_RESOLUTIONS \
--output /tmp/sunnypilot_model.pkl \
--frame-skip 4 \
--benchmark-runs 1 &
wait
- name: Run model replay
env:
DEV: "CPU"
run: |
python3 openpilot/sunnypilot/modeld_v2/model_replay.py \
--sunnypilot-model /tmp/sunnypilot_model.pkl \
--stock-model /tmp/stock_model.pkl \
--frames 20 \
--plot-dir /tmp/replay_plots
- name: Upload Replay Plots
uses: actions/upload-artifact@v4
if: always()
continue-on-error: true
with:
name: model_replay_plots_${{ github.event.number || github.sha }}
path: /tmp/replay_plots
- name: Checkout ci-artifacts
if: github.repository == 'sunnypilot/sunnypilot' && github.event_name == 'pull_request'
uses: actions/checkout@v4
with:
repository: sunnypilot/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
path: ${{ github.workspace }}/ci-artifacts
- name: Push plots to ci-artifacts
if: github.repository == 'sunnypilot/sunnypilot' && github.event_name == 'pull_request'
working-directory: ${{ github.workspace }}/ci-artifacts
run: |
git config user.name "GitHub Actions Bot"
git config user.email "<>"
BRANCH="model_replay_pr_${{ github.event.number }}"
git fetch origin $BRANCH || true
git checkout $BRANCH 2>/dev/null || git checkout --orphan $BRANCH
rm -rf plots && mkdir -p plots
cp /tmp/replay_plots/*.png plots/
echo "${{ github.sha }}" > ref_commit
git add plots ref_commit
git commit -m "Model replay plots for PR #${{ github.event.number }}@${{ github.sha }}" || echo "No changes to commit"
git push origin $BRANCH --force
- name: Comment Model Replay Report on PR
if: github.repository == 'sunnypilot/sunnypilot' && github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const prNumber = context.payload.pull_request.number;
const branch = `model_replay_pr_${prNumber}`;
const baseUrl = `https://raw.githubusercontent.com/sunnypilot/ci-artifacts/refs/heads/${branch}/plots`;
const priorityPlots = ['desiredCurvature.png', 'desiredAcceleration.png', 'velocity.x.png', 'leadsV3.x.png', 'execution_timings.png'];
const allFiles = fs.readdirSync('/tmp/replay_plots').filter(f => f.endsWith('.png'));
const orderedFiles = [
...priorityPlots.filter(f => allFiles.includes(f)),
...allFiles.filter(f => !priorityPlots.includes(f)).sort()
];
let table = '<table>';
for (let i = 0; i < orderedFiles.length; i += 2) {
table += '<tr>';
table += `<td><img src="${baseUrl}/${orderedFiles[i]}" alt="${orderedFiles[i]}"><br><b>${orderedFiles[i].replace('.png', '')}</b></td>`;
if (i + 1 < orderedFiles.length) {
table += `<td><img src="${baseUrl}/${orderedFiles[i+1]}" alt="${orderedFiles[i+1]}"><br><b>${orderedFiles[i+1].replace('.png', '')}</b></td>`;
} else {
table += '<td></td>';
}
table += '</tr>';
}
table += '</table>';
const body = `### Model Replay Parity Report for PR #${prNumber} (@${context.sha.substring(0, 7)})\n\n` +
`<details><summary>All Model Replay Plots</summary>\n\n${table}\n\n</details>`;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: body
});
+1 -3
View File
@@ -211,9 +211,7 @@ class Soundd(QuietMode):
def main():
from openpilot.sunnypilot.selfdrive.ui.soundd import SounddSP
s = SounddSP()
s = Soundd()
s.soundd_thread()
@@ -0,0 +1,187 @@
"""
Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
This file is part of sunnypilot and is licensed under the MIT License.
See the LICENSE.md file in the root directory for more details.
"""
import argparse
import os
import sys
import time
import matplotlib.pyplot as plt
import numpy as np
from tinygrad.device import Device
from openpilot.common.file_chunker import open_file_chunked
from openpilot.selfdrive.modeld.compile_modeld import MODELD_INPUTS, make_input_queues, nv12_copy_size
from openpilot.selfdrive.modeld.helpers import load_oob
from openpilot.selfdrive.test.process_replay.model_replay import SEGMENT, TEST_ROUTE
from openpilot.system.camerad.cameras.nv12_info import get_nv12_info
from openpilot.tools.lib.framereader import FrameReader
from openpilot.tools.lib.openpilotci import get_url
from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip
from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser
def get_replay_video_source(route_or_path=None, segment_index=SEGMENT, camera_type="fcamera.hevc"):
if route_or_path and os.path.exists(route_or_path):
return route_or_path
selected_route = route_or_path or TEST_ROUTE
return get_url(selected_route, segment_index, camera_type)
def initialize_replay_queues(model_dictionary, device="CPU"):
metadata = model_dictionary.get("metadata", {})
model_meta = metadata.get("model", metadata)
input_shapes = model_meta.get("input_shapes", {})
cam_resolutions = list(model_dictionary.get("run_model", {}).keys())
cam_width, cam_height = cam_resolutions[0] if cam_resolutions else (1928, 1208)
nv12_info = get_nv12_info(cam_width, cam_height)
frame_copy_size = nv12_copy_size(nv12_info[0], nv12_info[1], nv12_info[2])
frame_skip = model_meta.get("frame_skip") or derive_frame_skip({}, input_shapes)
queues, npy_views, frame_views = make_input_queues(input_shapes, frame_skip, device, frame_copy_size)
if "tfm" in npy_views:
npy_views["tfm"][:] = np.eye(3, dtype=np.float32)
if "big_tfm" in npy_views:
npy_views["big_tfm"][:] = np.eye(3, dtype=np.float32)
if "traffic_convention" in npy_views:
npy_views["traffic_convention"][:] = np.array([1.0, 0.0], dtype=np.float32)
return queues, npy_views, frame_views, model_meta
def replay_model_on_frames(model_path, frame_reader, number_of_frames=20):
with open_file_chunked(model_path) as file_handle:
model_data = load_oob(file_handle)
run_model_dict = model_data.get("run_model", {})
runner = next(iter(run_model_dict.values()), None)
if runner is None:
raise ValueError("Failed to resolve runner from model dictionary")
queues, npy_views, frame_views, model_meta = initialize_replay_queues(model_data)
output_slices = model_meta.get("output_slices", {})
hidden_state_slice = output_slices.get("hidden_state")
parser = Parser(ignore_missing=True)
recorded_outputs = []
max_frames = min(number_of_frames, getattr(frame_reader, "frame_count", number_of_frames))
for frame_index in range(max_frames):
frame_raw = frame_reader.get(frame_index)
if frame_raw is not None:
for view in frame_views.values():
copy_length = min(view.size, frame_raw.size)
view.flat[:copy_length] = frame_raw.flat[:copy_length]
execution_arguments = {key: queues[key] for key in MODELD_INPUTS if key in queues}
execution_start = time.perf_counter()
step_output = runner(**execution_arguments)
Device.default.synchronize()
step_duration = time.perf_counter() - execution_start
output_array = (step_output[0].numpy() if hasattr(step_output[0], "numpy") else np.array(step_output[0]))
flat_output = output_array.flatten()
if hidden_state_slice and "prev_feat" in npy_views:
features_flat = flat_output[hidden_state_slice]
target_slice = min(features_flat.size, npy_views["prev_feat"].size)
npy_views["prev_feat"].flat[:target_slice] = features_flat[:target_slice]
sliced_outputs = {slice_name: flat_output[np.newaxis, slice_range] for slice_name, slice_range in output_slices.items()}
parser.parse_outputs(sliced_outputs)
recorded_outputs.append({
"frame_index": frame_index,
"raw_output": output_array,
"parsed_outputs": sliced_outputs,
"execution_time": step_duration,
})
return recorded_outputs
def plot_comparison(series_a, series_b, title, output_directory, label_a="modeld_v2 model", label_b="stock"):
os.makedirs(output_directory, exist_ok=True)
figure, axis = plt.subplots()
axis.plot(series_b, label=label_b)
axis.plot(series_a, label=label_a, linestyle="--")
axis.set_title(title)
axis.legend(loc="best")
plot_path = os.path.join(output_directory, f"{title}.png")
figure.savefig(plot_path)
plt.close(figure)
return plot_path
def compare_models_on_route(new_model_path, old_model_path, route_or_path=None, segment_index=SEGMENT,
number_of_frames=20, tolerance=1e-4, label_a="modeld_v2 model", label_b="stock",
plot_directory=None, enforce_timings=False):
video_url_or_path = get_replay_video_source(route_or_path, segment_index)
frame_reader = FrameReader(video_url_or_path, pix_fmt="nv12")
old_results = replay_model_on_frames(old_model_path, frame_reader, number_of_frames)
new_results = replay_model_on_frames(new_model_path, frame_reader, number_of_frames)
for step_index, (new_step, old_step) in enumerate(zip(new_results, old_results, strict=True)):
new_array = new_step["raw_output"]
old_array = old_step["raw_output"]
if not np.allclose(new_array, old_array, atol=tolerance, rtol=tolerance):
max_absolute_error = np.max(np.abs(new_array - old_array))
sys.stderr.write(f"Replay mismatch at frame {step_index}: max absolute error {max_absolute_error:.6f} exceeds tolerance {tolerance}\n")
return False
if len(new_results) > 1 and len(old_results) > 1:
new_timings = [step["execution_time"] * 1000.0 for step in new_results[1:] if "execution_time" in step]
old_timings = [step["execution_time"] * 1000.0 for step in old_results[1:] if "execution_time" in step]
if new_timings and old_timings:
print("------------------------------------------------")
print("----------------- Model Timing -----------------")
print("------------------------------------------------")
print(f"{label_a}: avg {np.mean(new_timings):6.2f} ms | max {np.max(new_timings):6.2f} ms")
print(f"{label_b}: avg {np.mean(old_timings):6.2f} ms | max {np.max(old_timings):6.2f} ms")
if plot_directory:
first_step_outputs = new_results[0].get("parsed_outputs", {})
if "action" in first_step_outputs:
series_a_curv = [step["parsed_outputs"]["action"].flatten()[0] for step in new_results]
series_b_curv = [step["parsed_outputs"]["action"].flatten()[0] for step in old_results]
plot_comparison(series_a_curv, series_b_curv, "desiredCurvature", plot_directory, label_a, label_b)
series_a_accel = [step["parsed_outputs"]["action"].flatten()[1] for step in new_results]
series_b_accel = [step["parsed_outputs"]["action"].flatten()[1] for step in old_results]
plot_comparison(series_a_accel, series_b_accel, "desiredAcceleration", plot_directory, label_a, label_b)
if "plan" in first_step_outputs:
series_a_vel = [step["parsed_outputs"]["plan"].flatten()[0] for step in new_results]
series_b_vel = [step["parsed_outputs"]["plan"].flatten()[0] for step in old_results]
plot_comparison(series_a_vel, series_b_vel, "velocity.x", plot_directory, label_a, label_b)
if "lead" in first_step_outputs:
series_a_lead = [step["parsed_outputs"]["lead"].flatten()[0] for step in new_results]
series_b_lead = [step["parsed_outputs"]["lead"].flatten()[0] for step in old_results]
plot_comparison(series_a_lead, series_b_lead, "leadsV3.x", plot_directory, label_a, label_b)
plot_comparison(new_timings, old_timings, "execution_timings", plot_directory, label_a, label_b)
for slice_name in first_step_outputs:
series_a = [np.mean(step["parsed_outputs"][slice_name]) for step in new_results if slice_name in step["parsed_outputs"]]
series_b = [np.mean(step["parsed_outputs"][slice_name]) for step in old_results if slice_name in step["parsed_outputs"]]
if series_a and series_b:
plot_comparison(series_a, series_b, f"output_{slice_name}", plot_directory, label_a, label_b)
print(f"Replay comparison result on route ({label_a} vs {label_b}): True")
return True
if __name__ == "__main__":
argument_parser = argparse.ArgumentParser(description="Model Replay on Real Driving Video")
argument_parser.add_argument("--sunnypilot-model", dest="model_a", default=None)
argument_parser.add_argument("--stock-model", dest="model_b", default=None)
argument_parser.add_argument("--route", default=TEST_ROUTE)
argument_parser.add_argument("--segment", type=int, default=SEGMENT)
argument_parser.add_argument("--frames", type=int, default=20)
argument_parser.add_argument("--plot-dir", default=None)
parsed_arguments = argument_parser.parse_args()
matches = compare_models_on_route(parsed_arguments.model_a, parsed_arguments.model_b, route_or_path=parsed_arguments.route,
segment_index=parsed_arguments.segment, number_of_frames=parsed_arguments.frames,
tolerance=1e-4, label_a="modeld_v2 model",
label_b="stock", plot_directory=parsed_arguments.plot_dir)
if not matches:
sys.exit(1)
@@ -1,125 +0,0 @@
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import os
import sys
import threading
import time
import wave
import numpy as np
from openpilot.common.swaglog import cloudlog
AUDIO_DIR = Path("/data/media/0/custom_audio")
SAMPLE_RATE = 48000
MAX_SECONDS = 300
GEAR_TIMEOUT = 0.5
DUCK_FRAMES = int(0.05 * SAMPLE_RATE)
RESTORE_FRAMES = int(0.15 * SAMPLE_RATE)
class GearTransition:
def __init__(self):
self.previous = None
self.last_time = None
def update(self, gear, timestamp, valid=True):
if not valid or gear == "unknown":
self.previous = self.last_time = None
return None
if self.last_time is None or not 0 < timestamp - self.last_time <= GEAR_TIMEOUT:
self.previous = None
transition = gear if self.previous is not None and gear != self.previous else None
self.previous = gear
self.last_time = timestamp
return transition
def lower_loader_priority():
if sys.platform == "linux":
try:
tid = threading.get_native_id()
os.sched_setscheduler(tid, os.SCHED_OTHER, os.sched_param(0))
os.setpriority(os.PRIO_PROCESS, tid, max(10, os.getpriority(os.PRIO_PROCESS, tid)))
except OSError:
cloudlog.exception("Unable to lower custom audio loader priority")
def load_audio(filename):
path = AUDIO_DIR / filename
if not path.is_file():
return None
with wave.open(str(path), "rb") as wav:
if (wav.getnchannels(), wav.getsampwidth(), wav.getframerate(), wav.getcomptype()) != (1, 2, SAMPLE_RATE, "NONE"):
raise ValueError("Custom audio requires 48 kHz mono 16-bit PCM WAV")
count = wav.getnframes()
if not 0 < count <= MAX_SECONDS * SAMPLE_RATE:
raise ValueError("Custom audio must be between 0 and 300 seconds")
raw = wav.readframes(count)
if len(raw) != count * 2:
raise ValueError("Truncated custom audio")
return np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768
class AudioPlayer:
def __init__(self):
self.command = None
self.seen_command = None
self.samples = np.empty(0, dtype=np.float32)
self.position = 0
self.gain = 1.0
def render(self, frames, alert_active):
command = self.command
if command is not self.seen_command:
self.seen_command = command
self.samples = command if command is not None else np.empty(0, dtype=np.float32)
self.position = 0
self.gain = 1.0
out = np.zeros(frames, dtype=np.float32)
if self.position == len(self.samples):
return out
n = min(frames, len(self.samples) - self.position)
step = -0.5 / DUCK_FRAMES if alert_active else 0.5 / RESTORE_FRAMES
gains = np.clip(self.gain + np.arange(n) * step, 0.5, 1.0)
out[:n] = self.samples[self.position:self.position + n] * gains
self.position += n
self.gain = float(np.clip(self.gain + n * step, 0.5, 1.0))
return out
class CustomAudio:
def __init__(self):
self.player = AudioPlayer()
self.transition = GearTransition()
self.loader = ThreadPoolExecutor(max_workers=1, thread_name_prefix="custom-audio-source", initializer=lower_loader_priority)
self.pending = None
def update(self, messages):
now = time.monotonic()
event = None
if self.transition.last_time is not None and now - self.transition.last_time > GEAR_TIMEOUT:
self.transition.update(None, now, False)
for msg in messages:
timestamp = msg.logMonoTime / 1e9
valid = msg.valid and msg.carState.canValid and msg.carState.gearShifter != "unknown" and 0 <= now - timestamp <= GEAR_TIMEOUT
transition = self.transition.update(str(msg.carState.gearShifter), timestamp, valid)
if not valid:
event = None
elif transition:
event = transition
if event:
self.player.command = None
if self.pending:
self.pending.cancel()
self.pending = self.loader.submit(load_audio, f"{event}.wav")
if self.pending and self.pending.done():
try:
samples = self.pending.result()
if samples is not None:
self.player.command = samples
except Exception:
cloudlog.exception("Unable to load custom audio")
self.pending = None
@@ -1,22 +0,0 @@
# Custom Audio
Put short **48 kHz, mono, 16-bit PCM WAV** files in `/data/media/0/custom_audio/` (maximum five minutes each).
Each gear change plays `<gear>.wav`: `park.wav`, `drive.wav`, `sport.wav`, `reverse.wav`, `neutral.wav`, `low.wav`, `brake.wav`, `eco.wav`, or `manumatic.wav`.
- Entering a gear starts its audio from the beginning at 25% volume, with no fade-in.
- Changing gears cuts the previous audio without a fade-out. If the new file is missing or invalid, playback stays silent.
- Returning to a previous gear restarts its audio from the beginning. Staying in the same gear does not replay it.
- While any alert sounds, custom audio keeps playing and fades to **half its normal level (12.5%) over 50 ms**. Afterward it fades back to 25% over **150 ms**.
- Alert volume is unchanged. Custom audio may be reduced further to prevent clipping when both sounds are loud.
- Startup and recovery from unknown/invalid gear data or data gaps establish a silent baseline.
WAV loading runs on a low-priority worker thread on Linux (nice +10). Playback shares the existing safety-alert output callback; its priority is unchanged.
No settings or dismiss control. Replace or remove files to change future playback.
Convert files with:
```sh
ffmpeg -i input_audio -ar 48000 -ac 1 -c:a pcm_s16le drive.wav
```
@@ -1,25 +0,0 @@
import numpy as np
from openpilot.cereal import messaging
from openpilot.selfdrive.ui.soundd import Soundd
from openpilot.sunnypilot.selfdrive.ui.custom_audio import CustomAudio
class SounddSP(Soundd):
def __init__(self):
super().__init__()
self.custom_audio = CustomAudio()
self.gear_sock = messaging.sub_sock("carState", conflate=False)
def callback(self, data_out: np.ndarray, frames: int, time, status) -> None:
super().callback(data_out, frames, time, status)
alert_data = data_out[:frames, 0]
alert_peak = float(np.max(np.abs(alert_data)))
custom_data = self.custom_audio.player.render(frames, alert_peak > 0) * 0.25
custom_peak = float(np.max(np.abs(custom_data)))
gain = min(1.0, max(0.0, 1.0 - alert_peak) / max(custom_peak, 1e-9))
alert_data += custom_data * gain
def get_audible_alert(self, sm):
super().get_audible_alert(sm)
self.custom_audio.update(messaging.drain_sock(self.gear_sock, wait_for_one=False))