Merge branch 'upstream/openpilot/master' into sync-20250215

# Conflicts:
#	common/params.cc
#	opendbc_repo
#	panda
#	selfdrive/car/tests/test_models.py
#	selfdrive/modeld/fill_model_msg.py
#	selfdrive/test/process_replay/process_replay.py
#	selfdrive/test/process_replay/ref_commit
#	selfdrive/ui/qt/offroad/developer_panel.cc
#	selfdrive/ui/tests/test_ui/run.py
#	selfdrive/ui/translations/main_ar.ts
#	selfdrive/ui/translations/main_de.ts
#	selfdrive/ui/translations/main_es.ts
#	selfdrive/ui/translations/main_fr.ts
#	selfdrive/ui/translations/main_ja.ts
#	selfdrive/ui/translations/main_ko.ts
#	selfdrive/ui/translations/main_pt-BR.ts
#	selfdrive/ui/translations/main_th.ts
#	selfdrive/ui/translations/main_tr.ts
#	selfdrive/ui/translations/main_zh-CHS.ts
#	selfdrive/ui/translations/main_zh-CHT.ts
#	system/athena/athenad.py
This commit is contained in:
Jason Wen
2025-02-15 20:19:43 -05:00
141 changed files with 2619 additions and 1336 deletions
+1 -1
View File
@@ -226,7 +226,7 @@ std::pair<CanEventIter, CanEventIter> AbstractStream::eventsInRange(const Messag
if (!time_range) return {events.begin(), events.end()};
auto first = std::lower_bound(events.begin(), events.end(), can->toMonoTime(time_range->first), CompareCanEvent());
auto last = std::upper_bound(events.begin(), events.end(), can->toMonoTime(time_range->second), CompareCanEvent());
auto last = std::upper_bound(first, events.end(), can->toMonoTime(time_range->second), CompareCanEvent());
return {first, last};
}
+2 -17
View File
@@ -8,22 +8,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
ROOT="$DIR"/../
cd "$ROOT"
# updating uv on macOS results in 403 sometimes
function update_uv() {
if ! uv self update --help >/dev/null 2>&1; then
return 0
fi
for i in $(seq 1 5); do
if uv self update; then
return 0
else
sleep 2
fi
done
echo "Failed to update uv 5 times!"
}
if ! command -v "uv" > /dev/null 2>&1; then
echo "installing uv..."
curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -32,7 +16,8 @@ if ! command -v "uv" > /dev/null 2>&1; then
fi
echo "updating uv..."
update_uv
# ok to fail, can also fail due to installing with brew
uv self update || true
echo "installing python packages..."
uv sync --frozen --all-extras
+6 -22
View File
@@ -45,6 +45,7 @@ function install_ubuntu_common_requirements() {
libgles2-mesa-dev \
libglfw3-dev \
libglib2.0-0 \
libjpeg-dev \
libqt5charts5-dev \
libncurses5-dev \
libssl-dev \
@@ -79,16 +80,6 @@ function install_ubuntu_lts_latest_requirements() {
python3-venv
}
# Install Ubuntu 20.04 packages
function install_ubuntu_focal_requirements() {
install_ubuntu_common_requirements
$SUDO apt-get install -y --no-install-recommends \
libavresample-dev \
qt5-default \
python-dev
}
# Detect OS using /etc/os-release file
if [ -f "/etc/os-release" ]; then
source /etc/os-release
@@ -96,9 +87,6 @@ if [ -f "/etc/os-release" ]; then
"jammy" | "kinetic" | "noble")
install_ubuntu_lts_latest_requirements
;;
"focal")
install_ubuntu_focal_requirements
;;
*)
echo "$ID $VERSION_ID is unsupported. This setup script is written for Ubuntu 24.04."
read -p "Would you like to attempt installation anyway? " -n 1 -r
@@ -106,24 +94,20 @@ if [ -f "/etc/os-release" ]; then
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
exit 1
fi
if [ "$UBUNTU_CODENAME" = "focal" ]; then
install_ubuntu_focal_requirements
else
install_ubuntu_lts_latest_requirements
fi
install_ubuntu_lts_latest_requirements
esac
if [[ -d "/etc/udev/rules.d/" ]]; then
# Setup panda udev rules
$SUDO tee /etc/udev/rules.d/12-panda_jungle.rules > /dev/null <<EOF
SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddcf", MODE="0666"
SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddef", MODE="0666"
SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcf", MODE="0666"
SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddef", MODE="0666"
EOF
# Setup jungle udev rules
$SUDO tee /etc/udev/rules.d/11-panda.rules > /dev/null <<EOF
SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddcc", MODE="0666"
SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddee", MODE="0666"
SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcc", MODE="0666"
SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddee", MODE="0666"
EOF
$SUDO udevadm control --reload-rules && $SUDO udevadm trigger || true
+9 -1
View File
@@ -38,6 +38,14 @@ def save_log(dest, log_msgs, compress=True):
with open(dest, "wb") as f:
f.write(dat)
def decompress_stream(data: bytes):
dctx = zstd.ZstdDecompressor()
decompressed_data = b""
with dctx.stream_reader(data) as reader:
decompressed_data = reader.read()
return decompressed_data
class _LogFileReader:
def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False, dat=None):
@@ -58,7 +66,7 @@ class _LogFileReader:
dat = bz2.decompress(dat)
elif ext == ".zst" or dat.startswith(b'\x28\xB5\x2F\xFD'):
# https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
dat = zstd.decompress(dat)
dat = decompress_stream(dat)
ents = capnp_log.Event.read_multiple_bytes(dat)
@@ -159,7 +159,7 @@ if __name__ == '__main__':
lr = LogReader(args.route)
else:
segs = [seg for seg in os.listdir(Paths.log_root()) if args.route in seg]
lr = LogReader([os.path.join(Paths.log_root(), seg, 'rlog') for seg in segs])
lr = LogReader([os.path.join(Paths.log_root(), seg, 'rlog.zst') for seg in segs])
CP = lr.first('carParams')
ID = lr.first('initData')
+9 -10
View File
@@ -243,6 +243,11 @@ function op_setup() {
op_check
}
function op_auth() {
op_before_cmd
op_run_command tools/lib/auth.py
}
function op_activate_venv() {
# bash 3.2 can't handle this without the 'set +e'
set +e
@@ -340,7 +345,7 @@ function op_switch() {
function op_start() {
if [[ -f "/AGNOS" ]]; then
op_before_cmd
op_run_command sudo systemctl start comma $@
op_run_command sudo systemctl restart comma $@
fi
}
@@ -351,13 +356,6 @@ function op_stop() {
fi
}
function op_restart() {
if [[ -f "/AGNOS" ]]; then
op_before_cmd
op_run_command sudo systemctl restart comma $@
fi
}
function op_default() {
echo "An openpilot helper"
echo ""
@@ -374,15 +372,15 @@ function op_default() {
echo -e "${BOLD}${UNDERLINE}Usage:${NC} op [OPTIONS] <COMMAND>"
echo ""
echo -e "${BOLD}${UNDERLINE}Commands [System]:${NC}"
echo -e " ${BOLD}auth${NC} Authenticate yourself for API use"
echo -e " ${BOLD}check${NC} Check the development environment (git, os, python) to start using openpilot"
echo -e " ${BOLD}venv${NC} Activate the python virtual environment"
echo -e " ${BOLD}setup${NC} Install openpilot dependencies"
echo -e " ${BOLD}build${NC} Run the openpilot build system in the current working directory"
echo -e " ${BOLD}install${NC} Install the 'op' tool system wide"
echo -e " ${BOLD}switch${NC} Switch to a different git branch with a clean slate (nukes any changes)"
echo -e " ${BOLD}start${NC} Starts openpilot"
echo -e " ${BOLD}start${NC} Starts (or restarts) openpilot"
echo -e " ${BOLD}stop${NC} Stops openpilot"
echo -e " ${BOLD}restart${NC} Restarts openpilot"
echo ""
echo -e "${BOLD}${UNDERLINE}Commands [Tooling]:${NC}"
echo -e " ${BOLD}juggle${NC} Run PlotJuggler"
@@ -427,6 +425,7 @@ function _op() {
# parse Commands
case $1 in
auth ) shift 1; op_auth "$@" ;;
venv ) shift 1; op_venv "$@" ;;
check ) shift 1; op_check "$@" ;;
setup ) shift 1; op_setup "$@" ;;
+6 -2
View File
@@ -13,6 +13,7 @@ from functools import partial
from opendbc.car.fingerprints import MIGRATION
from openpilot.common.basedir import BASEDIR
from openpilot.tools.lib.logreader import LogReader, ReadMode, save_log
from openpilot.selfdrive.test.process_replay.migration import migrate_all
juggle_dir = os.path.dirname(os.path.realpath(__file__))
@@ -76,10 +77,12 @@ def process(can, lr):
return [d for d in lr if can or d.which() not in ['can', 'sendcan']]
def juggle_route(route_or_segment_name, can, layout, dbc=None):
def juggle_route(route_or_segment_name, can, layout, dbc, should_migrate):
sr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACTIVE)
all_data = sr.run_across_segments(24, partial(process, can))
if should_migrate:
all_data = migrate_all(all_data)
# Infer DBC name from logs
if dbc is None:
@@ -105,6 +108,7 @@ if __name__ == "__main__":
parser.add_argument("--demo", action="store_true", help="Use the demo route instead of providing one")
parser.add_argument("--can", action="store_true", help="Parse CAN data")
parser.add_argument("--stream", action="store_true", help="Start PlotJuggler in streaming mode")
parser.add_argument("--no-migration", action="store_true", help="Do not perform log migration")
parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout")
parser.add_argument("--install", action="store_true", help="Install or update PlotJuggler + plugins")
parser.add_argument("--dbc", help="Set the DBC name to load for parsing CAN data. If not set, the DBC will be automatically inferred from the logs.")
@@ -131,4 +135,4 @@ if __name__ == "__main__":
start_juggler(layout=args.layout)
else:
route_or_segment_name = DEMO_ROUTE if args.demo else args.route_or_segment_name.strip()
juggle_route(route_or_segment_name, args.can, args.layout, args.dbc)
juggle_route(route_or_segment_name, args.can, args.layout, args.dbc, not args.no_migration)
+6 -5
View File
@@ -8,14 +8,15 @@ echo boot > trace_clock
echo 1000 > buffer_size_kb
# /sys/kernel/tracing/available_events
echo 1 > events/irq/enable
echo 1 > events/sched/enable
echo 1 > events/kgsl/enable
echo 1 > events/camera/enable
echo 0 > events/enable
#echo 1 > events/irq/enable
#echo 1 > events/sched/enable
#echo 1 > events/kgsl/enable
#echo 1 > events/camera/enable
echo 1 > events/workqueue/enable
echo > trace
sleep 5
sleep 2
echo 0 > tracing_on
cp trace /tmp/trace
+8 -10
View File
@@ -8,7 +8,7 @@ import threading
os.environ['FILEREADER_CACHE'] = '1'
from openpilot.common.realtime import config_realtime_process, Ratekeeper, DT_CTRL
from openpilot.selfdrive.pandad import can_capnp_to_can_list
from openpilot.selfdrive.pandad import can_capnp_to_list
from openpilot.tools.lib.logreader import LogReader
from panda import PandaJungle
@@ -80,17 +80,15 @@ def connect():
time.sleep(1)
def process(lr):
return [can_capnp_to_can_list(m.can) for m in lr if m.which() == 'can']
def load_route(route_or_segment_name):
print("Loading log...")
sr = LogReader(route_or_segment_name)
CP = sr.first("carParams")
print(f"carFingerprint (for hardcoding fingerprint): '{CP.carFingerprint}'")
CAN_MSGS = sr.run_across_segments(24, process)
print("Finished loading...")
return CAN_MSGS
lr = LogReader(route_or_segment_name)
CP = lr.first("carParams")
print(f"carFingerprint: '{CP.carFingerprint}'")
mbytes = [m.as_builder().to_bytes() for m in lr if m.which() == 'can']
return [m[1] for m in can_capnp_to_list(mbytes)]
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Replay CAN messages from a route to all connected pandas and jungles in a loop.",
@@ -99,7 +97,7 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.route_or_segment_name is None:
args.route_or_segment_name = "77611a1fac303767/2020-03-24--09-50-38/1:3"
args.route_or_segment_name = "77611a1fac303767/2020-03-24--09-50-38/2:4"
CAN_MSGS = load_route(args.route_or_segment_name)
+8 -1
View File
@@ -177,7 +177,14 @@ bool VideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
break;
}
}
avio_seek(reader->input_ctx->pb, reader->packets_info[from_idx].pos, SEEK_SET);
auto pos = reader->packets_info[from_idx].pos;
int ret = avformat_seek_file(reader->input_ctx, 0, pos, pos, pos, AVSEEK_FLAG_BYTE);
if (ret < 0) {
rError("Failed to seek to byte position %lld: %d", pos, AVERROR(ret));
return false;
}
avcodec_flush_buffers(decoder_ctx);
}
reader->prev_idx = idx;
+6 -6
View File
@@ -109,6 +109,7 @@ void Replay::seekTo(double seconds, bool relative) {
interruptStream([&]() {
current_segment_.store(target_segment);
cur_mono_time_ = route_start_ts_ + target_time * 1e9;
cur_which_ = cereal::Event::Which::INIT_DATA;
seeking_to_.store(target_time, std::memory_order_relaxed);
return false;
});
@@ -250,7 +251,6 @@ void Replay::publishFrame(const Event *e) {
void Replay::streamThread() {
stream_thread_id = pthread_self();
cereal::Event::Which cur_which = cereal::Event::Which::INIT_DATA;
std::unique_lock lk(stream_lock_);
while (true) {
@@ -259,7 +259,7 @@ void Replay::streamThread() {
event_data_ = seg_mgr_->getEventData();
const auto &events = event_data_->events;
auto first = std::upper_bound(events.cbegin(), events.cend(), Event(cur_which, cur_mono_time_, {}));
auto first = std::upper_bound(events.cbegin(), events.cend(), Event(cur_which_, cur_mono_time_, {}));
if (first == events.cend()) {
rInfo("waiting for events...");
events_ready_ = false;
@@ -273,9 +273,7 @@ void Replay::streamThread() {
camera_server_->waitForSent();
}
if (it != events.cend()) {
cur_which = it->which;
} else if (!hasFlag(REPLAY_FLAG_NO_LOOP)) {
if (it == events.cend() && !hasFlag(REPLAY_FLAG_NO_LOOP)) {
int last_segment = seg_mgr_->route_.segments().rbegin()->first;
if (event_data_->isSegmentLoaded(last_segment)) {
rInfo("reaches the end of route, restart from beginning");
@@ -302,10 +300,12 @@ std::vector<Event>::const_iterator Replay::publishEvents(std::vector<Event>::con
seg_mgr_->setCurrentSegment(segment);
}
cur_mono_time_ = evt.mono_time;
cur_which_ = evt.which;
// Skip events if socket is not present
if (!sockets_[evt.which]) continue;
cur_mono_time_ = evt.mono_time;
const uint64_t current_nanos = nanos_since_boot();
const int64_t time_diff = (evt.mono_time - evt_start_ts) / speed_ - (current_nanos - loop_start_ts);
+1
View File
@@ -93,6 +93,7 @@ private:
std::time_t route_date_time_;
uint64_t route_start_ts_ = 0;
std::atomic<uint64_t> cur_mono_time_ = 0;
cereal::Event::Which cur_which_ = cereal::Event::Which::INIT_DATA;
double min_seconds_ = 0;
double max_seconds_ = 0;
SubMaster *sm_ = nullptr;
+9 -4
View File
@@ -6,7 +6,6 @@ SegmentManager::~SegmentManager() {
{
std::unique_lock lock(mutex_);
exit_ = true;
onSegmentMergedCallback_ = nullptr;
}
cv_.notify_one();
if (thread_.joinable()) thread_.join();
@@ -37,6 +36,8 @@ bool SegmentManager::load() {
void SegmentManager::setCurrentSegment(int seg_num) {
{
std::unique_lock lock(mutex_);
if (cur_seg_num_ == seg_num) return;
cur_seg_num_ = seg_num;
needs_update_ = true;
}
@@ -58,6 +59,8 @@ void SegmentManager::manageSegmentCache() {
auto end = std::next(begin, std::min<int>(segment_cache_limit_, std::distance(begin, segments_.end())));
begin = std::prev(end, std::min<int>(segment_cache_limit_, std::distance(segments_.begin(), end)));
lock.unlock();
loadSegmentsInRange(begin, cur, end);
bool merged = mergeSegments(begin, end);
@@ -65,8 +68,6 @@ void SegmentManager::manageSegmentCache() {
std::for_each(segments_.begin(), begin, [](auto &segment) { segment.second.reset(); });
std::for_each(end, segments_.end(), [](auto &segment) { segment.second.reset(); });
lock.unlock();
if (merged && onSegmentMergedCallback_) {
onSegmentMergedCallback_(); // Notify listener that segments have been merged
}
@@ -118,7 +119,11 @@ void SegmentManager::loadSegmentsInRange(SegmentMap::iterator begin, SegmentMap:
if (!segment_ptr) {
segment_ptr = std::make_shared<Segment>(
it->first, route_.at(it->first), flags_, filters_,
[this](int seg_num, bool success) { setCurrentSegment(cur_seg_num_); });
[this](int seg_num, bool success) {
std::unique_lock lock(mutex_);
needs_update_ = true;
cv_.notify_one();
});
}
if (segment_ptr->getState() == Segment::LoadState::Loading) {
+1 -1
View File
@@ -45,7 +45,7 @@ private:
std::mutex mutex_;
std::condition_variable cv_;
std::thread thread_;
std::atomic<int> cur_seg_num_ = -1;
int cur_seg_num_ = -1;
bool needs_update_ = false;
bool exit_ = false;
+11 -6
View File
@@ -1,11 +1,12 @@
import traceback
import cereal.messaging as messaging
from opendbc.can.packer import CANPacker
from opendbc.can.parser import CANParser
from opendbc.car.honda.values import HondaSafetyFlags
from openpilot.common.params import Params
from openpilot.selfdrive.pandad.pandad_api_impl import can_list_to_can_capnp
from openpilot.tools.sim.lib.common import SimulatorState
from panda.python import Panda
class SimulatedCar:
@@ -94,14 +95,18 @@ class SimulatedCar:
'controlsAllowed': True,
'safetyModel': 'hondaBosch',
'alternativeExperience': self.sm["carParams"].alternativeExperience,
'safetyParam': Panda.FLAG_HONDA_RADARLESS | Panda.FLAG_HONDA_BOSCH_LONG,
'safetyParam': HondaSafetyFlags.FLAG_HONDA_RADARLESS.value | HondaSafetyFlags.FLAG_HONDA_BOSCH_LONG.value,
}
self.pm.send('pandaStates', dat)
def update(self, simulator_state: SimulatorState):
self.send_can_messages(simulator_state)
try:
self.send_can_messages(simulator_state)
if self.idx % 50 == 0: # only send panda states at 2hz
self.send_panda_state(simulator_state)
if self.idx % 50 == 0: # only send panda states at 2hz
self.send_panda_state(simulator_state)
self.idx += 1
self.idx += 1
except Exception:
traceback.print_exc()
raise