Merge branch 'upstream/openpilot/master' into sync-20250710

# Conflicts:
#	.github/workflows/selfdrive_tests.yaml
#	README.md
#	common/params_keys.h
#	opendbc_repo
#	panda
#	selfdrive/controls/lib/longitudinal_planner.py
#	selfdrive/controls/lib/tests/test_latcontrol.py
#	selfdrive/ui/soundd.py
#	selfdrive/ui/translations/main_ar.ts
#	selfdrive/ui/translations/main_de.ts
#	selfdrive/ui/translations/main_es.ts
#	selfdrive/ui/translations/main_fr.ts
#	selfdrive/ui/translations/main_ja.ts
#	selfdrive/ui/translations/main_ko.ts
#	selfdrive/ui/translations/main_pt-BR.ts
#	selfdrive/ui/translations/main_th.ts
#	selfdrive/ui/translations/main_tr.ts
#	selfdrive/ui/translations/main_zh-CHS.ts
#	selfdrive/ui/translations/main_zh-CHT.ts
#	tinygrad_repo
This commit is contained in:
Jason Wen
2025-07-19 00:19:12 -04:00
64 changed files with 1457 additions and 1095 deletions

View File

@@ -29,7 +29,7 @@ env:
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
PYTEST: pytest --continue-on-collection-errors --durations=0 --durations-min=5 -n logical
jobs:
build_release:
@@ -163,12 +163,6 @@ jobs:
./selfdrive/ui/tests/create_test_translations.sh && \
QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
chmod -R 777 /tmp/comma_download_cache"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v4
with:
name: ${{ github.job }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
process_replay:
name: process replay
@@ -192,10 +186,8 @@ jobs:
- name: Run replay
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
run: |
${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
chmod -R 777 /tmp/comma_download_cache && \
coverage combine && \
coverage xml"
${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
chmod -R 777 /tmp/comma_download_cache"
- name: Print diff
id: print-diff
if: always()
@@ -207,7 +199,7 @@ jobs:
name: process_replay_diff.txt
path: selfdrive/test/process_replay/diff.txt
- name: Upload reference logs
if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
if: false # TODO: move this to github instead of azure
run: |
${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
- name: Run regen
@@ -216,12 +208,6 @@ jobs:
run: |
${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
chmod -R 777 /tmp/comma_download_cache"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v4
with:
name: ${{ github.job }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_cars:
name: cars
@@ -252,12 +238,6 @@ jobs:
env:
NUM_JOBS: 4
JOB_ID: ${{ matrix.job }}
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v4
with:
name: ${{ github.job }}-${{ matrix.job }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
car_docs_diff:
name: PR comments

View File

@@ -7,6 +7,7 @@ on:
env:
DAYS_BEFORE_PR_CLOSE: 2
DAYS_BEFORE_PR_STALE: 9
DAYS_BEFORE_PR_STALE_DRAFT: 30
jobs:
stale:
@@ -24,6 +25,28 @@ jobs:
exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
exempt-draft-pr: false
# issue config
days-before-issue-stale: -1 # ignore issues for now
# same as above, but give draft PRs more time
stale_drafts:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v9
with:
exempt-all-milestones: true
# pull request config
stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
stale-pr-label: stale
delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }}
days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
exempt-draft-pr: true
# issue config
days-before-issue-stale: -1 # ignore issues for now

2
.gitignore vendored
View File

@@ -70,8 +70,6 @@ flycheck_*
cppcheck_report.txt
comma*.sh
selfdrive/modeld/thneed/compile
selfdrive/modeld/models/*.thneed
selfdrive/modeld/models/*.pkl
sunnypilot/modeld*/thneed/compile
sunnypilot/modeld*/models/*.thneed

View File

@@ -73,7 +73,7 @@ By default, sunnypilot uploads the driving data to comma servers. You can also a
sunnypilot is open source software. The user is free to disable data collection if they wish to do so.
sunnypilot logs the road-facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
By using this software, you understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.

View File

@@ -1,5 +1,11 @@
Version 0.9.10 (2025-06-30)
Version 0.10.0 (2025-07-07)
========================
* New driving model
* Lead car ground-truth fixes
* Ported over VAE from the MLSIM stack
* New training architecture described in CVPR paper
* Enable live-learned steering actuation delay
* Opt-in audio recording for dashcam video
Version 0.9.9 (2025-05-23)
========================

View File

@@ -39,10 +39,6 @@ AddOption('--clazy',
action='store_true',
help='build with clazy')
AddOption('--compile_db',
action='store_true',
help='build clang compilation database')
AddOption('--ccflags',
action='store',
type='string',
@@ -234,8 +230,7 @@ if arch == "Darwin":
darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
env["LINKFLAGS"] += darwin_rpath_link_flags
if GetOption('compile_db'):
env.CompilationDatabase('compile_commands.json')
env.CompilationDatabase('compile_commands.json')
# Setup cache dir
default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'

View File

@@ -1083,7 +1083,7 @@ struct ModelDataV2 {
confidence @23: ConfidenceClass;
# Model perceived motion
temporalPose @21 :Pose;
temporalPoseDEPRECATED @21 :Pose;
# e2e lateral planner
action @26: Action;
@@ -2470,13 +2470,19 @@ struct DebugAlert {
struct UserFlag {
}
struct Microphone {
struct SoundPressure @0xdc24138990726023 {
soundPressure @0 :Float32;
# uncalibrated, A-weighted
soundPressureWeighted @3 :Float32;
soundPressureWeightedDb @1 :Float32;
filteredSoundPressureWeightedDb @2 :Float32;
filteredSoundPressureWeightedDbDEPRECATED @2 :Float32;
}
struct AudioData {
data @0 :Data;
sampleRate @1 :UInt32;
}
struct Touch {
@@ -2556,7 +2562,8 @@ struct Event {
livestreamDriverEncodeIdx @119 :EncodeIndex;
# microphone data
microphone @103 :Microphone;
soundPressure @103 :SoundPressure;
rawAudioData @147 :AudioData;
# systems stuff
androidLog @20 :AndroidLogEntry;

View File

@@ -73,7 +73,8 @@ _services: dict[str, tuple] = {
"navThumbnail": (True, 0.),
"qRoadEncodeIdx": (False, 20.),
"userFlag": (True, 0., 1),
"microphone": (True, 10., 10),
"soundPressure": (True, 10., 10),
"rawAudioData": (False, 20.),
# sunnypilot
"modelManagerSP": (False, 1., 1),

View File

@@ -1,13 +0,0 @@
comment: false
coverage:
status:
project:
default:
informational: true
patch: off
ignore:
- "**/test_*.py"
- "selfdrive/test/**"
- "system/version.py" # codecov changes depending on if we are in a branch or not
- "tools"

View File

@@ -1 +1 @@
#define DEFAULT_MODEL "Vegetarian Filet o Fish (Default)"
#define DEFAULT_MODEL "Tomb Raider 14 (Default)"

View File

@@ -99,9 +99,10 @@ inline static std::unordered_map<std::string, uint32_t> keys = {
{"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"PandaSignatures", CLEAR_ON_MANAGER_START},
{"PrimeType", PERSISTENT},
{"RecordAudio", PERSISTENT | BACKUP},
{"RecordFront", PERSISTENT | BACKUP},
{"RecordFrontLock", PERSISTENT}, // for the internal fleet
{"SecOCKey", PERSISTENT | DONT_LOG}, // Candidate for | BACKUP
{"SecOCKey", PERSISTENT | DONT_LOG | BACKUP},
{"RouteCount", PERSISTENT},
{"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"SshEnabled", PERSISTENT | BACKUP},

View File

@@ -1 +1 @@
#define COMMA_VERSION "0.9.10"
#define COMMA_VERSION "0.10.0"

View File

@@ -2,7 +2,6 @@ import contextlib
import gc
import os
import pytest
import random
from openpilot.common.prefix import OpenpilotPrefix
from openpilot.system.manager import manager
@@ -49,8 +48,6 @@ def clean_env():
@pytest.fixture(scope="function", autouse=True)
def openpilot_function_fixture(request):
random.seed(0)
with clean_env():
# setup a clean environment for each test
with OpenpilotPrefix(shared_download_cache=request.node.get_closest_marker("shared_download_cache") is not None) as prefix:

View File

@@ -187,7 +187,7 @@ A supported vehicle is one that just works when you install a comma device. All
|Lexus|GS F 2016|All|Stock|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=GS F 2016">Buy Here</a></sub></details>|||
|Lexus|IS 2017-19|All|Stock|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=IS 2017-19">Buy Here</a></sub></details>|||
|Lexus|IS 2022-24|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=IS 2022-24">Buy Here</a></sub></details>|||
|Lexus|LC 2024|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=LC 2024">Buy Here</a></sub></details>|||
|Lexus|LC 2024-25|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=LC 2024-25">Buy Here</a></sub></details>|||
|Lexus|NX 2018-19|All|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=NX 2018-19">Buy Here</a></sub></details>|||
|Lexus|NX 2020-21|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=NX 2020-21">Buy Here</a></sub></details>|||
|Lexus|NX Hybrid 2018-19|All|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v3<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=NX Hybrid 2018-19">Buy Here</a></sub></details>|||

View File

@@ -81,11 +81,9 @@ docs = [
]
testing = [
"coverage",
"hypothesis ==6.47.*",
"mypy",
"pytest",
"pytest-cov",
"pytest-cpp",
"pytest-subtests",
# https://github.com/pytest-dev/pytest-xdist/issues/1215
@@ -266,8 +264,5 @@ lint.flake8-implicit-str-concat.allow-multiline = false
"unittest".msg = "Use pytest"
"pyray.measure_text_ex".msg = "Use openpilot.system.ui.lib.text_measure"
[tool.coverage.run]
concurrency = ["multiprocessing", "thread"]
[tool.ruff.format]
quote-style = "preserve"

Binary file not shown.

View File

@@ -1,5 +1,4 @@
import numpy as np
from cereal import log
from opendbc.car.vehicle_model import ACCELERATION_DUE_TO_GRAVITY
from openpilot.common.realtime import DT_CTRL, DT_MDL
@@ -40,14 +39,6 @@ def clip_curvature(v_ego, prev_curvature, new_curvature, roll):
return float(new_curvature), limited_accel or limited_max_curv
def get_speed_error(modelV2: log.ModelDataV2, v_ego: float) -> float:
# ToDo: Try relative error, and absolute speed
if len(modelV2.temporalPose.trans):
vel_err = np.clip(modelV2.temporalPose.trans[0] - v_ego, -MAX_VEL_ERR, MAX_VEL_ERR)
return float(vel_err)
return 0.0
def get_accel_from_plan(speeds, accels, t_idxs, action_t=DT_MDL, vEgoStopping=0.05):
if len(speeds) == len(t_idxs):
v_now = speeds[0]

View File

@@ -31,7 +31,6 @@ class LatControlTorque(LatControl):
self.pid = PIDController(self.torque_params.kp, self.torque_params.ki,
k_f=self.torque_params.kf, pos_limit=self.steer_max, neg_limit=-self.steer_max)
self.torque_from_lateral_accel = CI.torque_from_lateral_accel()
self.use_steering_angle = self.torque_params.useSteeringAngle
self.steering_angle_deadzone_deg = self.torque_params.steeringAngleDeadzoneDeg
self.extension = LatControlTorqueExt(self, CP, CP_SP)
@@ -47,16 +46,9 @@ class LatControlTorque(LatControl):
output_torque = 0.0
pid_log.active = False
else:
actual_curvature_vm = -VM.calc_curvature(math.radians(CS.steeringAngleDeg - params.angleOffsetDeg), CS.vEgo, params.roll)
actual_curvature = -VM.calc_curvature(math.radians(CS.steeringAngleDeg - params.angleOffsetDeg), CS.vEgo, params.roll)
roll_compensation = params.roll * ACCELERATION_DUE_TO_GRAVITY
if self.use_steering_angle:
actual_curvature = actual_curvature_vm
curvature_deadzone = abs(VM.calc_curvature(math.radians(self.steering_angle_deadzone_deg), CS.vEgo, 0.0))
else:
assert calibrated_pose is not None
actual_curvature_pose = calibrated_pose.angular_velocity.yaw / CS.vEgo
actual_curvature = np.interp(CS.vEgo, [2.0, 5.0], [actual_curvature_vm, actual_curvature_pose])
curvature_deadzone = 0.0
curvature_deadzone = abs(VM.calc_curvature(math.radians(self.steering_angle_deadzone_deg), CS.vEgo, 0.0))
desired_lateral_accel = desired_curvature * CS.vEgo ** 2
# desired rate is the desired rate of change in the setpoint, not the absolute desired curvature

View File

@@ -11,7 +11,7 @@ from openpilot.selfdrive.modeld.constants import ModelConstants
from openpilot.selfdrive.controls.lib.longcontrol import LongCtrlState
from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import LongitudinalMpc
from openpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import T_IDXS as T_IDXS_MPC
from openpilot.selfdrive.controls.lib.drive_helpers import CONTROL_N, get_speed_error, get_accel_from_plan
from openpilot.selfdrive.controls.lib.drive_helpers import CONTROL_N, get_accel_from_plan
from openpilot.selfdrive.car.cruise import V_CRUISE_MAX, V_CRUISE_UNSET
from openpilot.common.swaglog import cloudlog
@@ -54,6 +54,7 @@ class LongitudinalPlanner(LongitudinalPlannerSP):
def __init__(self, CP, init_v=0.0, init_a=0.0, dt=DT_MDL):
self.CP = CP
self.mpc = LongitudinalMpc(dt=dt)
# TODO remove mpc modes when TR released
self.mpc.mode = 'acc'
LongitudinalPlannerSP.__init__(self, self.CP, self.mpc)
self.fcw = False
@@ -63,7 +64,6 @@ class LongitudinalPlanner(LongitudinalPlannerSP):
self.a_desired = init_a
self.v_desired_filter = FirstOrderFilter(init_v, 2.0, self.dt)
self.prev_accel_clip = [ACCEL_MIN, ACCEL_MAX]
self.v_model_error = 0.0
self.output_a_target = 0.0
self.output_should_stop = False
@@ -73,12 +73,12 @@ class LongitudinalPlanner(LongitudinalPlannerSP):
self.solverExecutionTime = 0.0
@staticmethod
def parse_model(model_msg, model_error):
def parse_model(model_msg):
if (len(model_msg.position.x) == ModelConstants.IDX_N and
len(model_msg.velocity.x) == ModelConstants.IDX_N and
len(model_msg.acceleration.x) == ModelConstants.IDX_N):
x = np.interp(T_IDXS_MPC, ModelConstants.T_IDXS, model_msg.position.x) - model_error * T_IDXS_MPC
v = np.interp(T_IDXS_MPC, ModelConstants.T_IDXS, model_msg.velocity.x) - model_error
x = np.interp(T_IDXS_MPC, ModelConstants.T_IDXS, model_msg.position.x)
v = np.interp(T_IDXS_MPC, ModelConstants.T_IDXS, model_msg.velocity.x)
a = np.interp(T_IDXS_MPC, ModelConstants.T_IDXS, model_msg.acceleration.x)
j = np.zeros(len(T_IDXS_MPC))
else:
@@ -137,9 +137,7 @@ class LongitudinalPlanner(LongitudinalPlannerSP):
# Prevent divergence, smooth in current v_ego
self.v_desired_filter.x = max(0.0, self.v_desired_filter.update(v_ego))
# Compute model v_ego error
self.v_model_error = get_speed_error(sm['modelV2'], v_ego)
x, v, a, j, throttle_prob = self.parse_model(sm['modelV2'], self.v_model_error)
x, v, a, j, throttle_prob = self.parse_model(sm['modelV2'])
# Don't clip at low speeds since throttle_prob doesn't account for creep
self.allow_throttle = throttle_prob > ALLOW_THROTTLE_THRESHOLD or v_ego <= MIN_ALLOW_THROTTLE_SPEED

View File

@@ -157,8 +157,7 @@ class LateralLagEstimator:
block_count: int = BLOCK_NUM, min_valid_block_count: int = BLOCK_NUM_NEEDED, block_size: int = BLOCK_SIZE,
window_sec: float = MOVING_WINDOW_SEC, okay_window_sec: float = MIN_OKAY_WINDOW_SEC, min_recovery_buffer_sec: float = MIN_RECOVERY_BUFFER_SEC,
min_vego: float = MIN_VEGO, min_yr: float = MIN_ABS_YAW_RATE, min_ncc: float = MIN_NCC,
max_lat_accel: float = MAX_LAT_ACCEL, max_lat_accel_diff: float = MAX_LAT_ACCEL_DIFF, min_confidence: float = MIN_CONFIDENCE,
enabled: bool = True):
max_lat_accel: float = MAX_LAT_ACCEL, max_lat_accel_diff: float = MAX_LAT_ACCEL_DIFF, min_confidence: float = MIN_CONFIDENCE):
self.dt = dt
self.window_sec = window_sec
self.okay_window_sec = okay_window_sec
@@ -173,7 +172,6 @@ class LateralLagEstimator:
self.min_confidence = min_confidence
self.max_lat_accel = max_lat_accel
self.max_lat_accel_diff = max_lat_accel_diff
self.enabled = enabled
self.t = 0.0
self.lat_active = False
@@ -208,7 +206,7 @@ class LateralLagEstimator:
liveDelay = msg.liveDelay
valid_mean_lag, valid_std, current_mean_lag, current_std = self.block_avg.get()
if self.enabled and self.block_avg.valid_blocks >= self.min_valid_block_count and not np.isnan(valid_mean_lag) and not np.isnan(valid_std):
if self.block_avg.valid_blocks >= self.min_valid_block_count and not np.isnan(valid_mean_lag) and not np.isnan(valid_std):
if valid_std > MAX_LAG_STD:
liveDelay.status = log.LiveDelayData.Status.invalid
else:
@@ -371,10 +369,7 @@ def main():
params = Params()
CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
# TODO: remove me, lagd is in shadow mode on release
is_release = params.get_bool("IsReleaseBranch")
lag_learner = LateralLagEstimator(CP, 1. / SERVICE_LIST['livePose'].frequency, enabled=not is_release)
lag_learner = LateralLagEstimator(CP, 1. / SERVICE_LIST['livePose'].frequency)
if (initial_lag_params := retrieve_initial_lag(params, CP)) is not None:
lag, valid_blocks = initial_lag_params
lag_learner.reset(lag, valid_blocks)

View File

@@ -110,19 +110,6 @@ class TestLagd:
assert msg.liveDelay.validBlocks == BLOCK_NUM_NEEDED
assert msg.liveDelay.calPerc == 100
def test_disabled_estimator(self):
mocked_CP = car.CarParams(steerActuatorDelay=0.8)
estimator = LateralLagEstimator(mocked_CP, DT, min_recovery_buffer_sec=0.0, min_yr=0.0, enabled=False)
lag_frames = 5
process_messages(estimator, lag_frames, int(MIN_OKAY_WINDOW_SEC / DT) + BLOCK_NUM_NEEDED * BLOCK_SIZE)
msg = estimator.get_msg(True)
assert msg.liveDelay.status == 'unestimated'
assert np.allclose(msg.liveDelay.lateralDelay, 1.0, atol=0.01)
assert np.allclose(msg.liveDelay.lateralDelayEstimate, lag_frames * DT, atol=0.01)
assert np.allclose(msg.liveDelay.lateralDelayEstimateStd, 0.0, atol=0.01)
assert msg.liveDelay.validBlocks == BLOCK_NUM_NEEDED
assert msg.liveDelay.calPerc == 100
def test_estimator_masking(self):
mocked_CP, lag_frames = car.CarParams(steerActuatorDelay=0.8), random.randint(1, 19)
estimator = LateralLagEstimator(mocked_CP, DT, min_recovery_buffer_sec=0.0, min_yr=0.0, min_valid_block_count=1)

View File

@@ -56,17 +56,15 @@ for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
tg_compile(flags, model_name)
# Compile BIG model if USB GPU is available
import subprocess
from tinygrad import Device
# because tg doesn't support multi-process
devs = subprocess.check_output('python3 -c "from tinygrad import Device; print(list(Device.get_available_devices()))"', shell=True, cwd=env.Dir('#').abspath)
if b"AMD" in devs:
del Device
print("USB GPU detected... building")
flags = "AMD=1 AMD_IFACE=USB AMD_LLVM=1 NOLOCALS=0 IMAGE=0"
bp = tg_compile(flags, "big_driving_policy")
bv = tg_compile(flags, "big_driving_vision")
lenv.SideEffect('lock', [bp, bv]) # tg doesn't support multi-process so build serially
else:
print("USB GPU not detected... skipping")
if "USBGPU" in os.environ:
import subprocess
# because tg doesn't support multi-process
devs = subprocess.check_output('python3 -c "from tinygrad import Device; print(list(Device.get_available_devices()))"', shell=True, cwd=env.Dir('#').abspath)
if b"AMD" in devs:
print("USB GPU detected... building")
flags = "AMD=1 AMD_IFACE=USB AMD_LLVM=1 NOLOCALS=0 IMAGE=0"
bp = tg_compile(flags, "big_driving_policy")
bv = tg_compile(flags, "big_driving_vision")
lenv.SideEffect('lock', [bp, bv]) # tg doesn't support multi-process so build serially
else:
print("USB GPU not detected... skipping")

View File

@@ -14,7 +14,6 @@ import pickle
import ctypes
import numpy as np
from pathlib import Path
from setproctitle import setproctitle
from cereal import messaging
from cereal.messaging import PubMaster, SubMaster
@@ -25,7 +24,6 @@ from openpilot.common.transformations.model import dmonitoringmodel_intrinsics,
from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye
from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame
from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid
from openpilot.system import sentry
MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE
CALIB_LEN = 3
@@ -133,12 +131,8 @@ def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts:
def main():
setproctitle(PROCESS_NAME)
config_realtime_process(7, 5)
sentry.set_tag("daemon", PROCESS_NAME)
cloudlog.bind(daemon=PROCESS_NAME)
cl_context = CLContext()
model = ModelState(cl_context)
cloudlog.warning("models loaded, dmonitoringmodeld starting")
@@ -180,7 +174,4 @@ if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
cloudlog.warning(f"child {PROCESS_NAME} got SIGINT")
except Exception:
sentry.capture_exception()
raise
cloudlog.warning("got SIGINT")

View File

@@ -89,13 +89,6 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
fill_xyzt(modelV2.orientation, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.T_FROM_CURRENT_EULER].T)
fill_xyzt(modelV2.orientationRate, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ORIENTATION_RATE].T)
# temporal pose
temporal_pose = modelV2.temporalPose
temporal_pose.trans = net_output_data['sim_pose'][0,:ModelConstants.POSE_WIDTH//2].tolist()
temporal_pose.transStd = net_output_data['sim_pose_stds'][0,:ModelConstants.POSE_WIDTH//2].tolist()
temporal_pose.rot = net_output_data['sim_pose'][0,ModelConstants.POSE_WIDTH//2:].tolist()
temporal_pose.rotStd = net_output_data['sim_pose_stds'][0,ModelConstants.POSE_WIDTH//2:].tolist()
# poly path
fill_xyz_poly(driving_model_data.path, ModelConstants.POLY_PATH_DEGREE, *net_output_data['plan'][0,:,Plan.POSITION].T)

View File

@@ -19,7 +19,6 @@ import numpy as np
import cereal.messaging as messaging
from cereal import car, log
from pathlib import Path
from setproctitle import setproctitle
from cereal.messaging import PubMaster, SubMaster
from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
from opendbc.car.car_helpers import get_demo_car_params
@@ -29,9 +28,8 @@ from openpilot.common.filter_simple import FirstOrderFilter
from openpilot.common.realtime import config_realtime_process, DT_MDL
from openpilot.common.transformations.camera import DEVICE_CAMERAS
from openpilot.common.transformations.model import get_warp_matrix
from openpilot.system import sentry
from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper
from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value
from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value, get_curvature_from_plan
from openpilot.selfdrive.modeld.parse_model_outputs import Parser
from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState
from openpilot.selfdrive.modeld.constants import ModelConstants, Plan
@@ -48,8 +46,8 @@ POLICY_PKL_PATH = Path(__file__).parent / 'models/driving_policy_tinygrad.pkl'
VISION_METADATA_PATH = Path(__file__).parent / 'models/driving_vision_metadata.pkl'
POLICY_METADATA_PATH = Path(__file__).parent / 'models/driving_policy_metadata.pkl'
LAT_SMOOTH_SECONDS = 0.0
LONG_SMOOTH_SECONDS = 0.0
LAT_SMOOTH_SECONDS = 0.1
LONG_SMOOTH_SECONDS = 0.3
MIN_LAT_CONTROL_SPEED = 0.3
@@ -62,7 +60,11 @@ def get_action_from_model(model_output: dict[str, np.ndarray], prev_action: log.
action_t=long_action_t)
desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, LONG_SMOOTH_SECONDS)
desired_curvature = model_output['desired_curvature'][0, 0]
desired_curvature = get_curvature_from_plan(plan[:,Plan.T_FROM_CURRENT_EULER][:,2],
plan[:,Plan.ORIENTATION_RATE][:,2],
ModelConstants.T_IDXS,
v_ego,
lat_action_t)
if v_ego > MIN_LAT_CONTROL_SPEED:
desired_curvature = smooth_value(desired_curvature, prev_action.desiredCurvature, LAT_SMOOTH_SECONDS)
else:
@@ -177,7 +179,7 @@ class ModelState:
# TODO model only uses last value now
self.full_prev_desired_curv[0,:-1] = self.full_prev_desired_curv[0,1:]
self.full_prev_desired_curv[0,-1,:] = policy_outputs_dict['desired_curvature'][0, :]
self.numpy_inputs['prev_desired_curv'][:] = self.full_prev_desired_curv[0, self.temporal_idxs]
self.numpy_inputs['prev_desired_curv'][:] = 0*self.full_prev_desired_curv[0, self.temporal_idxs]
combined_outputs_dict = {**vision_outputs_dict, **policy_outputs_dict}
if SEND_RAW_PRED:
@@ -189,9 +191,6 @@ class ModelState:
def main(demo=False):
cloudlog.warning("modeld init")
sentry.set_tag("daemon", PROCESS_NAME)
cloudlog.bind(daemon=PROCESS_NAME)
setproctitle(PROCESS_NAME)
if not USBGPU:
# USB GPU currently saturates a core so can't do this yet,
# also need to move the aux USB interrupts for good timings
@@ -379,7 +378,4 @@ if __name__ == "__main__":
args = parser.parse_args()
main(demo=args.demo)
except KeyboardInterrupt:
cloudlog.warning(f"child {PROCESS_NAME} got SIGINT")
except Exception:
sentry.capture_exception()
raise
cloudlog.warning("got SIGINT")

View File

@@ -88,6 +88,12 @@ class Parser:
self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,))
self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,))
self.parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,))
self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_LANE_LINES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH))
self.parse_mdn('road_edges', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_ROAD_EDGES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH))
self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION,
out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH))
for k in ['lead_prob', 'lane_lines_prob']:
self.parse_binary_crossentropy(k, outs)
self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN,ModelConstants.DESIRE_PRED_WIDTH))
self.parse_binary_crossentropy('meta', outs)
return outs
@@ -95,17 +101,10 @@ class Parser:
def parse_policy_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
self.parse_mdn('plan', outs, in_N=ModelConstants.PLAN_MHP_N, out_N=ModelConstants.PLAN_MHP_SELECTION,
out_shape=(ModelConstants.IDX_N,ModelConstants.PLAN_WIDTH))
self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_LANE_LINES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH))
self.parse_mdn('road_edges', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_ROAD_EDGES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH))
self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,))
self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION,
out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH))
if 'lat_planner_solution' in outs:
self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N,ModelConstants.LAT_PLANNER_SOLUTION_WIDTH))
if 'desired_curvature' in outs:
self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,))
for k in ['lead_prob', 'lane_lines_prob']:
self.parse_binary_crossentropy(k, outs)
self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,))
return outs

View File

@@ -1 +1 @@
f440c9e0469d32d350aa99ddaa8f44591a2ce690
de16c6fbe14e121c5e74cd944ce03a0e4672540d

View File

@@ -7,7 +7,7 @@ from openpilot.selfdrive.ui.layouts.settings.device import DeviceLayout
from openpilot.selfdrive.ui.layouts.settings.firehose import FirehoseLayout
from openpilot.selfdrive.ui.layouts.settings.software import SoftwareLayout
from openpilot.selfdrive.ui.layouts.settings.toggles import TogglesLayout
from openpilot.system.ui.lib.application import gui_app, FontWeight
from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos
from openpilot.system.ui.lib.text_measure import measure_text_cached
from openpilot.selfdrive.ui.layouts.network import NetworkLayout
from openpilot.system.ui.lib.widget import Widget
@@ -132,7 +132,7 @@ class SettingsLayout(Widget):
if panel.instance:
panel.instance.render(content_rect)
def _handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool:
def _handle_mouse_release(self, mouse_pos: MousePos) -> bool:
# Check close button
if rl.check_collision_point_rec(mouse_pos, self._close_btn_rect):
if self._close_callback:

View File

@@ -22,6 +22,7 @@ DESCRIPTIONS = {
"AlwaysOnDM": "Enable driver monitoring even when openpilot is not engaged.",
'RecordFront': "Upload data from the driver facing camera and help improve the driver monitoring algorithm.",
"IsMetric": "Display speed in km/h instead of mph.",
"RecordAudio": "Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect.",
}
@@ -74,6 +75,12 @@ class TogglesLayout(Widget):
self._params.get_bool("RecordFront"),
icon="monitoring.png",
),
toggle_item(
"Record Microphone Audio",
DESCRIPTIONS["RecordAudio"],
self._params.get_bool("RecordAudio"),
icon="microphone.png",
),
toggle_item(
"Use Metric System", DESCRIPTIONS["IsMetric"], self._params.get_bool("IsMetric"), icon="monitoring.png"
),

View File

@@ -4,7 +4,7 @@ from dataclasses import dataclass
from collections.abc import Callable
from cereal import log
from openpilot.selfdrive.ui.ui_state import ui_state
from openpilot.system.ui.lib.application import gui_app, FontWeight
from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos
from openpilot.system.ui.lib.text_measure import measure_text_cached
from openpilot.system.ui.lib.widget import Widget
@@ -135,7 +135,7 @@ class Sidebar(Widget):
else:
self._panda_status.update("VEHICLE", "ONLINE", Colors.GOOD)
def _handle_mouse_release(self, mouse_pos: rl.Vector2):
def _handle_mouse_release(self, mouse_pos: MousePos):
if rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN):
if self._on_settings_click:
self._on_settings_click()

View File

@@ -68,6 +68,13 @@ TogglesPanel::TogglesPanel(SettingsWindow *parent) : ListWidget(parent) {
"../assets/icons/monitoring.png",
true,
},
{
"RecordAudio",
tr("Record and Upload Microphone Audio"),
tr("Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect."),
"../assets/icons/microphone.png",
true,
},
{
"IsMetric",
tr("Use Metric System"),
@@ -342,26 +349,22 @@ void DevicePanel::updateCalibDescription() {
}
}
const bool is_release = params.getBool("IsReleaseBranch");
if (!is_release) {
int lag_perc = 0;
std::string lag_bytes = params.get("LiveDelay");
if (!lag_bytes.empty()) {
try {
AlignedBuffer aligned_buf;
capnp::FlatArrayMessageReader cmsg(aligned_buf.align(lag_bytes.data(), lag_bytes.size()));
lag_perc = cmsg.getRoot<cereal::Event>().getLiveDelay().getCalPerc();
} catch (kj::Exception) {
qInfo() << "invalid LiveDelay";
}
}
desc += "\n\n";
if (lag_perc < 100) {
desc += tr("Steering lag calibration is %1% complete.").arg(lag_perc);
} else {
desc += tr("Steering lag calibration is complete.");
int lag_perc = 0;
std::string lag_bytes = params.get("LiveDelay");
if (!lag_bytes.empty()) {
try {
AlignedBuffer aligned_buf;
capnp::FlatArrayMessageReader cmsg(aligned_buf.align(lag_bytes.data(), lag_bytes.size()));
lag_perc = cmsg.getRoot<cereal::Event>().getLiveDelay().getCalPerc();
} catch (kj::Exception) {
qInfo() << "invalid LiveDelay";
}
}
if (lag_perc < 100) {
desc += tr("\n\nSteering lag calibration is %1% complete.").arg(lag_perc);
} else {
desc += tr("\n\nSteering lag calibration is complete.");
}
std::string torque_bytes = params.get("LiveTorqueParameters");
if (!torque_bytes.empty()) {
@@ -372,11 +375,10 @@ void DevicePanel::updateCalibDescription() {
// don't add for non-torque cars
if (torque.getUseParams()) {
int torque_perc = torque.getCalPerc();
desc += is_release ? "\n\n" : " ";
if (torque_perc < 100) {
desc += tr("Steering torque response calibration is %1% complete.").arg(torque_perc);
desc += tr(" Steering torque response calibration is %1% complete.").arg(torque_perc);
} else {
desc += tr("Steering torque response calibration is complete.");
desc += tr(" Steering torque response calibration is complete.");
}
}
} catch (kj::Exception) {

View File

@@ -24,10 +24,11 @@ void Sidebar::drawMetric(QPainter &p, const QPair<QString, QString> &label, QCol
p.drawText(rect.adjusted(22, 0, 0, 0), Qt::AlignCenter, label.first + "\n" + label.second);
}
Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(false), settings_pressed(false) {
Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(false), settings_pressed(false), mic_indicator_pressed(false) {
home_img = loadPixmap("../assets/images/button_home.png", home_btn.size());
flag_img = loadPixmap("../assets/images/button_flag.png", home_btn.size());
settings_img = loadPixmap("../assets/images/button_settings.png", settings_btn.size(), Qt::IgnoreAspectRatio);
mic_img = loadPixmap("../assets/icons/microphone.png", QSize(30, 30));
connect(this, &Sidebar::valueChanged, [=] { update(); });
@@ -47,12 +48,15 @@ void Sidebar::mousePressEvent(QMouseEvent *event) {
} else if (settings_btn.contains(event->pos())) {
settings_pressed = true;
update();
} else if (recording_audio && mic_indicator_btn.contains(event->pos())) {
mic_indicator_pressed = true;
update();
}
}
void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
if (flag_pressed || settings_pressed) {
flag_pressed = settings_pressed = false;
if (flag_pressed || settings_pressed || mic_indicator_pressed) {
flag_pressed = settings_pressed = mic_indicator_pressed = false;
update();
}
if (onroad && home_btn.contains(event->pos())) {
@@ -61,6 +65,8 @@ void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
pm->send("userFlag", msg);
} else if (settings_btn.contains(event->pos())) {
emit openSettings();
} else if (recording_audio && mic_indicator_btn.contains(event->pos())) {
emit openSettings(2, "RecordAudio");
}
}
@@ -106,6 +112,8 @@ void Sidebar::updateState(const UIState &s) {
pandaStatus = {{tr("NO"), tr("PANDA")}, danger_color};
}
setProperty("pandaStatus", QVariant::fromValue(pandaStatus));
setProperty("recordingAudio", s.scene.recording_audio);
}
void Sidebar::paintEvent(QPaintEvent *event) {
@@ -124,6 +132,14 @@ void Sidebar::drawSidebar(QPainter &p) {
p.drawPixmap(settings_btn.x(), settings_btn.y(), settings_img);
p.setOpacity(onroad && flag_pressed ? 0.65 : 1.0);
p.drawPixmap(home_btn.x(), home_btn.y(), onroad ? flag_img : home_img);
if (recording_audio) {
p.setBrush(danger_color);
p.setOpacity(mic_indicator_pressed ? 0.65 : 1.0);
p.drawRoundedRect(mic_indicator_btn, mic_indicator_btn.height() / 2, mic_indicator_btn.height() / 2);
int icon_x = mic_indicator_btn.x() + (mic_indicator_btn.width() - mic_img.width()) / 2;
int icon_y = mic_indicator_btn.y() + (mic_indicator_btn.height() - mic_img.height()) / 2;
p.drawPixmap(icon_x, icon_y, mic_img);
}
p.setOpacity(1.0);
// network

View File

@@ -23,6 +23,7 @@ class Sidebar : public QFrame {
Q_PROPERTY(ItemStatus tempStatus MEMBER temp_status NOTIFY valueChanged);
Q_PROPERTY(QString netType MEMBER net_type NOTIFY valueChanged);
Q_PROPERTY(int netStrength MEMBER net_strength NOTIFY valueChanged);
Q_PROPERTY(bool recordingAudio MEMBER recording_audio NOTIFY valueChanged);
public:
explicit Sidebar(QWidget* parent = 0);
@@ -42,8 +43,8 @@ protected:
void drawMetric(QPainter &p, const QPair<QString, QString> &label, QColor c, int y);
virtual void drawSidebar(QPainter &p);
QPixmap home_img, flag_img, settings_img;
bool onroad, flag_pressed, settings_pressed;
QPixmap home_img, flag_img, settings_img, mic_img;
bool onroad, recording_audio, flag_pressed, settings_pressed, mic_indicator_pressed;
const QMap<cereal::DeviceState::NetworkType, QString> network_type = {
{cereal::DeviceState::NetworkType::NONE, tr("--")},
{cereal::DeviceState::NetworkType::WIFI, tr("Wi-Fi")},
@@ -56,6 +57,7 @@ protected:
const QRect home_btn = QRect(60, 860, 180, 180);
const QRect settings_btn = QRect(50, 35, 200, 117);
const QRect mic_indicator_btn = QRect(158, 252, 75, 40);
const QColor good_color = QColor(255, 255, 255);
const QColor warning_color = QColor(218, 202, 37);
const QColor danger_color = QColor(201, 34, 49);

View File

@@ -139,7 +139,7 @@ class Soundd(QuietMode):
# sounddevice must be imported after forking processes
import sounddevice as sd
sm = messaging.SubMaster(['selfdriveState', 'microphone'])
sm = messaging.SubMaster(['selfdriveState', 'soundPressure'])
with self.get_stream(sd) as stream:
rk = Ratekeeper(20)
@@ -150,8 +150,8 @@ class Soundd(QuietMode):
self.load_param()
if sm.updated['microphone'] and self.current_alert == AudibleAlert.none: # only update volume filter when not playing alert
self.spl_filter_weighted.update(sm["microphone"].soundPressureWeightedDb)
if sm.updated['soundPressure'] and self.current_alert == AudibleAlert.none: # only update volume filter when not playing alert
self.spl_filter_weighted.update(sm["soundPressure"].soundPressureWeightedDb)
self.current_volume = self.calculate_volume(float(self.spl_filter_weighted.x))
self.get_audible_alert(sm)

View File

@@ -61,6 +61,9 @@ void update_state(UIState *s) {
scene.light_sensor = -1;
}
scene.started = sm["deviceState"].getDeviceState().getStarted() && scene.ignition;
auto params = Params();
scene.recording_audio = params.getBool("RecordAudio") && scene.started;
}
void ui_update_params(UIState *s) {

View File

@@ -62,7 +62,7 @@ typedef struct UIScene {
cereal::LongitudinalPersonality personality;
float light_sensor = -1;
bool started, ignition, is_metric;
bool started, ignition, is_metric, recording_audio;
uint64_t started_frame;
} UIScene;

View File

@@ -1,12 +1,20 @@
import pyray as rl
import numpy as np
import time
import threading
from collections.abc import Callable
from enum import Enum
from cereal import messaging, log
from openpilot.common.filter_simple import FirstOrderFilter
from openpilot.common.params import Params, UnknownKeyName
from openpilot.common.swaglog import cloudlog
from openpilot.selfdrive.ui.lib.prime_state import PrimeState
from openpilot.system.ui.lib.application import DEFAULT_FPS
from openpilot.system.hardware import HARDWARE
from openpilot.system.ui.lib.application import gui_app
UI_BORDER_SIZE = 30
BACKLIGHT_OFFROAD = 50
class UIStatus(Enum):
@@ -139,10 +147,15 @@ class UIState:
class Device:
def __init__(self):
self._ignition = False
self._interaction_time: float = 0.0
self._interaction_time: float = -1
self._interactive_timeout_callbacks: list[Callable] = []
self._prev_timed_out = False
self.reset_interactive_timeout()
self._awake = False
self._offroad_brightness: int = BACKLIGHT_OFFROAD
self._last_brightness: int = 0
self._brightness_filter = FirstOrderFilter(BACKLIGHT_OFFROAD, 10.00, 1 / DEFAULT_FPS)
self._brightness_thread: threading.Thread | None = None
def reset_interactive_timeout(self, timeout: int = -1) -> None:
if timeout == -1:
@@ -153,18 +166,64 @@ class Device:
self._interactive_timeout_callbacks.append(callback)
def update(self):
# do initial reset
if self._interaction_time <= 0:
self.reset_interactive_timeout()
self._update_brightness()
self._update_wakefulness()
def set_offroad_brightness(self, brightness: int):
# TODO: not yet used, should be used in prime widget for QR code, etc.
self._offroad_brightness = min(max(brightness, 0), 100)
def _update_brightness(self):
clipped_brightness = self._offroad_brightness
if ui_state.started and ui_state.light_sensor >= 0:
clipped_brightness = ui_state.light_sensor
# CIE 1931 - https://www.photonstophotos.net/GeneralTopics/Exposure/Psychometric_Lightness_and_Gamma.htm
if clipped_brightness <= 8:
clipped_brightness = clipped_brightness / 903.3
else:
clipped_brightness = ((clipped_brightness + 16.0) / 116.0) ** 3.0
clipped_brightness = float(np.clip(100 * clipped_brightness, 10, 100))
brightness = round(self._brightness_filter.update(clipped_brightness))
if not self._awake:
brightness = 0
if brightness != self._last_brightness:
if self._brightness_thread is None or not self._brightness_thread.is_alive():
cloudlog.debug(f"setting display brightness {brightness}")
self._brightness_thread = threading.Thread(target=HARDWARE.set_screen_brightness, args=(brightness,))
self._brightness_thread.start()
self._last_brightness = brightness
def _update_wakefulness(self):
# Handle interactive timeout
ignition_just_turned_off = not ui_state.ignition and self._ignition
self._ignition = ui_state.ignition
interaction_timeout = time.monotonic() > self._interaction_time
if ignition_just_turned_off or rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT):
if ignition_just_turned_off or any(ev.left_down for ev in gui_app.mouse_events):
self.reset_interactive_timeout()
elif interaction_timeout and not self._prev_timed_out:
interaction_timeout = time.monotonic() > self._interaction_time
if interaction_timeout and not self._prev_timed_out:
for callback in self._interactive_timeout_callbacks:
callback()
self._prev_timed_out = interaction_timeout
self._set_awake(ui_state.ignition or not interaction_timeout)
def _set_awake(self, on: bool):
if on != self._awake:
self._awake = on
cloudlog.debug(f"setting display power {int(on)}")
HARDWARE.set_display_power(on)
# Global instance
ui_state = UIState()

View File

@@ -1 +1 @@
71979b29c4bab3007de1a4265442d79f44c0eaef066af66086dddfc432709b94
61bb1f3077ef8219a44c8c627d5345d1a96e7f859b850bed6215eec91090becb

View File

@@ -48,7 +48,8 @@ class LatControlTorqueExtBase(LagdToggle):
LagdToggle.__init__(self)
self.model_v2 = None
self.model_valid = False
self.use_steering_angle = lac_torque.use_steering_angle
self.torque_params = lac_torque.torque_params
self.use_steering_angle = lac_torque.torque_params.useSteeringAngle
self.actual_lateral_jerk: float = 0.0
self.lateral_jerk_setpoint: float = 0.0
@@ -56,7 +57,6 @@ class LatControlTorqueExtBase(LagdToggle):
self.lookahead_lateral_jerk: float = 0.0
self.torque_from_lateral_accel = lac_torque.torque_from_lateral_accel
self.torque_params = lac_torque.torque_params
self._ff = 0.0
self._pid_log = None

View File

@@ -23,6 +23,7 @@ from typing import cast
from collections.abc import Callable
import requests
from requests.adapters import HTTPAdapter, DEFAULT_POOLBLOCK
from jsonrpc import JSONRPCResponseManager, dispatcher
from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException,
create_connection)
@@ -56,6 +57,11 @@ WS_FRAME_SIZE = 4096
DEVICE_STATE_UPDATE_INTERVAL = 1.0 # in seconds
DEFAULT_UPLOAD_PRIORITY = 99 # higher number = lower priority
# https://bytesolutions.com/dscp-tos-cos-precedence-conversion-chart,
# https://en.wikipedia.org/wiki/Differentiated_services
UPLOAD_TOS = 0x20 # CS1, low priority background traffic
SSH_TOS = 0x90 # AF42, DSCP of 36/HDD_LINUX_AC_VI with the minimum delay flag
NetworkType = log.DeviceState.NetworkType
UploadFileDict = dict[str, str | int | float | bool]
@@ -64,6 +70,17 @@ UploadItemDict = dict[str, str | bool | int | float | dict[str, str]]
UploadFilesToUrlResponse = dict[str, int | list[UploadItemDict] | list[str]]
class UploadTOSAdapter(HTTPAdapter):
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
pool_kwargs["socket_options"] = [(socket.IPPROTO_IP, socket.IP_TOS, UPLOAD_TOS)]
super().init_poolmanager(connections, maxsize, block, **pool_kwargs)
UPLOAD_SESS = requests.Session()
UPLOAD_SESS.mount("http://", UploadTOSAdapter())
UPLOAD_SESS.mount("https://", UploadTOSAdapter())
@dataclass
class UploadFile:
fn: str
@@ -311,10 +328,10 @@ def _do_upload(upload_item: UploadItem, callback: Callable = None) -> requests.R
stream = None
try:
stream, content_length = get_upload_stream(path, compress)
response = requests.put(upload_item.url,
data=CallbackReader(stream, callback, content_length) if callback else stream,
headers={**upload_item.headers, 'Content-Length': str(content_length)},
timeout=30)
response = UPLOAD_SESS.put(upload_item.url,
data=CallbackReader(stream, callback, content_length) if callback else stream,
headers={**upload_item.headers, 'Content-Length': str(content_length)},
timeout=30)
return response
finally:
if stream:
@@ -501,8 +518,7 @@ def start_local_proxy_shim(global_end_event: threading.Event, local_port: int, w
raise Exception("Requested local port not whitelisted")
# Set TOS to keep connection responsive while under load.
# DSCP of 36/HDD_LINUX_AC_VI with the minimum delay flag
ws.sock.setsockopt(socket.IPPROTO_IP, socket.IP_TOS, 0x90)
ws.sock.setsockopt(socket.IPPROTO_IP, socket.IP_TOS, SSH_TOS)
ssock, csock = socket.socketpair()
local_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

View File

@@ -19,7 +19,7 @@ from cereal import messaging
from openpilot.common.params import Params
from openpilot.common.timeout import Timeout
from openpilot.system.athena import athenad
from openpilot.system.athena.athenad import MAX_RETRY_COUNT, dispatcher
from openpilot.system.athena.athenad import MAX_RETRY_COUNT, UPLOAD_SESS, dispatcher
from openpilot.system.athena.tests.helpers import HTTPRequestHandler, MockWebsocket, MockApi, EchoSocket
from openpilot.selfdrive.test.helpers import http_server_context
from openpilot.system.hardware.hw import Paths
@@ -29,7 +29,7 @@ def seed_athena_server(host, port):
with Timeout(2, 'HTTP Server seeding failed'):
while True:
try:
requests.put(f'http://{host}:{port}/qlog.zst', data='', timeout=10)
UPLOAD_SESS.put(f'http://{host}:{port}/qlog.zst', data='', timeout=10)
break
except requests.exceptions.ConnectionError:
time.sleep(0.1)
@@ -239,7 +239,7 @@ class TestAthenadMethods:
@pytest.mark.parametrize("status,retry", [(500,True), (412,False)])
@with_upload_handler
def test_upload_handler_retry(self, mocker, host, status, retry):
mock_put = mocker.patch('requests.put')
mock_put = mocker.patch('openpilot.system.athena.athenad.UPLOAD_SESS.put')
mock_put.return_value.__enter__.return_value.status_code = status
fn = self._create_file('qlog.zst')
item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)

View File

@@ -415,8 +415,8 @@ class Tici(HardwareBase):
# *** GPU config ***
# https://github.com/commaai/agnos-kernel-sdm845/blob/master/arch/arm64/boot/dts/qcom/sdm845-gpu.dtsi#L216
sudo_write("0", "/sys/class/kgsl/kgsl-3d0/min_pwrlevel")
sudo_write("0", "/sys/class/kgsl/kgsl-3d0/max_pwrlevel")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/min_pwrlevel")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/max_pwrlevel")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_bus_on")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_clk_on")
sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_rail_on")

View File

@@ -62,6 +62,7 @@ struct RemoteEncoder {
bool recording = false;
bool marked_ready_to_rotate = false;
bool seen_first_packet = false;
bool audio_initialized = false;
};
size_t write_encode_data(LoggerdState *s, cereal::Event::Reader event, RemoteEncoder &re, const EncoderInfo &encoder_info) {
@@ -78,12 +79,7 @@ size_t write_encode_data(LoggerdState *s, cereal::Event::Reader event, RemoteEnc
LOGW("%s: dropped %d non iframe packets before init", encoder_info.publish_name, re.dropped_frames);
re.dropped_frames = 0;
}
// if we aren't actually recording, don't create the writer
if (encoder_info.record) {
assert(encoder_info.filename != NULL);
re.writer.reset(new VideoWriter(s->logger.segmentPath().c_str(),
encoder_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C,
edata.getWidth(), edata.getHeight(), encoder_info.fps, idx.getType()));
// write the header
auto header = edata.getHeader();
re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof() / 1000, true, false);
@@ -138,12 +134,19 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
// if this is a new segment, we close any possible old segments, move to the new, and process any queued packets
if (re.current_segment != s->logger.segment()) {
if (re.recording) {
re.writer.reset();
// if we aren't actually recording, don't create the writer
if (encoder_info.record) {
assert(encoder_info.filename != NULL);
re.writer.reset(new VideoWriter(s->logger.segmentPath().c_str(),
encoder_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C,
edata.getWidth(), edata.getHeight(), encoder_info.fps, idx.getType()));
re.recording = false;
re.audio_initialized = false;
}
re.current_segment = s->logger.segment();
re.marked_ready_to_rotate = false;
}
if (re.audio_initialized || !encoder_info.include_audio) {
// we are in this segment now, process any queued messages before this one
if (!re.q.empty()) {
for (auto qmsg : re.q) {
@@ -153,9 +156,14 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
}
re.q.clear();
}
bytes_count += write_encode_data(s, event, re, encoder_info);
delete msg;
} else if (re.q.size() > MAIN_FPS*10) {
LOGE_100("%s: dropping frame waiting for audio initialization, queue is too large", name.c_str());
delete msg;
} else {
re.q.push_back(msg); // queue up all the new segment messages, they go in after audio is initialized
}
bytes_count += write_encode_data(s, event, re, encoder_info);
delete msg;
} else if (offset_segment_num > s->logger.segment()) {
// encoderd packet has a newer segment, this means encoderd has rolled over
if (!re.marked_ready_to_rotate) {
@@ -214,7 +222,7 @@ void loggerd_thread() {
typedef struct ServiceState {
std::string name;
int counter, freq;
bool encoder, user_flag;
bool encoder, user_flag, record_audio;
} ServiceState;
std::unordered_map<SubSocket*, ServiceState> service_state;
std::unordered_map<SubSocket*, struct RemoteEncoder> remote_encoders;
@@ -226,19 +234,22 @@ void loggerd_thread() {
for (const auto& [_, it] : services) {
const bool encoder = util::ends_with(it.name, "EncodeData");
const bool livestream_encoder = util::starts_with(it.name, "livestream");
if (!it.should_log && (!encoder || livestream_encoder)) continue;
LOGD("logging %s", it.name.c_str());
const bool record_audio = (it.name == "rawAudioData") && Params().getBool("RecordAudio");
if (it.should_log || (encoder && !livestream_encoder) || record_audio) {
LOGD("logging %s", it.name.c_str());
SubSocket * sock = SubSocket::create(ctx.get(), it.name);
assert(sock != NULL);
poller->registerSocket(sock);
service_state[sock] = {
.name = it.name,
.counter = 0,
.freq = it.decimation,
.encoder = encoder,
.user_flag = it.name == "userFlag",
};
SubSocket * sock = SubSocket::create(ctx.get(), it.name);
assert(sock != NULL);
poller->registerSocket(sock);
service_state[sock] = {
.name = it.name,
.counter = 0,
.freq = it.decimation,
.encoder = encoder,
.user_flag = it.name == "userFlag",
.record_audio = record_audio,
};
}
}
LoggerdState s;
@@ -247,6 +258,7 @@ void loggerd_thread() {
Params().put("CurrentRoute", s.logger.routeName());
std::map<std::string, EncoderInfo> encoder_infos_dict;
std::vector<RemoteEncoder*> encoders_with_audio;
for (const auto &cam : cameras_logged) {
for (const auto &encoder_info : cam.encoder_infos) {
encoder_infos_dict[encoder_info.publish_name] = encoder_info;
@@ -254,6 +266,13 @@ void loggerd_thread() {
}
}
for (auto &[sock, service] : service_state) {
auto it = encoder_infos_dict.find(service.name);
if (it != encoder_infos_dict.end() && it->second.include_audio) {
encoders_with_audio.push_back(&remote_encoders[sock]);
}
}
uint64_t msg_count = 0, bytes_count = 0;
double start_ts = millis_since_boot();
while (!do_exit) {
@@ -271,6 +290,20 @@ void loggerd_thread() {
Message *msg = nullptr;
while (!do_exit && (msg = sock->receive(true))) {
const bool in_qlog = service.freq != -1 && (service.counter++ % service.freq == 0);
if (service.record_audio) {
capnp::FlatArrayMessageReader cmsg(kj::ArrayPtr<capnp::word>((capnp::word *)msg->getData(), msg->getSize() / sizeof(capnp::word)));
auto event = cmsg.getRoot<cereal::Event>();
auto audio_data = event.getRawAudioData().getData();
auto sample_rate = event.getRawAudioData().getSampleRate();
for (auto* encoder : encoders_with_audio) {
if (encoder && encoder->writer) {
encoder->writer->write_audio((uint8_t*)audio_data.begin(), audio_data.size(), event.getLogMonoTime() / 1000, sample_rate);
encoder->audio_initialized = true;
}
}
}
if (service.encoder) {
s.last_camera_seen_tms = millis_since_boot();
bytes_count += handle_encoder_msg(&s, msg, service.name, remote_encoders[sock], encoder_infos_dict[service.name]);

View File

@@ -35,6 +35,7 @@ public:
const char *thumbnail_name = NULL;
const char *filename = NULL;
bool record = true;
bool include_audio = false;
int frame_width = -1;
int frame_height = -1;
int fps = MAIN_FPS;
@@ -106,6 +107,7 @@ const EncoderInfo qcam_encoder_info = {
.encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
.frame_width = 526,
.frame_height = 330,
.include_audio = Params().getBool("RecordAudio"),
INIT_ENCODE_FUNCTIONS(QRoadEncode),
};

View File

@@ -97,6 +97,50 @@ class TestLoggerd:
return sent_msgs
def _publish_camera_and_audio_messages(self, num_segs=1, segment_length=5):
d = DEVICE_CAMERAS[("tici", "ar0231")]
streams = [
(VisionStreamType.VISION_STREAM_ROAD, (d.fcam.width, d.fcam.height, 2048 * 2346, 2048, 2048 * 1216), "roadCameraState"),
(VisionStreamType.VISION_STREAM_DRIVER, (d.dcam.width, d.dcam.height, 2048 * 2346, 2048, 2048 * 1216), "driverCameraState"),
(VisionStreamType.VISION_STREAM_WIDE_ROAD, (d.ecam.width, d.ecam.height, 2048 * 2346, 2048, 2048 * 1216), "wideRoadCameraState"),
]
pm = messaging.PubMaster([s for _, _, s in streams] + ["rawAudioData"])
vipc_server = VisionIpcServer("camerad")
for stream_type, frame_spec, _ in streams:
vipc_server.create_buffers_with_sizes(stream_type, 40, *(frame_spec))
vipc_server.start_listener()
os.environ["LOGGERD_TEST"] = "1"
os.environ["LOGGERD_SEGMENT_LENGTH"] = str(segment_length)
managed_processes["loggerd"].start()
managed_processes["encoderd"].start()
assert pm.wait_for_readers_to_update("roadCameraState", timeout=5)
fps = 20
for n in range(1, int(num_segs * segment_length * fps) + 1):
# send video
for stream_type, frame_spec, state in streams:
dat = np.empty(frame_spec[2], dtype=np.uint8)
vipc_server.send(stream_type, dat[:].flatten().tobytes(), n, n / fps, n / fps)
camera_state = messaging.new_message(state)
frame = getattr(camera_state, state)
frame.frameId = n
pm.send(state, camera_state)
# send audio
msg = messaging.new_message('rawAudioData')
msg.rawAudioData.data = bytes(800 * 2) # 800 samples of int16
msg.rawAudioData.sampleRate = 16000
pm.send('rawAudioData', msg)
for _, _, state in streams:
assert pm.wait_for_readers_to_update(state, timeout=5, dt=0.001)
managed_processes["loggerd"].stop()
managed_processes["encoderd"].stop()
def test_init_data_values(self):
os.environ["CLEAN"] = random.choice(["0", "1"])
@@ -136,53 +180,23 @@ class TestLoggerd:
assert getattr(initData, initData_key) == v
assert logged_params[param_key].decode() == v
@pytest.mark.skip("FIXME: encoderd sometimes crashes in CI when running with pytest-xdist")
@pytest.mark.xdist_group("camera_encoder_tests") # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
def test_rotation(self):
os.environ["LOGGERD_TEST"] = "1"
Params().put("RecordFront", "1")
d = DEVICE_CAMERAS[("tici", "ar0231")]
expected_files = {"rlog.zst", "qlog.zst", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"}
streams = [(VisionStreamType.VISION_STREAM_ROAD, (d.fcam.width, d.fcam.height, 2048*2346, 2048, 2048*1216), "roadCameraState"),
(VisionStreamType.VISION_STREAM_DRIVER, (d.dcam.width, d.dcam.height, 2048*2346, 2048, 2048*1216), "driverCameraState"),
(VisionStreamType.VISION_STREAM_WIDE_ROAD, (d.ecam.width, d.ecam.height, 2048*2346, 2048, 2048*1216), "wideRoadCameraState")]
pm = messaging.PubMaster(["roadCameraState", "driverCameraState", "wideRoadCameraState"])
vipc_server = VisionIpcServer("camerad")
for stream_type, frame_spec, _ in streams:
vipc_server.create_buffers_with_sizes(stream_type, 40, *(frame_spec))
vipc_server.start_listener()
num_segs = random.randint(2, 3)
length = random.randint(4, 5) # H264 encoder uses 40 lookahead frames and does B-frame reordering, so minimum 3 seconds before qcam output
num_segs = random.randint(2, 5)
length = random.randint(1, 3)
os.environ["LOGGERD_SEGMENT_LENGTH"] = str(length)
managed_processes["loggerd"].start()
managed_processes["encoderd"].start()
assert pm.wait_for_readers_to_update("roadCameraState", timeout=5)
fps = 20.0
for n in range(1, int(num_segs*length*fps)+1):
for stream_type, frame_spec, state in streams:
dat = np.empty(frame_spec[2], dtype=np.uint8)
vipc_server.send(stream_type, dat[:].flatten().tobytes(), n, n/fps, n/fps)
camera_state = messaging.new_message(state)
frame = getattr(camera_state, state)
frame.frameId = n
pm.send(state, camera_state)
for _, _, state in streams:
assert pm.wait_for_readers_to_update(state, timeout=5, dt=0.001)
managed_processes["loggerd"].stop()
managed_processes["encoderd"].stop()
self._publish_camera_and_audio_messages(num_segs=num_segs, segment_length=length)
route_path = str(self._get_latest_log_dir()).rsplit("--", 1)[0]
for n in range(num_segs):
p = Path(f"{route_path}--{n}")
logged = {f.name for f in p.iterdir() if f.is_file()}
diff = logged ^ expected_files
assert len(diff) == 0, f"didn't get all expected files. run={_} seg={n} {route_path=}, {diff=}\n{logged=} {expected_files=}"
assert len(diff) == 0, f"didn't get all expected files. seg={n} {route_path=}, {diff=}\n{logged=} {expected_files=}"
def test_bootlog(self):
# generate bootlog with fake launch log
@@ -281,3 +295,30 @@ class TestLoggerd:
segment_dir = self._get_latest_log_dir()
assert getxattr(segment_dir, PRESERVE_ATTR_NAME) is None
@pytest.mark.xdist_group("camera_encoder_tests") # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
@pytest.mark.parametrize("record_front", [True, False])
def test_record_front(self, record_front):
params = Params()
params.put_bool("RecordFront", record_front)
self._publish_camera_and_audio_messages()
dcamera_hevc_exists = os.path.exists(os.path.join(self._get_latest_log_dir(), 'dcamera.hevc'))
assert dcamera_hevc_exists == record_front
@pytest.mark.xdist_group("camera_encoder_tests") # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
@pytest.mark.parametrize("record_audio", [True, False])
def test_record_audio(self, record_audio):
params = Params()
params.put_bool("RecordAudio", record_audio)
self._publish_camera_and_audio_messages()
qcamera_ts_path = os.path.join(self._get_latest_log_dir(), 'qcamera.ts')
ffprobe_cmd = f"ffprobe -i {qcamera_ts_path} -show_streams -select_streams a -loglevel error"
has_audio_stream = subprocess.run(ffprobe_cmd, shell=True, capture_output=True).stdout.strip() != b''
assert has_audio_stream == record_audio
raw_audio_in_rlog = any(m.which() == 'rawAudioData' for m in LogReader(os.path.join(self._get_latest_log_dir(), 'rlog.zst')))
assert raw_audio_in_rlog == record_audio

View File

@@ -50,6 +50,45 @@ VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing,
}
}
void VideoWriter::initialize_audio(int sample_rate) {
assert(this->ofmt_ctx->oformat->audio_codec != AV_CODEC_ID_NONE); // check output format supports audio streams
const AVCodec *audio_avcodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
assert(audio_avcodec);
this->audio_codec_ctx = avcodec_alloc_context3(audio_avcodec);
assert(this->audio_codec_ctx);
this->audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;
this->audio_codec_ctx->sample_rate = sample_rate;
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 28, 100) // FFmpeg 5.1+
av_channel_layout_default(&this->audio_codec_ctx->ch_layout, 1);
#else
this->audio_codec_ctx->channel_layout = AV_CH_LAYOUT_MONO;
#endif
this->audio_codec_ctx->bit_rate = 32000;
this->audio_codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
this->audio_codec_ctx->time_base = (AVRational){1, audio_codec_ctx->sample_rate};
int err = avcodec_open2(this->audio_codec_ctx, audio_avcodec, NULL);
assert(err >= 0);
av_log_set_level(AV_LOG_WARNING); // hide "QAvg" info msgs at the end of every segment
this->audio_stream = avformat_new_stream(this->ofmt_ctx, NULL);
assert(this->audio_stream);
err = avcodec_parameters_from_context(this->audio_stream->codecpar, this->audio_codec_ctx);
assert(err >= 0);
this->audio_frame = av_frame_alloc();
assert(this->audio_frame);
this->audio_frame->format = this->audio_codec_ctx->sample_fmt;
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 28, 100) // FFmpeg 5.1+
av_channel_layout_copy(&this->audio_frame->ch_layout, &this->audio_codec_ctx->ch_layout);
#else
this->audio_frame->channel_layout = this->audio_codec_ctx->channel_layout;
#endif
this->audio_frame->sample_rate = this->audio_codec_ctx->sample_rate;
this->audio_frame->nb_samples = this->audio_codec_ctx->frame_size;
err = av_frame_get_buffer(this->audio_frame, 0);
assert(err >= 0);
}
void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe) {
if (of && data) {
size_t written = util::safe_fwrite(data, 1, len, of);
@@ -67,8 +106,10 @@ void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecc
}
int err = avcodec_parameters_from_context(out_stream->codecpar, codec_ctx);
assert(err >= 0);
// if there is an audio stream, it must be initialized before this point
err = avformat_write_header(ofmt_ctx, NULL);
assert(err >= 0);
header_written = true;
} else {
// input timestamps are in microseconds
AVRational in_timebase = {1, 1000000};
@@ -77,6 +118,7 @@ void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecc
av_init_packet(&pkt);
pkt.data = data;
pkt.size = len;
pkt.stream_index = this->out_stream->index;
enum AVRounding rnd = static_cast<enum AVRounding>(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
pkt.pts = pkt.dts = av_rescale_q_rnd(timestamp, in_timebase, ofmt_ctx->streams[0]->time_base, rnd);
@@ -95,11 +137,80 @@ void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecc
}
}
void VideoWriter::write_audio(uint8_t *data, int len, long long timestamp, int sample_rate) {
if (!remuxing) return;
if (!audio_initialized) {
initialize_audio(sample_rate);
audio_initialized = true;
}
if (!audio_codec_ctx) return;
// sync logMonoTime of first audio packet with the timestampEof of first video packet
if (audio_pts == 0) {
audio_pts = (timestamp * audio_codec_ctx->sample_rate) / 1000000ULL;
}
// convert s16le samples to fltp and add to buffer
const int16_t *raw_samples = reinterpret_cast<const int16_t*>(data);
int sample_count = len / sizeof(int16_t);
constexpr float normalizer = 1.0f / 32768.0f;
const size_t max_buffer_size = sample_rate * 10; // 10 seconds
if (audio_buffer.size() + sample_count > max_buffer_size) {
size_t samples_to_drop = (audio_buffer.size() + sample_count) - max_buffer_size;
LOGE("Audio buffer overflow, dropping %zu oldest samples", samples_to_drop);
audio_buffer.erase(audio_buffer.begin(), audio_buffer.begin() + samples_to_drop);
audio_pts += samples_to_drop;
}
// Add new samples to the buffer
const size_t original_size = audio_buffer.size();
audio_buffer.resize(original_size + sample_count);
std::transform(raw_samples, raw_samples + sample_count, audio_buffer.begin() + original_size,
[](int16_t sample) { return sample * normalizer; });
if (!header_written) return; // header not written yet, process audio frame after header is written
while (audio_buffer.size() >= audio_codec_ctx->frame_size) {
audio_frame->pts = audio_pts;
float *f_samples = reinterpret_cast<float*>(audio_frame->data[0]);
std::copy(audio_buffer.begin(), audio_buffer.begin() + audio_codec_ctx->frame_size, f_samples);
audio_buffer.erase(audio_buffer.begin(), audio_buffer.begin() + audio_codec_ctx->frame_size);
encode_and_write_audio_frame(audio_frame);
}
}
void VideoWriter::encode_and_write_audio_frame(AVFrame* frame) {
if (!remuxing || !audio_codec_ctx) return;
int send_result = avcodec_send_frame(audio_codec_ctx, frame); // encode frame
if (send_result >= 0) {
AVPacket *pkt = av_packet_alloc();
while (avcodec_receive_packet(audio_codec_ctx, pkt) == 0) {
av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_stream->time_base);
pkt->stream_index = audio_stream->index;
int err = av_interleaved_write_frame(ofmt_ctx, pkt); // write encoded frame
if (err < 0) {
LOGW("AUDIO: Write frame failed - error: %d", err);
}
av_packet_unref(pkt);
}
av_packet_free(&pkt);
} else {
LOGW("AUDIO: Failed to send audio frame to encoder: %d", send_result);
}
audio_pts += audio_codec_ctx->frame_size;
}
VideoWriter::~VideoWriter() {
if (this->remuxing) {
if (this->audio_codec_ctx) {
encode_and_write_audio_frame(NULL); // flush encoder
avcodec_free_context(&this->audio_codec_ctx);
}
int err = av_write_trailer(this->ofmt_ctx);
if (err != 0) LOGE("av_write_trailer failed %d", err);
avcodec_free_context(&this->codec_ctx);
if (this->audio_frame) av_frame_free(&this->audio_frame);
err = avio_closep(&this->ofmt_ctx->pb);
if (err != 0) LOGE("avio_closep failed %d", err);
avformat_free_context(this->ofmt_ctx);

View File

@@ -1,6 +1,7 @@
#pragma once
#include <string>
#include <deque>
extern "C" {
#include <libavformat/avformat.h>
@@ -13,13 +14,28 @@ class VideoWriter {
public:
VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec);
void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe);
void write_audio(uint8_t *data, int len, long long timestamp, int sample_rate);
~VideoWriter();
private:
void initialize_audio(int sample_rate);
void encode_and_write_audio_frame(AVFrame* frame);
std::string vid_path, lock_path;
FILE *of = nullptr;
AVCodecContext *codec_ctx;
AVFormatContext *ofmt_ctx;
AVStream *out_stream;
bool audio_initialized = false;
bool header_written = false;
AVStream *audio_stream = nullptr;
AVCodecContext *audio_codec_ctx = nullptr;
AVFrame *audio_frame = nullptr;
uint64_t audio_pts = 0;
std::deque<float> audio_buffer;
bool remuxing;
};

View File

@@ -9,10 +9,10 @@ from openpilot.common.retry import retry
from openpilot.common.swaglog import cloudlog
RATE = 10
FFT_SAMPLES = 4096
FFT_SAMPLES = 1600 # 100ms
REFERENCE_SPL = 2e-5 # newtons/m^2
SAMPLE_RATE = 44100
SAMPLE_BUFFER = 4096 # approx 100ms
SAMPLE_RATE = 16000
SAMPLE_BUFFER = 800 # 50ms
@cache
@@ -45,7 +45,7 @@ def apply_a_weighting(measurements: np.ndarray) -> np.ndarray:
class Mic:
def __init__(self):
self.rk = Ratekeeper(RATE)
self.pm = messaging.PubMaster(['microphone'])
self.pm = messaging.PubMaster(['soundPressure', 'rawAudioData'])
self.measurements = np.empty(0)
@@ -61,12 +61,12 @@ class Mic:
sound_pressure_weighted = self.sound_pressure_weighted
sound_pressure_level_weighted = self.sound_pressure_level_weighted
msg = messaging.new_message('microphone', valid=True)
msg.microphone.soundPressure = float(sound_pressure)
msg.microphone.soundPressureWeighted = float(sound_pressure_weighted)
msg.microphone.soundPressureWeightedDb = float(sound_pressure_level_weighted)
msg = messaging.new_message('soundPressure', valid=True)
msg.soundPressure.soundPressure = float(sound_pressure)
msg.soundPressure.soundPressureWeighted = float(sound_pressure_weighted)
msg.soundPressure.soundPressureWeightedDb = float(sound_pressure_level_weighted)
self.pm.send('microphone', msg)
self.pm.send('soundPressure', msg)
self.rk.keep_time()
def callback(self, indata, frames, time, status):
@@ -76,6 +76,12 @@ class Mic:
Logged A-weighted equivalents are rough approximations of the human-perceived loudness.
"""
msg = messaging.new_message('rawAudioData', valid=True)
audio_data_int_16 = (indata[:, 0] * 32767).astype(np.int16)
msg.rawAudioData.data = audio_data_int_16.tobytes()
msg.rawAudioData.sampleRate = SAMPLE_RATE
self.pm.send('rawAudioData', msg)
with self.lock:
self.measurements = np.concatenate((self.measurements, indata[:, 0]))

View File

@@ -7,10 +7,10 @@ from openpilot.system.sensord.sensors.i2c_sensor import Sensor
class LSM6DS3_Temp(Sensor):
@property
def device_address(self) -> int:
return 0x6A # Default I2C address for LSM6DS3
return 0x6A
def _read_temperature(self) -> float:
scale = 16.0 if log.SensorEventData.SensorSource.lsm6ds3 else 256.0
scale = 16.0 if self.source == log.SensorEventData.SensorSource.lsm6ds3 else 256.0
data = self.read(0x20, 2)
return 25 + (self.parse_16bit(data[0], data[1]) / scale)

View File

@@ -136,6 +136,17 @@ class TTYPigeon:
return True
return False
def save_almanac(pigeon: TTYPigeon) -> None:
# store almanac in flash
pigeon.send(b"\xB5\x62\x09\x14\x04\x00\x00\x00\x00\x00\x21\xEC")
try:
if pigeon.wait_for_ack(ack=UBLOX_SOS_ACK, nack=UBLOX_SOS_NACK):
cloudlog.info("Done storing almanac")
else:
cloudlog.error("Error storing almanac")
except TimeoutError:
pass
def init_baudrate(pigeon: TTYPigeon):
# ublox default setting on startup is 9600 baudrate
pigeon.set_baud(9600)
@@ -245,16 +256,6 @@ def deinitialize_and_exit(pigeon: TTYPigeon | None):
# controlled GNSS stop
pigeon.send(b"\xB5\x62\x06\x04\x04\x00\x00\x00\x08\x00\x16\x74")
# store almanac in flash
pigeon.send(b"\xB5\x62\x09\x14\x04\x00\x00\x00\x00\x00\x21\xEC")
try:
if pigeon.wait_for_ack(ack=UBLOX_SOS_ACK, nack=UBLOX_SOS_NACK):
cloudlog.warning("Done storing almanac")
else:
cloudlog.error("Error storing almanac")
except TimeoutError:
pass
# turn off power and exit cleanly
set_power(False)
sys.exit(0)
@@ -281,6 +282,7 @@ def run_receiving(pigeon: TTYPigeon, pm: messaging.PubMaster, duration: int = 0)
def end_condition():
return True if duration == 0 else time.monotonic() - start_time < duration
last_almanac_save = time.monotonic()
while end_condition():
dat = pigeon.receive()
if len(dat) > 0:
@@ -294,6 +296,11 @@ def run_receiving(pigeon: TTYPigeon, pm: messaging.PubMaster, duration: int = 0)
msg = messaging.new_message('ubloxRaw', len(dat), valid=True)
msg.ubloxRaw = dat[:]
pm.send('ubloxRaw', msg)
# save almanac every 5 minutes
if (time.monotonic() - last_almanac_save) > 60*5:
save_almanac(pigeon)
last_almanac_save = time.monotonic()
else:
# prevent locking up a CPU core if ublox disconnects
time.sleep(0.001)

View File

@@ -3,21 +3,27 @@ import cffi
import os
import time
import pyray as rl
import threading
from collections.abc import Callable
from collections import deque
from dataclasses import dataclass
from enum import IntEnum
from typing import NamedTuple
from importlib.resources import as_file, files
from openpilot.common.swaglog import cloudlog
from openpilot.system.hardware import HARDWARE
from openpilot.system.hardware import HARDWARE, PC
from openpilot.common.realtime import Ratekeeper
DEFAULT_FPS = 60
DEFAULT_FPS = int(os.getenv("FPS", "60"))
FPS_LOG_INTERVAL = 5 # Seconds between logging FPS drops
FPS_DROP_THRESHOLD = 0.9 # FPS drop threshold for triggering a warning
FPS_CRITICAL_THRESHOLD = 0.5 # Critical threshold for triggering strict actions
MOUSE_THREAD_RATE = 140 # touch controller runs at 140Hz
ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "1") == "1"
SHOW_FPS = os.getenv("SHOW_FPS") == '1'
STRICT_MODE = os.getenv("STRICT_MODE") == '1'
ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "0") == "1"
SHOW_FPS = os.getenv("SHOW_FPS") == "1"
SHOW_TOUCHES = os.getenv("SHOW_TOUCHES") == "1"
STRICT_MODE = os.getenv("STRICT_MODE") == "1"
SCALE = float(os.getenv("SCALE", "1.0"))
DEFAULT_TEXT_SIZE = 60
@@ -45,6 +51,68 @@ class ModalOverlay:
callback: Callable | None = None
class MousePos(NamedTuple):
x: float
y: float
class MouseEvent(NamedTuple):
pos: MousePos
left_pressed: bool
left_released: bool
left_down: bool
t: float
class MouseState:
def __init__(self):
self._events: deque[MouseEvent] = deque(maxlen=MOUSE_THREAD_RATE) # bound event list
self._prev_mouse_event: MouseEvent | None = None
self._rk = Ratekeeper(MOUSE_THREAD_RATE)
self._lock = threading.Lock()
self._exit_event = threading.Event()
self._thread = None
def get_events(self) -> list[MouseEvent]:
with self._lock:
events = list(self._events)
self._events.clear()
return events
def start(self):
self._exit_event.clear()
if self._thread is None or not self._thread.is_alive():
self._thread = threading.Thread(target=self._run_thread, daemon=True)
self._thread.start()
def stop(self):
self._exit_event.set()
if self._thread is not None and self._thread.is_alive():
self._thread.join()
def _run_thread(self):
while not self._exit_event.is_set():
rl.poll_input_events()
self._handle_mouse_event()
self._rk.keep_time()
def _handle_mouse_event(self):
mouse_pos = rl.get_mouse_position()
ev = MouseEvent(
MousePos(mouse_pos.x, mouse_pos.y),
rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT),
rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT),
rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT),
time.monotonic(),
)
# Only add changes
if self._prev_mouse_event is None or ev[:-1] != self._prev_mouse_event[:-1]:
with self._lock:
self._events.append(ev)
self._prev_mouse_event = ev
class GuiApplication:
def __init__(self, width: int, height: int):
self._fonts: dict[FontWeight, rl.Font] = {}
@@ -61,6 +129,12 @@ class GuiApplication:
self._trace_log_callback = None
self._modal_overlay = ModalOverlay()
self._mouse = MouseState()
self._mouse_events: list[MouseEvent] = []
# Debug variables
self._mouse_history: deque[MousePos] = deque(maxlen=MOUSE_THREAD_RATE)
def request_close(self):
self._window_close_requested = True
@@ -89,6 +163,9 @@ class GuiApplication:
self._set_styles()
self._load_fonts()
if not PC:
self._mouse.start()
def set_modal_overlay(self, overlay, callback: Callable | None = None):
self._modal_overlay = ModalOverlay(overlay=overlay, callback=callback)
@@ -149,11 +226,25 @@ class GuiApplication:
rl.unload_render_texture(self._render_texture)
self._render_texture = None
if not PC:
self._mouse.stop()
rl.close_window()
@property
def mouse_events(self) -> list[MouseEvent]:
return self._mouse_events
def render(self):
try:
while not (self._window_close_requested or rl.window_should_close()):
if PC:
# Thread is not used on PC, need to manually add mouse events
self._mouse._handle_mouse_event()
# Store all mouse events for the current frame
self._mouse_events = self._mouse.get_events()
if self._render_texture:
rl.begin_texture_mode(self._render_texture)
rl.clear_background(rl.BLACK)
@@ -190,6 +281,20 @@ class GuiApplication:
if SHOW_FPS:
rl.draw_fps(10, 10)
if SHOW_TOUCHES:
for mouse_event in self._mouse_events:
if mouse_event.left_pressed:
self._mouse_history.clear()
self._mouse_history.append(mouse_event.pos)
if self._mouse_history:
mouse_pos = self._mouse_history[-1]
rl.draw_circle(int(mouse_pos.x), int(mouse_pos.y), 15, rl.RED)
for idx, mouse_pos in enumerate(self._mouse_history):
perc = idx / len(self._mouse_history)
color = rl.Color(min(int(255 * (1.5 - perc)), 255), int(min(255 * (perc + 0.5), 255)), 50, 255)
rl.draw_circle(int(mouse_pos.x), int(mouse_pos.y), 5, color)
rl.end_drawing()
self._monitor_fps()
except KeyboardInterrupt:

View File

@@ -2,7 +2,7 @@ import os
import pyray as rl
from collections.abc import Callable
from abc import ABC
from openpilot.system.ui.lib.application import gui_app, FontWeight
from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos
from openpilot.system.ui.lib.text_measure import measure_text_cached
from openpilot.system.ui.lib.wrap_text import wrap_text
from openpilot.system.ui.lib.button import gui_button, ButtonStyle
@@ -229,7 +229,7 @@ class ListItem(Widget):
super().set_parent_rect(parent_rect)
self._rect.width = parent_rect.width
def _handle_mouse_release(self, mouse_pos: rl.Vector2):
def _handle_mouse_release(self, mouse_pos: MousePos):
if not self.is_visible:
return

View File

@@ -1,6 +1,8 @@
import time
import pyray as rl
from collections import deque
from enum import IntEnum
from openpilot.system.ui.lib.application import gui_app, MouseEvent, MousePos
# Scroll constants for smooth scrolling behavior
MOUSE_WHEEL_SCROLL_SPEED = 30
@@ -38,51 +40,54 @@ class GuiScrollPanel:
self._bounds_rect: rl.Rectangle | None = None
def handle_scroll(self, bounds: rl.Rectangle, content: rl.Rectangle) -> rl.Vector2:
# TODO: HACK: this class is driven by mouse events, so we need to ensure we have at least one event to process
for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), False, False, False, time.monotonic())]:
self._handle_mouse_event(mouse_event, bounds, content)
return self._offset
def _handle_mouse_event(self, mouse_event: MouseEvent, bounds: rl.Rectangle, content: rl.Rectangle):
# Store rectangles for reference
self._content_rect = content
self._bounds_rect = bounds
# Calculate time delta
current_time = rl.get_time()
mouse_pos = rl.get_mouse_position()
max_scroll_y = max(content.height - bounds.height, 0)
# Start dragging on mouse press
if rl.check_collision_point_rec(mouse_pos, bounds) and rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT):
if rl.check_collision_point_rec(mouse_event.pos, bounds) and mouse_event.left_pressed:
if self._scroll_state == ScrollState.IDLE or self._scroll_state == ScrollState.BOUNCING:
self._scroll_state = ScrollState.DRAGGING_CONTENT
if self._show_vertical_scroll_bar:
scrollbar_width = rl.gui_get_style(rl.GuiControl.LISTVIEW, rl.GuiListViewProperty.SCROLLBAR_WIDTH)
scrollbar_x = bounds.x + bounds.width - scrollbar_width
if mouse_pos.x >= scrollbar_x:
if mouse_event.pos.x >= scrollbar_x:
self._scroll_state = ScrollState.DRAGGING_SCROLLBAR
# TODO: hacky
# when clicking while moving, go straight into dragging
self._is_dragging = abs(self._velocity_y) > MIN_VELOCITY
self._last_mouse_y = mouse_pos.y
self._start_mouse_y = mouse_pos.y
self._last_drag_time = current_time
self._last_mouse_y = mouse_event.pos.y
self._start_mouse_y = mouse_event.pos.y
self._last_drag_time = mouse_event.t
self._velocity_history.clear()
self._velocity_y = 0.0
self._bounce_offset = 0.0
# Handle active dragging
if self._scroll_state == ScrollState.DRAGGING_CONTENT or self._scroll_state == ScrollState.DRAGGING_SCROLLBAR:
if rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT):
delta_y = mouse_pos.y - self._last_mouse_y
if mouse_event.left_down:
delta_y = mouse_event.pos.y - self._last_mouse_y
# Track velocity for inertia
time_since_last_drag = current_time - self._last_drag_time
time_since_last_drag = mouse_event.t - self._last_drag_time
if time_since_last_drag > 0:
drag_velocity = delta_y / time_since_last_drag / 60.0
# TODO: HACK: /2 since we usually get two touch events per frame
drag_velocity = delta_y / time_since_last_drag / 60.0 / 2 # TODO: shouldn't be hardcoded
self._velocity_history.append(drag_velocity)
self._last_drag_time = current_time
self._last_drag_time = mouse_event.t
# Detect actual dragging
total_drag = abs(mouse_pos.y - self._start_mouse_y)
total_drag = abs(mouse_event.pos.y - self._start_mouse_y)
if total_drag > DRAG_THRESHOLD:
self._is_dragging = True
@@ -96,9 +101,9 @@ class GuiScrollPanel:
scroll_ratio = content.height / bounds.height
self._offset.y -= delta_y * scroll_ratio
self._last_mouse_y = mouse_pos.y
self._last_mouse_y = mouse_event.pos.y
elif rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
elif mouse_event.left_released:
# Calculate flick velocity
if self._velocity_history:
total_weight = 0
@@ -167,8 +172,6 @@ class GuiScrollPanel:
elif self._offset.y < -(max_scroll_y + MAX_BOUNCE_DISTANCE):
self._offset.y = -(max_scroll_y + MAX_BOUNCE_DISTANCE)
return self._offset
def is_touch_valid(self):
return not self._is_dragging

View File

@@ -1,4 +1,5 @@
import pyray as rl
from openpilot.system.ui.lib.application import MousePos
from openpilot.system.ui.lib.widget import Widget
ON_COLOR = rl.Color(51, 171, 76, 255)
@@ -23,7 +24,7 @@ class Toggle(Widget):
def set_rect(self, rect: rl.Rectangle):
self._rect = rl.Rectangle(rect.x, rect.y, WIDTH, HEIGHT)
def _handle_mouse_release(self, mouse_pos: rl.Vector2):
def _handle_mouse_release(self, mouse_pos: MousePos):
if not self._enabled:
return

View File

@@ -2,6 +2,7 @@ import abc
import pyray as rl
from enum import IntEnum
from collections.abc import Callable
from openpilot.system.ui.lib.application import gui_app, MousePos
class DialogResult(IntEnum):
@@ -66,18 +67,18 @@ class Widget(abc.ABC):
ret = self._render(self._rect)
# Keep track of whether mouse down started within the widget's rectangle
mouse_pos = rl.get_mouse_position()
if rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._touch_valid():
if rl.check_collision_point_rec(mouse_pos, self._rect):
self._is_pressed = True
for mouse_event in gui_app.mouse_events:
if mouse_event.left_pressed and self._touch_valid():
if rl.check_collision_point_rec(mouse_event.pos, self._rect):
self._is_pressed = True
elif not self._touch_valid():
self._is_pressed = False
elif not self._touch_valid():
self._is_pressed = False
elif rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
if self._is_pressed and rl.check_collision_point_rec(mouse_pos, self._rect):
self._handle_mouse_release(mouse_pos)
self._is_pressed = False
elif mouse_event.left_released:
if self._is_pressed and rl.check_collision_point_rec(mouse_event.pos, self._rect):
self._handle_mouse_release(mouse_event.pos)
self._is_pressed = False
return ret
@@ -91,6 +92,6 @@ class Widget(abc.ABC):
def _update_layout_rects(self) -> None:
"""Optionally update any layout rects on Widget rect change."""
def _handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool:
def _handle_mouse_release(self, mouse_pos: MousePos) -> bool:
"""Optionally handle mouse release events."""
return False

View File

@@ -204,7 +204,7 @@ class WifiManager:
'connection': {
'type': Variant('s', '802-11-wireless'),
'uuid': Variant('s', str(uuid.uuid4())),
'id': Variant('s', ssid),
'id': Variant('s', f'openpilot connection {ssid}'),
'autoconnect-retries': Variant('i', 0),
},
'802-11-wireless': {
@@ -212,7 +212,10 @@ class WifiManager:
'hidden': Variant('b', is_hidden),
'mode': Variant('s', 'infrastructure'),
},
'ipv4': {'method': Variant('s', 'auto')},
'ipv4': {
'method': Variant('s', 'auto'),
'dns-priority': Variant('i', 600),
},
'ipv6': {'method': Variant('s', 'ignore')},
}

View File

@@ -198,15 +198,12 @@ def maybe_update_radar_points(lt, lid_overlay):
ar_pts = {}
for track in lt:
ar_pts[track.trackId] = [track.dRel, track.yRel, track.vRel, track.aRel]
for ids, pt in ar_pts.items():
for pt in ar_pts.values():
# negative here since radar is left positive
px, py = to_topdown_pt(pt[0], -pt[1])
if px != -1:
color = 255
if int(ids) == 1:
lid_overlay[px - 2:px + 2, py - 10:py + 10] = 100
else:
lid_overlay[px - 2:px + 2, py - 2:py + 2] = color
lid_overlay[px - 4:px + 4, py - 4:py + 4] = 0
lid_overlay[px - 2:px + 2, py - 2:py + 2] = 255
def get_blank_lid_overlay(UP):
lid_overlay = np.zeros((UP.lidar_x, UP.lidar_y), 'uint8')

77
tools/scripts/extract_audio.py Executable file
View File

@@ -0,0 +1,77 @@
#!/usr/bin/env python3
import os
import sys
import wave
import argparse
import numpy as np
from openpilot.tools.lib.logreader import LogReader, ReadMode
def extract_audio(route_or_segment_name, output_file=None, play=False):
lr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACTIVE)
audio_messages = list(lr.filter("rawAudioData"))
if not audio_messages:
print("No rawAudioData messages found in logs")
return
sample_rate = audio_messages[0].sampleRate
audio_chunks = []
total_frames = 0
for msg in audio_messages:
audio_array = np.frombuffer(msg.data, dtype=np.int16)
audio_chunks.append(audio_array)
total_frames += len(audio_array)
full_audio = np.concatenate(audio_chunks)
print(f"Found {total_frames} frames from {len(audio_messages)} audio messages at {sample_rate} Hz")
if output_file:
if write_wav_file(output_file, full_audio, sample_rate):
print(f"Audio written to {output_file}")
else:
print("Audio extraction canceled.")
if play:
play_audio(full_audio, sample_rate)
def write_wav_file(filename, audio_data, sample_rate):
if os.path.exists(filename):
if input(f"File '{filename}' exists. Overwrite? (y/N): ").lower() not in ['y', 'yes']:
return False
with wave.open(filename, 'wb') as wav_file:
wav_file.setnchannels(1) # Mono
wav_file.setsampwidth(2) # 16-bit
wav_file.setframerate(sample_rate)
wav_file.writeframes(audio_data.tobytes())
return True
def play_audio(audio_data, sample_rate):
try:
import sounddevice as sd
print("Playing audio... Press Ctrl+C to stop")
sd.play(audio_data, sample_rate)
sd.wait()
except KeyboardInterrupt:
print("\nPlayback stopped")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Extract audio data from openpilot logs")
parser.add_argument("-o", "--output", help="Output WAV file path")
parser.add_argument("--play", action="store_true", help="Play audio with sounddevice")
parser.add_argument("route_or_segment_name", nargs='?', help="The route or segment name")
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
args = parser.parse_args()
output_file = args.output
if not args.output and not args.play:
output_file = "extracted_audio.wav"
extract_audio(args.route_or_segment_name.strip(), output_file, args.play)

View File

@@ -11,7 +11,14 @@ class Camera:
self.stream_type = stream_type
self.cur_frame_id = 0
print(f"Opening {cam_type_state} at {camera_id}")
self.cap = cv.VideoCapture(camera_id)
self.cap.set(cv.CAP_PROP_FRAME_WIDTH, 1280.0)
self.cap.set(cv.CAP_PROP_FRAME_HEIGHT, 720.0)
self.cap.set(cv.CAP_PROP_FPS, 25.0)
self.W = self.cap.get(cv.CAP_PROP_FRAME_WIDTH)
self.H = self.cap.get(cv.CAP_PROP_FRAME_HEIGHT)
@@ -25,6 +32,8 @@ class Camera:
ret, frame = self.cap.read()
if not ret:
break
# Rotate the frame 180 degrees (flip both axes)
frame = cv.flip(frame, -1)
yuv = Camera.bgr2nv12(frame)
yield yuv.data.tobytes()
self.cap.release()

View File

@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import threading
import os
import platform
from collections import namedtuple
from msgq.visionipc import VisionIpcServer, VisionStreamType
@@ -30,8 +31,7 @@ class Camerad:
self.cameras = []
for c in CAMERAS:
cam_device = f"/dev/video{c.cam_id}"
print(f"opening {c.msg_name} at {cam_device}")
cam_device = f"/dev/video{c.cam_id}" if platform.system() != "Darwin" else c.cam_id
cam = Camera(c.msg_name, c.stream_type, cam_device)
self.cameras.append(cam)
self.vipc_server.create_buffers(c.stream_type, 20, cam.W, cam.H)

1404
uv.lock generated

File diff suppressed because it is too large Load Diff