Compare commits

..

13 Commits

Author SHA1 Message Date
rav4kumar a598f43690 ref 2026-11-17 10:01:10 -07:00
rav4kumar 7f8db47fd0 Merge remote-tracking branch 'origin/master' into toyota-bsm 2026-11-17 09:59:36 -07:00
rav4kumar 1291d14108 Merge origin/master into toyota-bsm 2026-11-11 13:14:18 -07:00
rav4kumar c831334d76 gate op long 2026-11-11 13:11:34 -07:00
Kumar 5958b2d25b Merge branch 'master' into toyota-bsm 2026-11-05 11:26:28 -07:00
Kumar 73c8b30fa9 Merge branch 'master' into toyota-bsm 2026-11-04 12:14:55 -07:00
rav4kumar 8bdb2f54d9 Merge branch 'master' into toyota-bsm 2026-11-03 10:37:16 -07:00
rav4kumar 9c1bdde65b toyota: enable the enhanced BSM toggle for TSS-P cars with a smartDSU 2026-11-03 10:31:07 -07:00
Kumar 914aaefac1 Merge branch 'master' into toyota-bsm 2026-10-26 11:35:14 -07:00
rav4kumar d05649c0a4 toyota: bump opendbc and fix BSM wiring 2026-10-26 11:32:51 -07:00
rav4kumar 3521140920 ref 2026-10-25 11:56:26 -07:00
rav4kumar 56628fc770 ref 2026-10-25 11:55:52 -07:00
rav4kumar 60a9cc8657 toyota: add Enhanced Blind Spot Monitoring setting 2026-10-25 11:55:32 -07:00
16 changed files with 174 additions and 298 deletions
+1
View File
@@ -4,6 +4,7 @@
[submodule "opendbc"]
path = opendbc_repo
url = https://github.com/sunnypilot/opendbc.git
branch = toyotabsmhack
[submodule "msgq"]
path = msgq_repo
url = https://github.com/sunnypilot/msgq.git
+1
View File
@@ -245,6 +245,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaMadsScreenButton", {PERSISTENT | BACKUP, INT, "0"}},
{"ToyotaEnhancedBsm", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaTSS2Smooth", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -249,11 +249,7 @@ class ModelsLayout(Widget):
return {"name": name, "downloading": True, "progress": progress, "status_text": tr("verifying")}
if ds.downloading in statuses:
speed = max((p.speed for p in progresses), default=0.0)
speed_text = f"{speed / 1024 ** 2:.1f} MB/s" if speed > 0 else "--.- MB/s"
if any(p.resuming for p in progresses):
status_text = tr("resuming...") + f" {speed_text}"
else:
status_text = speed_text
status_text = f"{speed / 1024 ** 2:.1f} MB/s" if speed > 0 else "--.- MB/s"
return {"name": name, "downloading": True, "progress": progress, "status_text": status_text}
if statuses <= {ds.downloaded, ds.cached}:
return {"name": name, "text_color": ON_COLOR, "icon": "icons/checkmark.png"}
@@ -11,11 +11,17 @@ from openpilot.system.ui.lib.multilang import tr, tr_noop
from openpilot.system.ui.widgets import DialogResult
from openpilot.system.ui.widgets.confirm_dialog import ConfirmDialog
from openpilot.system.ui.sunnypilot.widgets.list_view import toggle_item_sp
from opendbc.car.toyota.values import ToyotaFlags
from opendbc.car.toyota.values import CAR, ToyotaFlags
from opendbc.sunnypilot.car.toyota.values import ToyotaFlagsSP
ONROAD_ONLY_DESCRIPTION = tr_noop("Start the vehicle to check vehicle compatibility.")
SNG_HACK_UNAVAILABLE = tr_noop("sunnypilot Longitudinal Control must be available and enabled for your vehicle to use this feature.")
BSM_UNAVAILABLE = tr_noop("Enhanced Blind Spot Monitoring requires a TSS2 Toyota/Lexus without SecOC, " +
"or a smartDSU with sunnypilot longitudinal control.")
BSM_STOCK_LONG_UNAVAILABLE = tr_noop("Enhanced Blind Spot Monitoring is not available while Enforce Factory Longitudinal Control is enabled.")
BSM_REQUIRED = tr_noop("This vehicle does not report blind spot status over the standard CAN signals. " +
"Enhanced Blind Spot Monitoring is required for blind spot detection.")
PLATFORM_UNAVAILABLE = tr_noop("This feature is currently not available on this platform.")
DESCRIPTIONS = {
@@ -26,6 +32,10 @@ DESCRIPTIONS = {
'sunnypilot will allow some Toyota/Lexus cars to auto resume during stop and go traffic. ' +
'This feature is only applicable to certain models that are able to use longitudinal control. This is an alpha feature. Use at your own risk.'
),
'enhanced_bsm': tr_noop(
'sunnypilot will poll the blind spot monitor diagnostics directly from the radar. Use this on vehicles whose blind spot monitors ' +
'do not report blind spot status over the standard CAN signals. This is an alpha feature. Use at your own risk.'
),
'tss2_smooth': tr_noop(
'Enable smoother longitudinal control for TSS2 vehicles. ' +
'Uses a custom speed-dependent integral gain curve for smoother traffic driving.'
@@ -53,6 +63,14 @@ class ToyotaSettings(BrandSettings):
enabled=lambda: not ui_state.engaged,
)
self.enhanced_bsm = toggle_item_sp(
lambda: tr("Enhanced Blind Spot Monitoring (Alpha)"),
description=lambda: tr(DESCRIPTIONS["enhanced_bsm"]),
initial_state=ui_state.params.get_bool("ToyotaEnhancedBsm"),
callback=self._on_enable_enhanced_bsm,
enabled=lambda: not ui_state.engaged,
)
self.tss2_smooth = toggle_item_sp(
lambda: tr("TSS2 Smooth Longitudinal (Alpha)"),
description=lambda: tr(DESCRIPTIONS["tss2_smooth"]),
@@ -64,6 +82,7 @@ class ToyotaSettings(BrandSettings):
self.items = [
self.enforce_stock_longitudinal,
self.stop_and_go_hack,
self.enhanced_bsm,
self.tss2_smooth,
]
@@ -130,6 +149,25 @@ class ToyotaSettings(BrandSettings):
ui_state.params.put_bool("ToyotaStopAndGoHack", False)
ui_state.params.put_bool("OnroadCycleRequested", True)
def _on_enable_enhanced_bsm(self, state: bool):
if state:
def confirm_callback(result: int):
if result == DialogResult.CONFIRM:
ui_state.params.put_bool("ToyotaEnhancedBsm", True)
ui_state.params.put_bool("OnroadCycleRequested", True)
else:
self.enhanced_bsm.action_item.set_state(False)
content = (f"<h1>{self.enhanced_bsm.title}</h1><br>" +
f"<p>{self.enhanced_bsm.description}</p>")
dlg = ConfirmDialog(content, tr("Enable"), rich=True, callback=confirm_callback)
gui_app.push_widget(dlg)
else:
ui_state.params.put_bool("ToyotaEnhancedBsm", False)
ui_state.params.put_bool("OnroadCycleRequested", True)
def update_settings(self):
if ui_state.CP is not None:
longitudinal = ui_state.CP.openpilotLongitudinalControl
@@ -150,6 +188,33 @@ class ToyotaSettings(BrandSettings):
if show_desc:
self.stop_and_go_hack.show_description(True)
# Enhanced BSM needs the radar diagnostic address, which is only in the TX allowlist for
# TSS2 cars without SecOC, or for TSS-P cars with a smartDSU. Both need openpilot long.
sdsu_bsm_supported = (ui_state.CP_SP is not None and
bool(ui_state.CP_SP.flags & ToyotaFlagsSP.SMART_DSU))
bsm_supported = (not bool(ui_state.CP.flags & ToyotaFlags.SECOC) and
(bool(ui_state.CP.flags & ToyotaFlags.TSS2) or sdsu_bsm_supported))
bsm_available = bsm_supported and longitudinal and not enforce_stock
self.enhanced_bsm.action_item.set_enabled(bsm_available and not ui_state.engaged)
bsm_desc = tr(DESCRIPTIONS["enhanced_bsm"])
if not bsm_supported:
self.enhanced_bsm.action_item.set_state(False)
bsm_desc = "<b>" + tr(BSM_UNAVAILABLE) + "</b>\n\n" + bsm_desc
elif enforce_stock:
self.enhanced_bsm.action_item.set_state(False)
bsm_desc = "<b>" + tr(BSM_STOCK_LONG_UNAVAILABLE) + "</b>\n\n" + bsm_desc
elif not longitudinal:
self.enhanced_bsm.action_item.set_state(False)
bsm_desc = "<b>" + tr(SNG_HACK_UNAVAILABLE) + "</b>\n\n" + bsm_desc
elif ui_state.CP.carFingerprint == CAR.TOYOTA_PRIUS_TSS2:
bsm_desc = "<b>" + tr(BSM_REQUIRED) + "</b>\n\n" + bsm_desc
if self.enhanced_bsm.description != bsm_desc:
self.enhanced_bsm.set_description(bsm_desc)
if not bsm_available:
self.enhanced_bsm.show_description(True)
tss2 = bool(ui_state.CP.flags & ToyotaFlags.TSS2)
if longitudinal and not enforce_stock and tss2:
self.tss2_smooth.action_item.set_enabled(not ui_state.engaged)
@@ -173,6 +238,12 @@ class ToyotaSettings(BrandSettings):
self.stop_and_go_hack.set_description(new_desc)
self.stop_and_go_hack.show_description(True)
self.enhanced_bsm.action_item.set_enabled(False)
bsm_desc = "<b>" + tr(ONROAD_ONLY_DESCRIPTION) + "</b>\n\n" + tr(DESCRIPTIONS["enhanced_bsm"])
if self.enhanced_bsm.description != bsm_desc:
self.enhanced_bsm.set_description(bsm_desc)
self.enhanced_bsm.show_description(True)
self.tss2_smooth.action_item.set_enabled(False)
new_desc = "<b>" + tr(ONROAD_ONLY_DESCRIPTION) + "</b>\n\n" + tr(DESCRIPTIONS["tss2_smooth"])
if self.tss2_smooth.description != new_desc:
@@ -267,8 +267,8 @@ class ModelsLayoutMici(NavScroller):
self.current_model_info.info_text.set_text(info_text)
if manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed:
self.current_model_info.info_header.set_text(tr("error"))
self.current_model_info.info_text.set_text(f"{manager.selectedBundle.internalName.lower()} | {tr('download failed')}")
self.current_model_info.info_header.set_text(tr("error") + self._download_progress)
self.current_model_info.info_text.set_text(tr("download failed"))
elif manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloading:
self.cancel_download_btn.set_visible(ui_state.params.get("ModelManager_DownloadRef") is not None)
@@ -277,7 +277,6 @@ class ModelsLayoutMici(NavScroller):
count = 0
verifying = False
canceling = False
resuming = False
speed = 0.0
for model in manager.selectedBundle.models:
count += 1
@@ -287,13 +286,12 @@ class ModelsLayoutMici(NavScroller):
progress += p.progress
verifying = verifying or p.status == custom.ModelManagerSP.DownloadStatus.verifying
canceling = canceling or p.canceling
resuming = resuming or p.resuming
speed = max(speed, p.speed) # only one file downloads at a time, so the non-zero speed is the live one
elif p.status in (custom.ModelManagerSP.DownloadStatus.downloaded,
custom.ModelManagerSP.DownloadStatus.cached):
progress += 100.0
header = tr("canceling...") if canceling else tr("verifying") if verifying else tr("resuming...") if resuming else tr("downloading")
header = tr("canceling...") if canceling else tr("verifying") if verifying else tr("downloading")
self.current_model_info.current_model_header.set_text(header)
self.cancel_download_btn.set_text(tr("cancel verification") if verifying else tr("cancel download"))
self.current_model_info.current_model_header._shimmer = True
+29 -120
View File
@@ -6,7 +6,6 @@ See the LICENSE.md file in the root directory for more details.
"""
import asyncio
import json
import os
import re
import threading
@@ -29,11 +28,8 @@ DOWNLOAD_TIMEOUT = (30, 30)
# HuggingFace throttles each connection to ~1-2 MB/s but not the connection count; 12 saturates the device link
MAX_CONCURRENT_PIECES = 12
PIECE_SIZE = 8 * 1024 * 1024
PIECE_ATTEMPTS = 3 # per piece, before the transfer as a whole is retried
PIECE_ATTEMPTS = 3
PIECE_BACKOFF = 1.0 # seconds, growing linearly: waits 1 s then 2 s between a piece's three attempts
DOWNLOAD_ATTEMPTS = 4 # whole-transfer attempts, each resuming from the sidecar
RETRY_BACKOFF = 2.0 # seconds, doubling: waits 2, 4 then 8 s between the four transfer attempts
RETRY_POLL = 0.25 # how often a backoff checks for a cancel
TRANSIENT_HTTP_STATUS = frozenset({408, 429, 500, 502, 503, 504})
REPORT_INTERVAL = 0.5 # seconds between progress publications
# byte ranges count encoded bytes but iter_content yields decoded ones, so ask for an unencoded body
@@ -53,69 +49,26 @@ def _is_transient(e: BaseException) -> bool:
class _Progress:
def __init__(self, done: set[int], done_bytes: int):
self._bytes = done_bytes
self._done = set(done)
def __init__(self):
self._bytes = 0
self._lock = threading.Lock()
def add_bytes(self, n: int) -> None:
with self._lock:
self._bytes += n
def mark_done(self, index: int) -> None:
def snapshot(self) -> int:
with self._lock:
self._done.add(index)
def snapshot(self) -> tuple[int, set[int]]:
with self._lock:
return self._bytes, set(self._done)
return self._bytes
def _piece_ranges(total: int, piece_size: int) -> list[tuple[int, int]]:
return [(start, min(start + piece_size, total)) for start in range(0, total, piece_size)]
def _sidecar_path(path: str) -> str:
return f"{path}.download"
def _download_in_progress(path: str) -> bool:
return os.path.isfile(_sidecar_path(path))
def _save_resume_state(path: str, layout: dict, done: set[int]) -> None:
tmp = _sidecar_path(path) + ".tmp"
with open(tmp, "w") as f:
json.dump({**layout, "done": sorted(done)}, f)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, _sidecar_path(path)) # a power loss leaves the old sidecar or the new one, never a torn file
dir_fd = os.open(os.path.dirname(path) or ".", os.O_RDONLY)
try:
os.fsync(dir_fd)
finally:
os.close(dir_fd)
def _remove_download_state(path: str) -> None:
for stale in (_sidecar_path(path), _sidecar_path(path) + ".tmp"):
if os.path.isfile(stale):
os.remove(stale)
def _prepare_target(path: str, layout: dict) -> set[int]:
total = layout["total"]
try:
with open(_sidecar_path(path)) as f:
state = json.load(f)
if isinstance(state, dict) and all(state.get(k) == v for k, v in layout.items()) and os.path.getsize(path) == total:
return {i for i in state.get("done", []) if isinstance(i, int)}
except (OSError, ValueError, TypeError): # missing, unreadable or malformed sidecar: start over
pass
_save_resume_state(path, layout, set())
def _prepare_target(path: str, total: int) -> None:
with open(path, "wb") as f:
f.truncate(total)
return set()
def _content_range_total(response: requests.Response) -> int | None:
@@ -157,9 +110,6 @@ def _fetch_piece(url: str, path: str, index: int, start: int, end: int | None, p
f.truncate() # unknown length: the body defines the file size
if end is not None and written != end - start:
raise requests.exceptions.ConnectionError(f"short read: {written} of {end - start} bytes")
f.flush()
os.fsync(f.fileno()) # before it is recorded, so a power loss never leaves the sidecar naming unsynced bytes
progress.mark_done(index)
return
except requests.RequestException as e:
progress.add_bytes(-written) # the piece restarts from scratch
@@ -170,11 +120,6 @@ def _fetch_piece(url: str, path: str, index: int, start: int, end: int | None, p
raise DownloadCancelled("Download cancelled") from None
def _remove_file(path: str) -> None:
if os.path.isfile(path):
os.remove(path)
class ModelManagerSP:
"""Manages model downloads and status reporting"""
@@ -241,12 +186,11 @@ class ModelManagerSP:
else:
eta = self._calculate_eta(artifact.fileName, progress)
self._set_progress(artifact, custom.ModelManagerSP.DownloadStatus.downloading, progress, eta, speed)
artifact.downloadProgress.resuming = False # shown only for the first report, when the bar jumps to the resumed point
async def _report_until_done(self, tasks: list[asyncio.Future], artifact, progress: _Progress, path: str, layout: dict,
started: tuple[int, set[int], float]) -> None:
async def _report_until_done(self, tasks: list[asyncio.Future], artifact, progress: _Progress, total: int,
started: tuple[int, float]) -> None:
pending = set(tasks)
last_bytes, saved, last_time = started
last_bytes, last_time = started
speed = 0.0
while pending:
done, pending = await asyncio.wait(pending, timeout=REPORT_INTERVAL)
@@ -255,17 +199,14 @@ class ModelManagerSP:
if self._download_interrupted():
raise DownloadCancelled("Download cancelled")
now = time.monotonic()
done_bytes, done_pieces = progress.snapshot()
if done_pieces != saved:
_save_resume_state(path, layout, done_pieces)
saved = done_pieces
done_bytes = progress.snapshot()
if (dt := now - last_time) > 0:
instant = max(0.0, (done_bytes - last_bytes) / dt) # bytes/s this tick; clamped since retried pieces can lower the byte count
speed = instant if speed == 0 else SPEED_SMOOTHING * speed + (1 - SPEED_SMOOTHING) * instant
if speed < SPEED_FLOOR: # a stall decays toward 0 without reaching it, and a near-0 speed overflows the eta
speed = 0.0
last_time, last_bytes = now, done_bytes
self._publish_progress(artifact, done_bytes, layout["total"], speed)
self._publish_progress(artifact, done_bytes, total, speed)
async def _download_file(self, url: str, path: str, artifact) -> None:
self._download_start_times[artifact.fileName] = time.monotonic()
@@ -273,18 +214,16 @@ class ModelManagerSP:
total, ranged = await loop.run_in_executor(None, _probe_size, url)
pieces: list[tuple[int, int | None]] = list(_piece_ranges(total, PIECE_SIZE)) if ranged else [(0, None)]
layout = {"total": total, "piece_size": PIECE_SIZE, "ranged": ranged, "sha256": artifact.downloadUri.sha256}
done = await loop.run_in_executor(None, _prepare_target, path, layout)
resumed_bytes = sum(end - start for i, (start, end) in enumerate(pieces) if i in done and end is not None)
progress = _Progress(done, resumed_bytes) # the bar and eta start from the resumed pieces
await loop.run_in_executor(None, _prepare_target, path, total)
progress = _Progress()
cancel = threading.Event()
started = *progress.snapshot(), time.monotonic() # before any worker can land bytes
started = progress.snapshot(), time.monotonic() # before any worker can land bytes
with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_PIECES) as pool:
tasks = [loop.run_in_executor(pool, _fetch_piece, url, path, i, start, end, progress, cancel, self._block_size)
for i, (start, end) in enumerate(pieces) if i not in done]
for i, (start, end) in enumerate(pieces)]
try:
await self._report_until_done(tasks, artifact, progress, path, layout, started)
await self._report_until_done(tasks, artifact, progress, total, started)
except BaseException as e:
cancel.set()
if isinstance(e, DownloadCancelled):
@@ -293,32 +232,9 @@ class ModelManagerSP:
for task in tasks:
task.cancel() # drops queued pieces; running ones see the event at their next block
await asyncio.gather(*tasks, return_exceptions=True)
await loop.run_in_executor(None, pool.shutdown) # let running pieces stop so none lands after the save
_save_resume_state(path, layout, progress.snapshot()[1])
raise
_remove_download_state(path)
artifact.downloadProgress.resuming = False
del self._download_start_times[artifact.fileName]
async def _download_with_retries(self, url: str, path: str, artifact) -> None:
for attempt in range(DOWNLOAD_ATTEMPTS):
try:
await self._download_file(url, path, artifact)
return
except Exception as e:
if not _is_transient(e) or attempt == DOWNLOAD_ATTEMPTS - 1:
raise
delay = RETRY_BACKOFF * 2 ** attempt
cloudlog.warning(f"Retrying {artifact.fileName} in {delay:g}s after {type(e).__name__}: {e}")
progress = artifact.downloadProgress
self._set_progress(artifact, progress.status, progress.progress, progress.eta, 0.0) # stalled: no speed
deadline = time.monotonic() + delay
while (remaining := deadline - time.monotonic()) > 0:
if self._download_interrupted():
raise DownloadCancelled("Download cancelled") from e
await asyncio.sleep(min(RETRY_POLL, remaining))
async def _process_artifact(self, artifact, destination_path: str) -> None:
if not artifact.downloadUri.uri:
return None
@@ -330,39 +246,37 @@ class ModelManagerSP:
filename = artifact.fileName
full_path = os.path.join(destination_path, filename)
status = custom.ModelManagerSP.DownloadStatus
artifact.downloadProgress.resuming = False
artifact.downloadProgress.canceling = False
try:
if _download_in_progress(full_path):
cloudlog.info(f"Resuming interrupted download of {filename}")
artifact.downloadProgress.resuming = True
else:
self._set_progress(artifact, status.verifying, 0)
if await verify_file(full_path, expected_hash):
self._set_progress(artifact, status.cached, 100)
return
self._set_progress(artifact, status.verifying, 0)
if await verify_file(full_path, expected_hash):
self._set_progress(artifact, status.cached, 100)
return
self._set_progress(artifact, status.downloading, 0)
await self._download_with_retries(url, full_path, artifact)
await self._download_file(url, full_path, artifact)
self._set_progress(artifact, status.verifying, 99)
if not await verify_file(full_path, expected_hash):
_remove_file(full_path)
_remove_download_state(full_path)
raise ValueError(f"Hash validation failed for {filename}")
self._set_progress(artifact, status.downloaded, 100)
except DownloadCancelled: # not a failure: the request handler clears the row
except DownloadCancelled:
self._download_start_times.pop(artifact.fileName, None)
artifact.downloadProgress.status = status.failed
artifact.downloadProgress.eta = 0
artifact.downloadProgress.speed = 0
self._sync_artifact_progress(artifact)
if self.selected_bundle:
self.selected_bundle.status = status.failed
self._report_status()
raise
except Exception as e:
cloudlog.error(f"Error downloading {filename}: {str(e)}")
if os.path.isfile(full_path): # noqa: ASYNC240
os.remove(full_path)
artifact.downloadProgress.status = status.failed
artifact.downloadProgress.eta = 0
artifact.downloadProgress.speed = 0
@@ -419,8 +333,6 @@ class ModelManagerSP:
self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True)
self.active_bundle = get_active_bundle(self.params, chestnut=self.chestnut_present)
except DownloadCancelled:
raise # not a failure
except Exception:
if self.selected_bundle is not None:
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed
@@ -448,14 +360,11 @@ class ModelManagerSP:
self._download_ref = ref_to_download
try:
self.download(model_to_download, Paths.model_root(), source)
except DownloadCancelled:
self.selected_bundle = None
except Exception as e:
cloudlog.exception(e) # a failure stays on the row until the next request
else:
self.selected_bundle = None
cloudlog.exception(e)
finally:
self._release_download_ref()
self.selected_bundle = None
def main_thread(self) -> None:
"""Main thread for model management"""
@@ -11,13 +11,11 @@ import contextlib
import dataclasses
import hashlib
import http.server
import json
import math
import os
import random
import re
import requests
import stat
import tempfile
import threading
import time
@@ -57,8 +55,6 @@ class ServerState:
fail_ranges: dict[tuple[int, int], int] = dataclasses.field(default_factory=dict) # range -> HTTP status, every time
fail_times: dict[tuple[int, int], int] = dataclasses.field(default_factory=dict) # range -> how many more requests 503
short_once: set[tuple[int, int]] = dataclasses.field(default_factory=set) # ranges whose first answer is cut short
fail_after_others: bool = False # hold each failure until every other piece has been sent
sent: set[tuple[int, int]] = dataclasses.field(default_factory=set) # ranges sent in full
stall_ranges: set[tuple[int, int]] = dataclasses.field(default_factory=set)
stall_event: threading.Event | None = None
stall_released_by_event: bool | None = None
@@ -92,10 +88,6 @@ class DownloadHandler(http.server.BaseHTTPRequestHandler):
state.fail_times[rng] -= 1
status = 503
if status is not None:
others = {piece_range(i) for i in range(NUM_PIECES)} - {rng}
deadline = time.monotonic() + 5
while state.fail_after_others and not others <= state.sent and time.monotonic() < deadline:
time.sleep(0.01)
self.send_response(status)
self.end_headers()
return
@@ -123,8 +115,6 @@ class DownloadHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(payload[100:])
else:
self.wfile.write(payload)
if rng is not None:
state.sent.add(rng)
@contextlib.contextmanager
@@ -143,10 +133,7 @@ def threaded_server(handler):
class InlineExecutor(concurrent.futures.Executor):
def submit(self, fn, /, *args, **kwargs):
future: concurrent.futures.Future = concurrent.futures.Future()
try:
future.set_result(fn(*args, **kwargs)) # the piece lands before the reporter starts
except BaseException as e:
future.set_exception(e)
future.set_result(fn(*args, **kwargs)) # the piece lands before the reporter starts
return future
@@ -160,12 +147,11 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
self.addCleanup(self._tmp.cleanup)
self.dest = self._tmp.name
for name, value in (('PIECE_SIZE', PIECE_SIZE), ('REPORT_INTERVAL', 0.05), ('PIECE_BACKOFF', 0.01), ('SPEED_FLOOR', 100), ('RETRY_BACKOFF', 0.01)):
for name, value in (('PIECE_SIZE', PIECE_SIZE), ('REPORT_INTERVAL', 0.05), ('PIECE_BACKOFF', 0.01), ('SPEED_FLOOR', 100)):
self.enterContext(mock.patch.object(manager_module, name, value))
self.reported: list[float] = []
self.reported_statuses: list[int] = []
self.reported_resuming: list[bool] = []
self.manager = ModelManagerSP.__new__(ModelManagerSP)
self.params = self.manager.params = mock.MagicMock()
@@ -186,7 +172,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
self.reported.append(float(artifact.downloadProgress.progress))
status = artifact.downloadProgress.status
self.reported_statuses.append(getattr(status, 'raw', status)) # .raw: _DynamicEnum equals its int but does not hash like it
self.reported_resuming.append(bool(artifact.downloadProgress.resuming))
def make_artifact(self):
bundle = custom.ModelManagerSP.ModelBundle.new_message()
@@ -203,24 +188,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
def path(self) -> str:
return os.path.join(self.dest, FILE_NAME)
@property
def sidecar(self) -> str:
return manager_module._sidecar_path(self.path)
def resume_state(self) -> dict:
with open(self.sidecar) as f:
return json.load(f)
def write_resume_state(self, done: list[int], total: int = len(WHOLE_BODY), file_size: int | None = None,
file_sha256: str = sha256(WHOLE_BODY)) -> None:
with open(self.path, 'wb') as f:
f.truncate(len(WHOLE_BODY) if file_size is None else file_size)
for i in done:
f.seek(i * PIECE_SIZE)
f.write(WHOLE_BODY[i * PIECE_SIZE:(i + 1) * PIECE_SIZE])
with open(self.sidecar, 'w') as f:
json.dump({"total": total, "piece_size": PIECE_SIZE, "ranged": True, "sha256": file_sha256, "done": done}, f)
def piece_requests(self) -> list[tuple[int, int]]:
return [r for r in self.server.request_ranges if r is not None and r != (0, 0)]
@@ -242,17 +209,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
def download_bundle(self, source: str):
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, source))
def request_download(self) -> dict:
self.make_artifact()
self._bundle.ref = "ref"
self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
self.manager.source_models = {"qcom": [self._bundle], "chestnut": []}
self.manager.params, store = self._make_params_with_store()
store["ModelManager_DownloadRef"] = "ref"
with mock.patch.object(manager_module.Paths, 'model_root', return_value=self.dest):
self.manager._process_download_requests()
return store
def read_path(self) -> bytes:
with open(self.path, 'rb') as f:
return f.read()
@@ -269,7 +225,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
params.get.side_effect = get
params.put.side_effect = put
params.remove.side_effect = lambda key: put(key, None)
return params, store
@@ -283,7 +238,6 @@ class TestManagerDownload(ManagerDownloadTestBase):
assert max(self.reported) <= 99.0, "progress holds below 100 until the hash check"
assert artifact.fileName not in self.manager._download_start_times
assert artifact.downloadProgress.speed > 0, "a running download shows its speed"
assert not os.path.exists(self.sidecar), "the resume state goes once every piece has landed"
def test_speed_counts_bytes_landed_before_the_first_report(self):
self.enterContext(mock.patch.object(manager_module, 'ThreadPoolExecutor', lambda max_workers: InlineExecutor()))
@@ -342,42 +296,9 @@ class TestManagerDownload(ManagerDownloadTestBase):
def test_permanent_failure_is_not_retried(self):
self.server.fail_ranges = {piece_range(1): 404}
self.enterContext(mock.patch.object(manager_module, 'ThreadPoolExecutor', lambda max_workers: InlineExecutor()))
with self.assertRaises(requests.exceptions.HTTPError):
self.process_artifact()
self.download_file()
assert self.piece_requests().count(piece_range(1)) == 1
assert os.path.isfile(self.path)
assert self.resume_state()["done"] == [i for i in range(NUM_PIECES) if i != 1]
def test_failed_transfer_resumes_on_retry(self):
self.server.fail_times = {piece_range(1): manager_module.PIECE_ATTEMPTS} # the first transfer gives up on piece 1
self.server.fail_after_others = True # by then every other piece is saved
self.process_artifact()
assert self.read_path() == WHOLE_BODY
assert self.piece_requests().count(piece_range(1)) == manager_module.PIECE_ATTEMPTS + 1, "the retry fetched piece 1"
others = [i for i in range(NUM_PIECES) if i != 1]
assert all(self.piece_requests().count(piece_range(i)) == 1 for i in others), "the retry skipped every saved piece"
def test_lasting_transient_failure_gives_up(self):
self.server.fail_ranges = {piece_range(1): 503}
with self.assertRaises(requests.exceptions.HTTPError):
self.process_artifact()
assert self.piece_requests().count(piece_range(1)) == manager_module.PIECE_ATTEMPTS * manager_module.DOWNLOAD_ATTEMPTS
def test_cancel_ends_retry_backoff(self):
self.enterContext(mock.patch.object(manager_module, 'RETRY_BACKOFF', 30.0))
self.manager._download_ref = b"ref"
def fail_then_cancel(*args):
self.params.get.return_value = None # the user cancels while the manager waits to retry
raise requests.exceptions.ConnectionError("dropped")
transfer = self.enterContext(mock.patch.object(self.manager, '_download_file', side_effect=fail_then_cancel))
started = time.monotonic()
with self.assertRaises(DownloadCancelled):
self.process_artifact()
assert time.monotonic() - started < 5, "the wait must end at the cancel, not after 30 s"
assert transfer.call_count == 1, "no second transfer after a cancel"
assert self.artifact.downloadProgress.status != custom.ModelManagerSP.DownloadStatus.failed
def test_removing_download_ref_cancels(self):
stall_event = self.stall(3)
@@ -395,51 +316,6 @@ class TestManagerDownload(ManagerDownloadTestBase):
assert self.server.stall_released_by_event is True, "the cancel must have caught the piece mid-transfer"
end = piece_range(3)[1] + 1
assert self.read_path()[end - 100:end] == bytes(100), "the cancelled piece must stop before its last bytes land"
done = self.resume_state()["done"]
assert done and 3 not in done, "a cancel records the finished pieces, and only those, for resume"
def test_resume_skips_complete_pieces(self):
self.write_resume_state(done=[0, 4])
self.download_file()
assert piece_range(0) not in self.piece_requests() and piece_range(4) not in self.piece_requests()
assert self.read_path() == WHOLE_BODY
assert min(self.reported) >= 2 * PIECE_SIZE / len(WHOLE_BODY) * 100 - 1, "progress must not restart below the resumed share"
def test_unusable_resume_state_starts_over(self):
def malformed(content: bytes):
self.write_resume_state(done=[0])
with open(self.sidecar, 'wb') as f:
f.write(content)
layout = {"total": len(WHOLE_BODY), "piece_size": PIECE_SIZE, "ranged": True, "sha256": sha256(WHOLE_BODY)}
cases = {
"other size": lambda: self.write_resume_state(done=[0], total=len(WHOLE_BODY) + 1),
"other bytes": lambda: self.write_resume_state(done=[0], file_sha256="0" * 64),
"truncated file": lambda: self.write_resume_state(done=[0], file_size=PIECE_SIZE),
"not an object": lambda: malformed(b'[]'),
"cut-off json": lambda: malformed(b'{"total": '),
"done not a list": lambda: malformed(json.dumps({**layout, "done": 3}).encode()),
"done not ints": lambda: malformed(json.dumps({**layout, "done": ["0", None]}).encode()),
}
for name, write_state in cases.items():
with self.subTest(name):
self.server.request_ranges = []
write_state()
self.download_file()
assert sorted(self.piece_requests()) == [piece_range(i) for i in range(NUM_PIECES)], "every piece is fetched again"
assert self.read_path() == WHOLE_BODY
def test_pieces_and_resume_state_are_synced_to_disk(self):
real_fsync, synced = os.fsync, []
def record_sync(fd):
st = os.fstat(fd)
synced.append("dir" if stat.S_ISDIR(st.st_mode) else "model" if os.path.exists(self.path) and st.st_ino == os.stat(self.path).st_ino else "sidecar")
real_fsync(fd)
with mock.patch.object(manager_module.os, 'fsync', side_effect=record_sync):
self.download_file()
assert synced.count("model") == NUM_PIECES, f"expected one fsync per piece, got {synced.count('model')}"
assert "sidecar" in synced and "dir" in synced, "sidecar file and directory must be fsync'd"
def test_replaced_download_ref_queues_instead_of_canceling(self):
self.params.get.return_value = b"other-ref"
@@ -469,26 +345,9 @@ class TestProcessArtifact(ManagerDownloadTestBase):
self.server.corrupt = True
with self.assertRaises(ValueError):
self.process_artifact()
assert not os.path.isfile(self.path) and not os.path.exists(self.sidecar), "a corrupt file must not be resumed from"
assert not os.path.isfile(self.path)
assert self.artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.failed
def test_interrupted_download_resumes_without_reverifying(self):
self.write_resume_state(done=[0, 1, 2])
artifact = self.process_artifact()
ds = custom.ModelManagerSP.DownloadStatus
assert self.reported_statuses[0] == ds.downloading, "an in-progress download is not verified first"
assert piece_range(0) not in self.piece_requests()
assert artifact.downloadProgress.status == ds.downloaded
assert any(self.reported_resuming), "a resumed download is flagged resuming until its first progress report"
assert not artifact.downloadProgress.resuming
def test_resume_with_every_piece_done_clears_resuming(self):
self.write_resume_state(done=list(range(NUM_PIECES)))
artifact = self.process_artifact()
assert self.piece_requests() == []
assert artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.downloaded
assert not artifact.downloadProgress.resuming
def test_probe_failure_moves_past_verifying(self):
self.server.fail_ranges = {(0, 0): 404} # the size probe
with self.assertRaises(requests.exceptions.HTTPError):
@@ -515,23 +374,6 @@ class TestProcessArtifact(ManagerDownloadTestBase):
self.download_bundle("qcom")
assert ACTIVE_BUNDLE_KEYS["qcom"] not in store
assert os.path.isfile(self.path), "cancel must not delete a cached model"
assert self.manager.selected_bundle.status != custom.ModelManagerSP.DownloadStatus.failed, "a cancel is not a failure"
def test_failed_download_stays_on_the_row(self):
self.server.fail_ranges = {piece_range(1): 404}
store = self.request_download()
selected = self.manager.selected_bundle
assert selected is not None and selected.status == custom.ModelManagerSP.DownloadStatus.failed, "a failure stays on the row"
assert store["ModelManager_DownloadRef"] is None, "the failed request is released"
def test_finished_download_clears_the_row(self):
self.request_download()
assert self.manager.selected_bundle is None
def test_cancelled_download_clears_the_row(self):
self.pm.send.side_effect = lambda *args: self.manager.params.remove("ModelManager_DownloadRef") # cancel at the first report
self.request_download()
assert self.manager.selected_bundle is None
def test_download_ref_is_released_only_if_unchanged(self):
for current, removed in ((b"new-ref", False), (b"ref", True)):
@@ -126,6 +126,7 @@ def initialize_params(params) -> list[dict[str, Any]]:
keys.extend([
"ToyotaEnforceStockLongitudinal",
"ToyotaStopAndGoHack",
"ToyotaEnhancedBsm",
"ToyotaTSS2Smooth",
])
+15 -3
View File
@@ -13,6 +13,7 @@ from opendbc.car.subaru.values import CAR as SUBARU_CAR, SubaruFlags
from opendbc.car.toyota.values import CAR as TOYOTA_CAR, ToyotaFlags
from opendbc.sunnypilot.car.interfaces import has_bsm
from opendbc.sunnypilot.car.tesla.values import TeslaFlagsSP
from opendbc.sunnypilot.car.toyota.values import ToyotaFlagsSP
from openpilot.common.params import Params
from openpilot.common.swaglog import cloudlog
from openpilot.common.hardware import HARDWARE
@@ -44,6 +45,7 @@ CAPABILITY_FIELDS = (
"device_type",
"subaru_has_sng",
"hyundai_alpha_long_available",
"toyota_enhanced_bsm_available",
"toyota_tss2",
"custom_model_active",
)
@@ -68,6 +70,7 @@ CAPABILITY_LABELS: dict[str, str] = {
"device_type": "Device type",
"subaru_has_sng": "Subaru Stop-and-Go available",
"hyundai_alpha_long_available": "Hyundai Alpha Longitudinal available",
"toyota_enhanced_bsm_available": "Toyota Enhanced BSM available",
"toyota_tss2": "Toyota TSS2 platform",
"custom_model_active": "Non-Default model selected",
}
@@ -89,6 +92,7 @@ CAR_CAPABILITY_FIELDS = (
"subaru_has_sng",
"hyundai_alpha_long_available",
"toyota_tss2",
"toyota_enhanced_bsm_available",
)
# Explicit defaults for non-boolean capability fields
@@ -104,7 +108,7 @@ def _bundle_field(bundle: dict | None, key: str) -> str:
return bundle.get(key, "") if isinstance(bundle, dict) else ""
def _resolve_brand_capabilities(caps: dict, bundle_platform: str, CP) -> None:
def _resolve_brand_capabilities(caps: dict, bundle_platform: str, CP, CP_SP=None) -> None:
"""Set brand-specific capabilities from bundle platform or CarParams fallback.
Bundle (manual car selection) is a pre-fingerprint approximation.
@@ -138,11 +142,19 @@ def _resolve_brand_capabilities(caps: dict, bundle_platform: str, CP) -> None:
elif brand == "toyota":
if bundle_platform:
try:
caps["toyota_tss2"] = bool(TOYOTA_CAR[bundle_platform].config.flags & ToyotaFlags.TSS2)
flags = TOYOTA_CAR[bundle_platform].config.flags
caps["toyota_tss2"] = bool(flags & ToyotaFlags.TSS2)
caps["toyota_enhanced_bsm_available"] = bool(flags & ToyotaFlags.TSS2) and not bool(flags & ToyotaFlags.SECOC)
except KeyError:
cloudlog.exception(f"capabilities: unknown toyota platform {bundle_platform!r}")
elif CP is not None:
caps["toyota_tss2"] = bool(CP.flags & ToyotaFlags.TSS2)
# same gate as _initialize_toyota: openpilot longitudinal, not SecOC, and either TSS2 or a
# smartDSU. The bundle path can't see 0x2FF or longitudinal, so it stays TSS2-only.
sdsu_bsm = CP_SP is not None and bool(CP_SP.flags & ToyotaFlagsSP.SMART_DSU)
caps["toyota_enhanced_bsm_available"] = (bool(CP.openpilotLongitudinalControl) and
not bool(CP.flags & ToyotaFlags.SECOC) and
(bool(CP.flags & ToyotaFlags.TSS2) or sdsu_bsm))
def generate_capabilities(params: Params | None = None) -> dict:
@@ -217,7 +229,7 @@ def generate_capabilities(params: Params | None = None) -> dict:
if CP is not None:
caps["has_bsm"] = has_bsm(CP, CP_SP)
_resolve_brand_capabilities(caps, bundle_platform, CP)
_resolve_brand_capabilities(caps, bundle_platform, CP, CP_SP)
return caps
@@ -156,7 +156,7 @@ The tables below describe the **compiled** `settings_ui.json` schema — what th
**Enablement rules**: Grayed out (disabled) when rules fail. Frontend shows a contextual badge explaining why.
**Capability fields** (referenced in rules): `has_longitudinal_control`, `has_icbm`, `icbm_available`, `torque_allowed`, `brand`, `pcm_cruise`, `alpha_long_available`, `steer_control_type`, `has_bsm`, `is_release`, `is_sp_release`, `is_development`, `tesla_has_vehicle_bus`, `has_stop_and_go`, `stock_longitudinal`
**Capability fields** (referenced in rules): `has_longitudinal_control`, `has_icbm`, `icbm_available`, `torque_allowed`, `brand`, `pcm_cruise`, `alpha_long_available`, `steer_control_type`, `has_bsm`, `is_release`, `is_sp_release`, `is_development`, `tesla_has_vehicle_bus`, `has_stop_and_go`, `stock_longitudinal`, `toyota_enhanced_bsm_available`
---
@@ -2361,6 +2361,28 @@
}
]
},
{
"key": "ToyotaEnhancedBsm",
"widget": "toggle",
"needs_onroad_cycle": true,
"title": "Enhanced Blind Spot Monitoring (Alpha)",
"description": "sunnypilot will poll the blind spot monitor diagnostics directly from the radar. Use this on vehicles whose blind spot monitors do not report blind spot status over the standard CAN signals. This is an alpha feature. Use at your own risk.",
"enablement": [
{
"type": "not_engaged"
},
{
"type": "capability",
"field": "toyota_enhanced_bsm_available",
"equals": true
},
{
"type": "capability",
"field": "stock_longitudinal",
"equals": false
}
]
},
{
"key": "ToyotaTSS2Smooth",
"widget": "toggle",
@@ -101,6 +101,21 @@ sections:
- type: param
key: ToyotaEnforceStockLongitudinal
equals: false
- key: ToyotaEnhancedBsm
widget: toggle
needs_onroad_cycle: true
title: Enhanced Blind Spot Monitoring (Alpha)
description: sunnypilot will poll the blind spot monitor diagnostics directly from the radar. Use this on vehicles whose
blind spot monitors do not report blind spot status over the standard CAN signals. This is an alpha feature. Use at
your own risk.
enablement:
- $ref: '#/macros/not_engaged'
- type: capability
field: toyota_enhanced_bsm_available
equals: true
- type: capability
field: stock_longitudinal
equals: false
- key: ToyotaTSS2Smooth
widget: toggle
needs_onroad_cycle: true
@@ -70,12 +70,18 @@ class TestOpaquePerBrandFlags(OpenpilotTestCase):
def test_hyundai_alpha_long_available_field_present(self):
assert "hyundai_alpha_long_available" in CAPABILITY_FIELDS
def test_toyota_enhanced_bsm_available_field_present(self):
assert "toyota_enhanced_bsm_available" in CAPABILITY_FIELDS
def test_subaru_has_sng_default_false(self, caps):
assert caps["subaru_has_sng"] is False
def test_hyundai_alpha_long_available_default_false(self, caps):
assert caps["hyundai_alpha_long_available"] is False
def test_toyota_enhanced_bsm_available_default_false(self, caps):
assert caps["toyota_enhanced_bsm_available"] is False
def test_toyota_tss2_field_present(self):
assert "toyota_tss2" in CAPABILITY_FIELDS
@@ -207,6 +207,7 @@ class TestNotEngagedReplacement(OpenpilotTestCase):
"AlphaLongitudinalEnabled",
"ToyotaEnforceStockLongitudinal",
"ToyotaStopAndGoHack",
"ToyotaEnhancedBsm",
"ToyotaTSS2Smooth",
], names=["key"])
def test_offroad_only_replaced_with_not_engaged(self, schema, key):
@@ -286,6 +286,7 @@ class TestKnownVehicleSettings(OpenpilotTestCase):
keys = {i["key"] for i in _brand_items(schema["vehicle_settings"].get("toyota"))}
assert "ToyotaEnforceStockLongitudinal" in keys
assert "ToyotaStopAndGoHack" in keys
assert "ToyotaEnhancedBsm" in keys
assert "ToyotaTSS2Smooth" in keys
def test_tesla_has_coop_steering(self, schema):