From 29f498d77ebd63443476651da6a0404cc0969e22 Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 30 Oct 2026 17:39:24 -0700 Subject: [PATCH 01/38] StarPort: Unofficial dongle option, v4 head unit version reply fallback, patience for not-ready cars --- .../starpilot/galaxy/tests/test_starport.mjs | 9 +- openpilot/starpilot/galaxy/web/js/starport.js | 6 +- .../starpilot/system/starport/bootstrap.py | 86 ++++++++++++++- .../system/starport/bugsink_diagnosis.py | 5 + .../starpilot/system/starport/car_profiles.py | 4 +- .../starpilot/system/starport/supervisor.py | 9 +- .../tests/test_bootstrap_receivers.py | 102 +++++++++++++++++- .../starport/tests/test_car_profiles.py | 10 +- 8 files changed, 214 insertions(+), 17 deletions(-) diff --git a/openpilot/starpilot/galaxy/tests/test_starport.mjs b/openpilot/starpilot/galaxy/tests/test_starport.mjs index 275679293b..f2ef23f936 100644 --- a/openpilot/starpilot/galaxy/tests/test_starport.mjs +++ b/openpilot/starpilot/galaxy/tests/test_starport.mjs @@ -524,15 +524,16 @@ assert.equal(dongle.adapterRow.value, 'Choose one', 'no default: the owner picks assert.equal(settingControl(dongle.adapterRow), 'select', 'the initial adapter prompt must open a picker') assert.match(dongle.pairingReason, /adapter model/) dongle.runtime = { car_type: 'adapter', car_profile: 'default', car_profile_source: 'override' } -assert.equal(dongle.adapterRow.value, 'Other') -assert.deepEqual(dongle.adapterRow.choices, ['AAWireless', 'Other']) +assert.equal(dongle.adapterRow.value, 'Motorola MA1') +assert.deepEqual(dongle.adapterRow.choices, ['AAWireless', 'Motorola MA1', 'Unofficial dongle']) dongle.runtime = { car_type: 'adapter', car_profile: 'aawireless', car_profile_source: 'override' } assert.equal(dongle.adapterRow.value, 'AAWireless') const dongleCalls = [] dongle.control = (action, extra) => dongleCalls.push([action, extra]) -dongle.saveAdapter(0, 'Other') +dongle.saveAdapter(0, 'Motorola MA1') dongle.saveAdapter(0, 'AAWireless') -assert.deepEqual(dongleCalls, [['car_profile', { profile: 'default' }], ['car_profile', { profile: 'aawireless' }]]) +dongle.saveAdapter(0, 'Unofficial dongle') +assert.deepEqual(dongleCalls, [['car_profile', { profile: 'default' }], ['car_profile', { profile: 'aawireless' }], ['car_profile', { profile: 'unofficial' }]]) assert.match(StarPortPage.template, /adapterSetup" :row="adapterRow"/) assert.match(StarPortPage.template, /What’s in your car\?|carTypeRow/) assert.match(StarPortPage.template, /runtime\?\.pairing_note/) diff --git a/openpilot/starpilot/galaxy/web/js/starport.js b/openpilot/starpilot/galaxy/web/js/starport.js index 7e45891075..37a0f25f3b 100644 --- a/openpilot/starpilot/galaxy/web/js/starport.js +++ b/openpilot/starpilot/galaxy/web/js/starport.js @@ -44,7 +44,7 @@ export function uploadPackage(path, options, progress, makeRequest = () => new X } const ADDRESS = /^(?:[0-9A-F]{2}:){5}[0-9A-F]{2}$/ -const ADAPTER_MODELS = { aawireless: "AAWireless", default: "Other" } +const ADAPTER_MODELS = { aawireless: "AAWireless", default: "Motorola MA1", unofficial: "Unofficial dongle" } const HEAD_UNIT_APPS = { headunitpad: "HeadunitPad", openheadunit: "Open Headunit" } const HOTSPOT_LINK = `Galaxy Wi-Fi Hotspot` const WIRED_STATUS = `
@@ -564,9 +564,9 @@ export const StarPortPage = { }, adapterRow() { const chosen = this.runtime?.car_profile_source === "override" - return { label: "Adapter", value: !chosen ? "Choose one" : this.runtime.car_profile === "aawireless" ? ADAPTER_MODELS.aawireless : ADAPTER_MODELS.default, + return { label: "Adapter", value: !chosen ? "Choose one" : ADAPTER_MODELS[this.runtime.car_profile] || ADAPTER_MODELS.default, choices: [...(!chosen ? ["Choose one"] : []), ...Object.values(ADAPTER_MODELS)], available: true, action: true, - reason: "AAWireless gets video and car-data settings tuned for that dongle. Other adapters use the regular pairing." } + reason: "AAWireless gets video and car-data settings tuned for that dongle. Motorola MA1 uses the regular pairing. Pick Unofficial dongle for other wireless dongles that will not connect; it tries alternate handshakes and logs more." } }, appDisabled() { return !this.setup?.enabled || !this.setup.parked || !!this.pairing?.active || this.linked diff --git a/openpilot/starpilot/system/starport/bootstrap.py b/openpilot/starpilot/system/starport/bootstrap.py index 4ae9c37513..bd00289ec1 100644 --- a/openpilot/starpilot/system/starport/bootstrap.py +++ b/openpilot/starpilot/system/starport/bootstrap.py @@ -30,6 +30,7 @@ import re import struct import threading import time +import uuid from collections.abc import Callable from dataclasses import dataclass, field as dataclass_field @@ -53,6 +54,11 @@ NAMES = {1: "WifiStartRequest", 2: "WifiInfoRequest", 3: "WifiInfoResponse", 4: STATUS_SUCCESS = 0 STATUS_NETWORK_UNAVAILABLE = -1 +# Fallback for a v4+ head unit that goes silent after our usual reply: a phone-shaped WifiVersionResponse, where 1 is +# success (open-android-auto WifiVersionStatus) and the channel type and session info are filled in. +MODERN_MAJOR = 4 +MODERN_VERSION_SUCCESS = 1 +CHANNEL_5GHZ, CHANNEL_24GHZ, CHANNEL_DUAL = 1, 2, 3 SECURITY_OPEN = 1 SECURITY_NAMES = {0: "unknown", 1: "open", 2: "wep64", 3: "wep128", 4: "wpa", 8: "wpa2", 12: "wpa/wpa2", @@ -61,9 +67,19 @@ SECURITY_NAMES = {0: "unknown", 1: "open", 2: "wep64", 3: "wep128", 4: "wpa", 8: MAX_FRAME = 4096 MAX_FRAMES = 64 INITIAL_KICK_SECONDS = 2.5 +# A car that keeps answering "not ready" gets up to this many stage timeouts before the setup is abandoned. +REFUSAL_PATIENCE = 3 # Phone-side WifiPing while joining the receiver's Wi-Fi, so an impatient receiver (a wireless adapter) keeps the # idle RFCOMM link. Sent only to a receiver that pings us first: one that never pings may not expect them. JOIN_PING_INTERVAL = 2.0 +# Opt-in handshake variations for dongles that stall the standard exchange; "" is the standard behavior. +EXPERIMENTS = { + "start_response_first": ("Answer start request first", {"start_response_first"}), + "version_endpoint": ("Use version endpoint", {"version_endpoint"}), + "all": ("All of these", {"start_response_first", "version_endpoint"}), +} +# Frames that carry credentials or the vehicle identifier are never dumped as hex. +HEX_LOG_SKIP = (WIFI_INFO_RESPONSE, WIFI_SETUP_INFO, WIFI_VERSION_REQUEST) MAC_RE = re.compile(r"^(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}$") @@ -165,7 +181,25 @@ def parse_setup_info(payload: bytes) -> tuple[Endpoint | None, WifiCredentials | return endpoint, credentials -def parse_version_request(payload: bytes) -> tuple[int, int, Endpoint | None, dict]: +def channel_type(request: bytes) -> int: + """The WifiChannelType to select: the car's stated preference, else what its channel list allows.""" + fields = parse_fields(request) + preference = one(fields, 3) + if isinstance(preference, int) and preference in (CHANNEL_5GHZ, CHANNEL_24GHZ, CHANNEL_DUAL): + return preference + channels = [value for value in fields.get(4, []) if isinstance(value, int) and value > 0] + if channels and all(value >= 5000 for value in channels): + return CHANNEL_5GHZ + if channels and all(value < 5000 for value in channels): + return CHANNEL_24GHZ + return CHANNEL_DUAL + + +def valid_experiment(value: str) -> bool: + return value == "" or value in EXPERIMENTS + + +def parse_version_request(payload: bytes, endpoint_fields: tuple[int, ...] = ()) -> tuple[int, int, Endpoint | None, dict]: """Major/minor, an optional projection endpoint, and head-unit identity strings.""" fields = parse_fields(payload) major, minor = int(one(fields, 1, 1) or 1), int(one(fields, 2, 0) or 0) @@ -185,6 +219,13 @@ def parse_version_request(payload: bytes) -> tuple[int, int, Endpoint | None, di labels = {1: "car_make", 2: "car_model", 3: "car_year", 5: "head_unit_make", 6: "head_unit_model", 7: "head_unit_software_build", 8: "head_unit_software_version"} head_unit.update({label: text(info, key) for key, label in labels.items() if text(info, key)}) + for number in endpoint_fields: + for value in fields.get(number, []): + if isinstance(value, bytes): + try: + endpoint = parse_endpoint(value) or endpoint + except ValueError: + continue return major, minor, endpoint, head_unit @@ -224,8 +265,15 @@ class WirelessBootstrap: def __init__(self, sock, log: Callable[..., None], *, device_serial: str = "starpilot", version_status: int = STATUS_SUCCESS, stage_timeout: float = 20.0, start_request_delay: float = 5.0, - initial_kick_delay: float = INITIAL_KICK_SECONDS, join_ping_interval: float = JOIN_PING_INTERVAL): + initial_kick_delay: float = INITIAL_KICK_SECONDS, join_ping_interval: float = JOIN_PING_INTERVAL, + experiment: str = "", modern_reply: bool = False): self.sock = sock + self.experiment = experiment if valid_experiment(experiment) else "" + self.lifetime_id = uuid.uuid4().hex + self.modern_reply = modern_reply + self.version_major = 0 + self.car_started = False + self.flags = EXPERIMENTS[self.experiment][1] if self.experiment else set() self.initial_kick_delay = initial_kick_delay self.join_ping_interval = join_ping_interval self.peer_uses_pings = False @@ -251,6 +299,11 @@ class WirelessBootstrap: self.sock.sendall(encode_frame(message_id, payload)) if message_id not in (WIFI_PING_REQUEST, WIFI_PING_RESPONSE): self.log("bootstrap_tx", message=NAMES.get(message_id, message_id), bytes=len(payload)) + self._log_hex("tx", message_id, payload) + + def _log_hex(self, direction: str, message_id: int, payload: bytes) -> None: + if self.experiment and message_id not in HEX_LOG_SKIP: + self.log("bootstrap_frame_hex", direction=direction, message=NAMES.get(message_id, message_id), hex=payload.hex()) def next_frame(self, timeout: float) -> tuple[int, bytes]: deadline = time.monotonic() + timeout @@ -277,8 +330,25 @@ class WirelessBootstrap: if self.frames_seen > MAX_FRAMES: raise BootstrapError(self.stage, "too many bootstrap frames") self.log("bootstrap_rx", message=NAMES.get(message_id, message_id), bytes=len(payload)) + self._log_hex("rx", message_id, payload) return message_id, payload + def _version_response(self, major: int, minor: int, request: bytes) -> bytes: + self.version_major = major + if major < MODERN_MAJOR or not self.modern_reply: + return field(1, major) + field(2, minor) + field(3, self.device_serial) + field(4, self.version_status) + selected = channel_type(request) + session = field(1, self.device_serial) + field(2, self.lifetime_id) + reply = (field(1, major) + field(2, minor) + field(3, self.device_serial) + field(4, MODERN_VERSION_SUCCESS) + + field(5, selected) + field(6, session)) + self.log("bootstrap_version_reply", style="modern", channel_type=selected, hex=reply.hex()) + return reply + + def silent_after_modern_version(self, error: BaseException) -> bool: + """A v4+ head unit took our version reply but never started; the other reply style may suit it.""" + return (isinstance(error, BootstrapTimeout) and error.stage == "wifi_start" and self.version_major >= MODERN_MAJOR + and not self.car_started) + def service(self, message_id: int, payload: bytes) -> bool: """Handle stage-independent frames; returns True when consumed.""" if message_id == WIFI_PING_REQUEST: @@ -306,6 +376,7 @@ class WirelessBootstrap: # exchange if the car has not started it; newer head units (2025 Honda) wait for it. start_request_at: float | None = None deadline = time.monotonic() + self.stage_timeout + latest_deadline = time.monotonic() + REFUSAL_PATIENCE * self.stage_timeout initial_kick_at: float | None = time.monotonic() + self.initial_kick_delay hint_ready_at: float | None = None while endpoint is None: @@ -337,23 +408,27 @@ class WirelessBootstrap: continue initial_kick_at = None # setup messages drive the exchange; pings alone do not if message_id == WIFI_VERSION_REQUEST: - major, minor, version_endpoint, info = parse_version_request(payload) + major, minor, version_endpoint, info = parse_version_request( + payload, (6,) if "version_endpoint" in self.flags else ()) version, head_unit = (major, minor), {**head_unit, **info} self.log("bootstrap_version", major=major, minor=minor, head_unit=info, endpoint=version_endpoint.__dict__ if version_endpoint else None, message=describe_version_request(payload)) - self.send(WIFI_VERSION_RESPONSE, field(1, major) + field(2, minor) + field(3, self.device_serial) + - field(4, self.version_status)) + self.send(WIFI_VERSION_RESPONSE, self._version_response(major, minor, payload)) hinted = version_endpoint or hinted if hinted is not None and hint_ready_at is None: hint_ready_at = time.monotonic() + 3.0 if start_request_at is None: start_request_at = time.monotonic() + self.start_request_delay elif message_id == WIFI_START_REQUEST: + self.car_started = True parsed = parse_endpoint(payload) if parsed is None: raise BootstrapError(self.stage, "WifiStartRequest has no valid IPv4 endpoint") endpoint = parsed + if "start_response_first" in self.flags: + self.send(WIFI_START_RESPONSE, field(3, STATUS_SUCCESS)) elif message_id == WIFI_START_RESPONSE: + self.car_started = True # The car's answer to our WifiStartRequest (2025 Honda): ip=1, port=2, status=3 once it is # ready, or a negative status alone before then (it may follow up on its own; ask again anyway). status = signed(one(parse_fields(payload), 3, STATUS_SUCCESS)) @@ -362,6 +437,7 @@ class WirelessBootstrap: endpoint = parsed else: self.log("bootstrap_start_refused", status=status, endpoint=parsed is not None) + deadline = min(latest_deadline, max(deadline, time.monotonic() + self.stage_timeout)) # a car that answers is not gone start_request_at = time.monotonic() + self.start_request_delay elif message_id == WIFI_SETUP_INFO: setup_endpoint, setup_credentials = parse_setup_info(payload) diff --git a/openpilot/starpilot/system/starport/bugsink_diagnosis.py b/openpilot/starpilot/system/starport/bugsink_diagnosis.py index 69f1f172c5..058b9505cf 100644 --- a/openpilot/starpilot/system/starport/bugsink_diagnosis.py +++ b/openpilot/starpilot/system/starport/bugsink_diagnosis.py @@ -125,6 +125,11 @@ def diagnosis(session): explanation = "Authentication did not complete. A timeout alone does not prove a bad certificate." actions = ["Check whether TLS completed and whether the head unit replied to authentication.", "Retry with the head unit fully awake; compare with any successful attempt in this session."] + elif stage in ("wifi_start", "wifi_info"): + category, confidence = "head_unit_bootstrap_stalled", "observed" + explanation = "Bluetooth connected and the version exchange ran, but the head unit stopped answering the Wi-Fi setup." + actions = ["Read the attached handshake trace for the last messages the head unit sent and the ones it ignored.", + "Compare the head unit make, model and protocol version with other reports before changing timings."] elif stage in ("connecting_bluetooth", "rfcomm", "bootstrap"): category, confidence = "head_unit_unavailable", "suspected" explanation = "The Bluetooth/bootstrap exchange did not complete; the head unit may still be waking or already off." diff --git a/openpilot/starpilot/system/starport/car_profiles.py b/openpilot/starpilot/system/starport/car_profiles.py index 52ac24a729..faeda26273 100644 --- a/openpilot/starpilot/system/starport/car_profiles.py +++ b/openpilot/starpilot/system/starport/car_profiles.py @@ -1,13 +1,15 @@ """Android Auto tuning selected from the saved head-unit fingerprint or an override.""" import re -DEFAULTS = {"connect_delay_s": 0.0, "max_window": 2, "phone_services": False, "phone_frames": False} +DEFAULTS = {"connect_delay_s": 0.0, "max_window": 2, "phone_services": False, "phone_frames": False, + "bootstrap_experiments": False} FINGERPRINT_KEYS = ("head_unit_make", "head_unit_model", "car_make", "car_model", "car_year") RULES: list[dict] = [ {"id": "aawireless", "label": "AAWireless", "match": {}, "tuning": {"max_window": 30, "phone_services": True, "phone_frames": True}}, + {"id": "unofficial", "label": "Unofficial dongle", "match": {}, "tuning": {"bootstrap_experiments": True}}, ] diff --git a/openpilot/starpilot/system/starport/supervisor.py b/openpilot/starpilot/system/starport/supervisor.py index 0c161d4e2c..246202f3ab 100644 --- a/openpilot/starpilot/system/starport/supervisor.py +++ b/openpilot/starpilot/system/starport/supervisor.py @@ -353,6 +353,7 @@ class Supervisor: self._pairing_seen: tuple = () self._car_seen_at = -CAR_LINK_HOLD self._hfp_link = threading.Event() # set when the chosen car opens the hands-free link + self._modern_reply: dict[str, bool] = {} # car address -> answer its v4+ version request like a current phone self.auto = AutoConnectPolicy() self._companion_retry_at = 0.0 self.log = EventLog() @@ -1234,16 +1235,22 @@ class Supervisor: channel, source = self._discover_channel(address), "sdp" self.log("rfcomm_channel", channel=channel, source=source) + boot = None try: self._stage("rfcomm", f"channel {channel}") rfcomm = self._track(bt_sockets.connect_rfcomm(address, channel)) boot = WirelessBootstrap(rfcomm, self._bootstrap_log, device_serial=config["device_name"], - version_status=int(config.get("version_status", 0))) + version_status=int(config.get("version_status", 0)), + experiment="all" if car_profiles.tuning(config)["bootstrap_experiments"] else "", + modern_reply=self._modern_reply.get(address, False)) self._set(state="wifi_start") result = boot.run(lambda credentials: lease.acquire(credentials, cancelled=boot.join_is_cancelled), cancelled=self._stop.is_set) except Exception as error: if _stray_link(error): raise StrayLinkError(f"Bluetooth link to the car is stuck: {error}") from error # says nothing about the cached channel + if boot is not None and boot.silent_after_modern_version(error): + self._modern_reply[address] = not self._modern_reply.get(address, False) # try the other version reply next + self.log("bootstrap_reply_switched", modern=self._modern_reply[address]) if source == "cache" and not self._stop.is_set() and self._status["last_stage"] in ("rfcomm", "wifi_start") and car_never_answered(error): self._remember_channel(address, None) # the car never answered there; ask it over SDP next time self.log("rfcomm_cache_dropped", channel=channel) diff --git a/openpilot/starpilot/system/starport/tests/test_bootstrap_receivers.py b/openpilot/starpilot/system/starport/tests/test_bootstrap_receivers.py index c61051d162..4c41072a91 100644 --- a/openpilot/starpilot/system/starport/tests/test_bootstrap_receivers.py +++ b/openpilot/starpilot/system/starport/tests/test_bootstrap_receivers.py @@ -14,9 +14,10 @@ INFO = field(1, "AAWireless") + field(2, "secret-key") + field(3, "AA:BB:CC:DD:E SETUP_INFO = field(4, ENDPOINT) + field(5, field(1, "AAWireless") + field(3, "secret-key") + field(4, 8)) -def scripted(frames, *, ping_interval=bs.JOIN_PING_INTERVAL): +def scripted(frames, *, ping_interval=bs.JOIN_PING_INTERVAL, experiment=""): """A bootstrap whose car sends `frames` in order, then stays silent; records what the comma sent.""" - boot = bs.WirelessBootstrap(None, lambda *a, **k: None, stage_timeout=2, join_ping_interval=ping_interval) + boot = bs.WirelessBootstrap(None, lambda *a, **k: None, stage_timeout=2, join_ping_interval=ping_interval, + experiment=experiment) sent = [] boot.send = Mock(side_effect=lambda message_id, payload=b"": sent.append(message_id)) queue = list(frames) @@ -100,3 +101,100 @@ def test_stop_during_the_join_is_not_reported_as_connected(): with pytest.raises(bs.BootstrapError, match="cancelled"): boot.run(join, cancelled=stop.is_set) assert bs.WIFI_CONNECT_STATUS not in sent + + +VERSION_F6 = (field(1, 1) + field(2, 7) + field(4, 5180) + field(5, field(1, "Link") + field(5, "CarLetter")) + + field(6, field(1, "192.168.43.158") + field(2, 40707))) + + +def test_version_endpoint_in_field_six_is_read_only_when_asked(): + assert bs.parse_version_request(VERSION_F6)[2] is None + assert bs.parse_version_request(VERSION_F6, (6,))[2] == bs.Endpoint("192.168.43.158", 40707) + + +def test_start_request_is_answered_early_only_by_the_experiment(): + for experiment, early in (("", False), ("start_response_first", True), ("all", True)): + boot, sent = scripted([(bs.WIFI_START_REQUEST, ENDPOINT), (bs.WIFI_INFO_RESPONSE, INFO)], experiment=experiment) + boot.run(slow_join(0, [])) + assert (sent[:2] == [bs.WIFI_START_RESPONSE, bs.WIFI_INFO_REQUEST]) is early + + +def test_experiment_logs_frames_but_never_credentials(): + events = [] + boot = bs.WirelessBootstrap(None, lambda name, **v: events.append((name, v)), experiment="all") + boot.sock = Mock() + boot.send(bs.WIFI_START_RESPONSE, b"\x18\x00") + boot.queue = [(bs.WIFI_INFO_RESPONSE, INFO), (bs.WIFI_START_REQUEST, ENDPOINT)] + boot.next_frame(1) + boot.next_frame(1) + dumped = [v for name, v in events if name == "bootstrap_frame_hex"] + assert [d["message"] for d in dumped] == ["WifiStartResponse", "WifiStartRequest"] + assert INFO.hex() not in str(events) + assert bs.WirelessBootstrap(None, lambda *a, **k: events.append(a), experiment="bogus").experiment == "" + + +def test_a_car_that_keeps_refusing_gets_more_time_but_not_forever(): + refusal = field(3, 1) + boot, sent = scripted([(bs.WIFI_START_RESPONSE, refusal)] * 3 + [(bs.WIFI_START_REQUEST, ENDPOINT), (bs.WIFI_INFO_RESPONSE, INFO)]) + boot.stage_timeout = 0.2 + boot.start_request_delay = 0.01 + assert boot.run(slow_join(0, [])).endpoint.port == 5288 + silent, _ = scripted([]) + silent.stage_timeout = 0.2 + idle, late = silent.next_frame, [True] + + def refuse_late(timeout): + if late: + late.clear() + time.sleep(0.15) # just before the original deadline + return bs.WIFI_START_RESPONSE, refusal + return idle(timeout) + + silent.next_frame = refuse_late + started = time.monotonic() + with pytest.raises(bs.BootstrapTimeout): + silent.run(slow_join(0, [])) + elapsed = time.monotonic() - started + assert 0.3 < elapsed < 0.2 * bs.REFUSAL_PATIENCE + 0.5 + + +KIA_V41 = field(1, 4) + field(2, 1) + field(3, 0) + field(4, 5745) + field(5, field(1, "KIA") + field(5, "MOBIS")) +DHU_V42 = field(1, 4) + field(2, 2) + field(3, 2) + field(5, field(1, "Google") + field(6, "Desktop Head Unit")) +FORD_V16 = field(1, 1) + field(2, 6) + field(3, 0) + field(4, 5180) + field(4, 5745) + field(5, field(1, "Ford")) + + +def version_reply(request, modern=True): + boot, _ = scripted([]) + boot.modern_reply = modern + sent = [] + boot.send = Mock(side_effect=lambda message_id, payload=b"": sent.append(payload)) + boot.next_frame = Mock(side_effect=[(bs.WIFI_VERSION_REQUEST, request), bs.BootstrapError("wifi_start", "stop")]) + with pytest.raises(bs.BootstrapError): + boot.run(slow_join(0, [])) + return bs.parse_fields(sent[0]) + + +def test_version_four_head_units_get_a_phone_shaped_version_reply(): + kia, dhu = version_reply(KIA_V41), version_reply(DHU_V42) + assert kia[1] == [4] and kia[2] == [1] and kia[4] == [1] and kia[5] == [bs.CHANNEL_5GHZ] + assert dhu[4] == [1] and dhu[5] == [bs.CHANNEL_24GHZ] + session = bs.parse_fields(kia[6][0]) + assert bs.text(session, 1) == "starpilot" and len(bs.text(session, 2)) == 32 + + +def test_older_head_units_and_the_first_try_keep_the_reply_that_works_today(): + for reply in (version_reply(FORD_V16), version_reply(KIA_V41, modern=False)): + assert reply[4] == [0] and 5 not in reply and 6 not in reply + + +def test_only_a_silent_v4_head_unit_asks_for_the_other_reply(): + timeout = bs.BootstrapTimeout("wifi_start", "head unit did not answer in time") + boot, _ = scripted([]) + boot.version_major = 4 + assert boot.silent_after_modern_version(timeout) + boot.car_started = True + assert not boot.silent_after_modern_version(timeout) + boot.car_started, boot.version_major = False, 1 + assert not boot.silent_after_modern_version(timeout) + boot.version_major = 4 + assert not boot.silent_after_modern_version(bs.BootstrapTimeout("wifi_info", "head unit did not answer in time")) diff --git a/openpilot/starpilot/system/starport/tests/test_car_profiles.py b/openpilot/starpilot/system/starport/tests/test_car_profiles.py index 70bbb793b9..68183cab20 100644 --- a/openpilot/starpilot/system/starport/tests/test_car_profiles.py +++ b/openpilot/starpilot/system/starport/tests/test_car_profiles.py @@ -39,7 +39,8 @@ def config(**values): def test_aawireless_tuning_applies_only_when_chosen(): chosen = config(car_profile_override="aawireless", receiver_name="AAWireless-4F2A") assert car_profiles.resolve(chosen) == ("aawireless", "override") - assert car_profiles.tuning(chosen) == {"connect_delay_s": 0.0, "max_window": 30, "phone_services": True, "phone_frames": True} + assert car_profiles.tuning(chosen) == {"connect_delay_s": 0.0, "max_window": 30, "phone_services": True, "phone_frames": True, + "bootstrap_experiments": False} assert car_profiles.summary(chosen)["car_profile_label"] == "AAWireless" for other in (config(receiver_name="AAWireless-4F2A"), config(car_profile_override="default")): assert car_profiles.tuning(other) == car_profiles.DEFAULTS @@ -221,3 +222,10 @@ def test_a_new_adapter_setup_is_not_migrated(tmp_path): path = tmp_path / "config.json" path.write_text(json.dumps({"config_version": identity_store.CONFIG_VERSION, "car_type": "adapter"})) assert identity_store.load_config(path)["car_profile_override"] == "" + + +def test_unofficial_dongle_turns_on_the_handshake_experiments_only_when_chosen(): + assert car_profiles.valid_override("unofficial") + assert car_profiles.tuning(config(car_profile_override="unofficial"))["bootstrap_experiments"] is True + for other in (config(), config(car_profile_override="default"), config(car_profile_override="aawireless")): + assert car_profiles.tuning(other)["bootstrap_experiments"] is False From b8f3bcebcbdde0b06428c3096c964d5baafe83ce Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 30 Oct 2026 18:01:28 -0700 Subject: [PATCH 02/38] StarPort: Remember which version reply each v4 head unit started with --- .../starpilot/system/starport/identity.py | 3 ++ .../starpilot/system/starport/supervisor.py | 29 +++++++++++++++---- .../system/starport/tests/test_forget_car.py | 10 +++++++ .../starport/tests/test_identity_migration.py | 8 +++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/openpilot/starpilot/system/starport/identity.py b/openpilot/starpilot/system/starport/identity.py index 425ea86308..6859db0cf3 100644 --- a/openpilot/starpilot/system/starport/identity.py +++ b/openpilot/starpilot/system/starport/identity.py @@ -146,6 +146,7 @@ DEFAULT_CONFIG = { "phone_class": True, # while pairing/connecting, present as a phone: HFP gateway + smartphone Class of Device "auto_connect": True, # start projection on its own when the chosen car is on (onroad, or it reaches the comma) "rfcomm_cache": {}, # car address -> Android Auto RFCOMM channel learned over SDP, to skip discovery next time + "modern_reply_cache": {}, # car address -> True when its v4+ head unit only started after the phone-style version reply "car_type": "", # wireless setup shown in Galaxy: "builtin" or "adapter"; "" = work it out from the saved companion car "car_profile_override": "", # "" = pick by fingerprint (a wireless adapter must choose); "default" = standard; otherwise a car_profiles id "car_fingerprint": {}, # what the chosen car's head unit reported last time, used to pick a profile before it connects @@ -193,6 +194,8 @@ def load_config(path: Path | None = None) -> dict: config["render_profile_kb"] = max(16, min(4096, int(config["render_profile_kb"]))) config["rfcomm_cache"] = {str(address).upper(): channel for address, channel in config["rfcomm_cache"].items() if type(channel) is int and 1 <= channel <= 30} + config["modern_reply_cache"] = {str(address).upper(): True for address, modern in config["modern_reply_cache"].items() + if modern is True} return config diff --git a/openpilot/starpilot/system/starport/supervisor.py b/openpilot/starpilot/system/starport/supervisor.py index 246202f3ab..e7673e2b69 100644 --- a/openpilot/starpilot/system/starport/supervisor.py +++ b/openpilot/starpilot/system/starport/supervisor.py @@ -35,7 +35,7 @@ from typing import Any from openpilot.common.hardware import COMMA_HARDWARE from openpilot.starpilot.system.starport import car_profiles, device_health, identity as identity_store, system_slice from openpilot.starpilot.system.starport.auto_connect import AutoConnectPolicy -from openpilot.starpilot.system.starport.bootstrap import NAMES, BootstrapError, WirelessBootstrap +from openpilot.starpilot.system.starport.bootstrap import MODERN_MAJOR, NAMES, BootstrapError, WirelessBootstrap from openpilot.starpilot.system.starport.bugsink_report import PairingAttempt from openpilot.starpilot.system.starport.frame_source import (DEFAULT_PATH as DEFAULT_FRAME_PATH, FLAG_ASYNC_READBACK, FLAG_NV12, FORMAT_NV12, FrameRequest) @@ -498,11 +498,13 @@ class Supervisor: self.stop() with self._lock: companion = self.config.get("companion_address", "").upper() == address - cache = dict(self.config["rfcomm_cache"]) + cache, replies = dict(self.config["rfcomm_cache"]), dict(self.config.get("modern_reply_cache", {})) cached = cache.pop(address, None) is not None + cached = replies.pop(address, None) is not None or cached if not (chosen or companion or cached): return False # nothing of this device was kept - self.config["rfcomm_cache"] = cache + self.config.update(rfcomm_cache=cache, modern_reply_cache=replies) + self._modern_reply.pop(address, None) if chosen: if self._bluez is not None: self._bluez.release() @@ -987,6 +989,21 @@ class Supervisor: else: bluez.restore_class() + def _remember_reply(self, address: str, modern: bool) -> None: + """Keep the version reply a v4+ head unit started with, so the next session leads with it.""" + with self._lock: + self._modern_reply[address] = modern + replies = dict(self.config.get("modern_reply_cache", {})) + if modern == replies.get(address, False): + return + if modern: + replies[address] = True + else: + replies.pop(address, None) + self.config["modern_reply_cache"] = replies + identity_store.save_config(self.config) + self.log("bootstrap_reply_remembered", modern=modern) + def _remember_channel(self, address: str, channel: int | None) -> None: with self._lock: cache = dict(self.config["rfcomm_cache"]) @@ -1242,14 +1259,14 @@ class Supervisor: boot = WirelessBootstrap(rfcomm, self._bootstrap_log, device_serial=config["device_name"], version_status=int(config.get("version_status", 0)), experiment="all" if car_profiles.tuning(config)["bootstrap_experiments"] else "", - modern_reply=self._modern_reply.get(address, False)) + modern_reply=self._modern_reply.get(address, config.get("modern_reply_cache", {}).get(address, False))) self._set(state="wifi_start") result = boot.run(lambda credentials: lease.acquire(credentials, cancelled=boot.join_is_cancelled), cancelled=self._stop.is_set) except Exception as error: if _stray_link(error): raise StrayLinkError(f"Bluetooth link to the car is stuck: {error}") from error # says nothing about the cached channel if boot is not None and boot.silent_after_modern_version(error): - self._modern_reply[address] = not self._modern_reply.get(address, False) # try the other version reply next + self._modern_reply[address] = not boot.modern_reply # try the other version reply next self.log("bootstrap_reply_switched", modern=self._modern_reply[address]) if source == "cache" and not self._stop.is_set() and self._status["last_stage"] in ("rfcomm", "wifi_start") and car_never_answered(error): self._remember_channel(address, None) # the car never answered there; ask it over SDP next time @@ -1257,6 +1274,8 @@ class Supervisor: raise if source == "sdp": self._remember_channel(address, channel) + if boot.version_major >= MODERN_MAJOR: + self._remember_reply(address, boot.modern_reply) self._set(head_unit=result.head_unit) self._remember_fingerprint(result.head_unit) bluez.restore_class() # the car has accepted the comma; stop looking like a phone to everything else diff --git a/openpilot/starpilot/system/starport/tests/test_forget_car.py b/openpilot/starpilot/system/starport/tests/test_forget_car.py index 541594d3cb..d20bf89df7 100644 --- a/openpilot/starpilot/system/starport/tests/test_forget_car.py +++ b/openpilot/starpilot/system/starport/tests/test_forget_car.py @@ -67,3 +67,13 @@ def test_forget_car_edits_the_saved_config_when_the_daemon_is_not_running(tmp_pa assert saved()["companion_address"] == "" and saved()["receiver_address"] == ADAPTER assert protocol.forget_car(ADAPTER.lower(), client) is True assert saved()["receiver_address"] == "" and saved()["rfcomm_cache"] == {} + + +def test_the_version_reply_a_car_started_with_is_kept_and_forgotten_with_it(sup): + sup._remember_reply(ADAPTER, True) + sup._remember_reply(OTHER, True) + assert saved()["modern_reply_cache"] == {ADAPTER: True, OTHER: True} + sup._remember_reply(OTHER, False) + assert saved()["modern_reply_cache"] == {ADAPTER: True} + daemon.handle(sup, {"command": "forget_receiver", "address": ADAPTER}) + assert saved()["modern_reply_cache"] == {} and ADAPTER not in sup._modern_reply diff --git a/openpilot/starpilot/system/starport/tests/test_identity_migration.py b/openpilot/starpilot/system/starport/tests/test_identity_migration.py index 05731ea7cc..837adf77f4 100644 --- a/openpilot/starpilot/system/starport/tests/test_identity_migration.py +++ b/openpilot/starpilot/system/starport/tests/test_identity_migration.py @@ -28,3 +28,11 @@ def test_existing_identity_survives_directory_migration(tmp_path, monkeypatch, o assert directory == (legacy if outcome == 'denied' else current) assert (directory / 'certificate').read_bytes() == b'existing identity' + + +def test_remembered_version_replies_load_only_as_true_per_address(tmp_path): + path = tmp_path / "config.json" + path.write_text('{"config_version": 7, "modern_reply_cache": {"aa:bb:cc:dd:ee:01": true, "AA:BB:CC:DD:EE:02": false, "x": 1}}') + assert identity.load_config(path)["modern_reply_cache"] == {"AA:BB:CC:DD:EE:01": True} + path.write_text('{"config_version": 7}') + assert identity.load_config(path)["modern_reply_cache"] == {} From 085aa2a173d18c1b65a710b0b29cb741458f6972 Mon Sep 17 00:00:00 2001 From: dirwin31 <83434411+dirwin31@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:18:58 -0700 Subject: [PATCH 03/38] Send StarPort logs from Logs & Diagnostics Adds a Send logs card to the StarPort Logs page so a user can report a problem with all of their reportable logs: an optional note of up to 60 characters, and the same zip as Download all logs, sent from the device to the developer's log inbox. Each device keeps its own thread, titled with the car fingerprint and a hashed device tag. Manual sends only; no automatic failure reports. If the zip is over the 10 MB size limit, it is resent without the device snapshot. --- openpilot/starpilot/galaxy/server.py | 38 ++- openpilot/starpilot/galaxy/starport_logs.py | 4 +- openpilot/starpilot/galaxy/starport_send.py | 261 ++++++++++++++++++ .../galaxy/tests/test_starport_send.mjs | 62 +++++ .../galaxy/tests/test_starport_send.py | 211 ++++++++++++++ openpilot/starpilot/galaxy/web/css/galaxy.css | 3 + openpilot/starpilot/galaxy/web/js/logs.js | 26 +- .../starpilot/galaxy/web/js/starport-logs.js | 74 +++++ 8 files changed, 670 insertions(+), 9 deletions(-) create mode 100644 openpilot/starpilot/galaxy/starport_send.py create mode 100644 openpilot/starpilot/galaxy/tests/test_starport_send.mjs create mode 100644 openpilot/starpilot/galaxy/tests/test_starport_send.py diff --git a/openpilot/starpilot/galaxy/server.py b/openpilot/starpilot/galaxy/server.py index 9de5434177..7423609558 100644 --- a/openpilot/starpilot/galaxy/server.py +++ b/openpilot/starpilot/galaxy/server.py @@ -97,7 +97,7 @@ def make_server(*, port=8082, host='127.0.0.1', monitor=None, owner=None, crashe sounds=None, software_operations=None, drive_stats=None, layout_preview_socket=None, controllers_socket=None, remote_pairing=None, parked=None, camera_snapshot=None, clock=time.monotonic, starport_setup=None, starport_client=None, navigation=None, offline_maps=None, drive_state=None, cloud_provider=None, cloud_offroad=None, projection_layout=None, - local_access=None, tmux_live=None, starport_logs=None, hotspot=None, toyota_tools=None, replay_client=None, replay_root=None, + local_access=None, tmux_live=None, starport_logs=None, starport_log_sender=None, hotspot=None, toyota_tools=None, replay_client=None, replay_root=None, spotify_setup=None, companion=None, phone_location=None, reverse_proxy=None): from openpilot.starpilot.galaxy.companion import CompanionRegistry, APK_PATH from openpilot.starpilot.gps.phone import PhoneGpsPublisher @@ -139,6 +139,17 @@ def make_server(*, port=8082, host='127.0.0.1', monitor=None, owner=None, crashe from openpilot.starpilot.galaxy.starport_logs import StarPortLogs starport_logs_source = StarPortLogs() return starport_logs_source + starport_log_sender_source = starport_log_sender + starport_log_sender_lock = threading.Lock() + + def starport_log_sender(): + nonlocal starport_log_sender_source + with starport_log_sender_lock: + if starport_log_sender_source is None: + from openpilot.starpilot.galaxy.starport_send import StarPortLogSender, device_summary + source = starport_logs() + starport_log_sender_source = StarPortLogSender(source.bundle, summary=lambda: device_summary(source.directory)) + return starport_log_sender_source software_source = software software_operations_source = software_operations software_lock = threading.Lock() @@ -928,6 +939,9 @@ def make_server(*, port=8082, host='127.0.0.1', monitor=None, owner=None, crashe # immediately before sending potentially sensitive content. if self.require_session(): self.json(200, result) + elif path == '/api/starport/logs/send': + if self.require_session(): + self.json(200, starport_log_sender().status()) elif path == '/api/starport/logs' or path.startswith('/api/starport/logs/'): if not self.require_session(): return @@ -1635,6 +1649,7 @@ def make_server(*, port=8082, host='127.0.0.1', monitor=None, owner=None, crashe '/api/replay/export-start', '/api/galaxy/pair', '/api/galaxy/unpair', '/api/galaxy/device-name', '/api/galaxy/hotspot', '/api/cameras/snapshot', '/api/starport/layout', '/api/starport/enable', '/api/starport/control', '/api/starport/pairing', + '/api/starport/logs/send', '/api/starport/pairing/response', '/api/starport/pairing/cancel', '/api/starport/pairing/select', '/api/spotify/enable', '/api/spotify/key', '/api/spotify/preferences', '/api/spotify/control', '/api/spotify/maintenance', '/api/ui/layout', '/api/ui/layout/preview', '/api/favorites/slots', @@ -2314,6 +2329,27 @@ def make_server(*, port=8082, host='127.0.0.1', monitor=None, owner=None, crashe else: self.json(401, {'error': 'Sign in to Galaxy'}) return + if path == '/api/starport/logs/send': + if type(payload) is not dict or set(payload) != {'message'}: + self.json(400, {'error': 'Invalid StarPort log send request'}) + return + identity = self.settings_session() + if identity is None: + self.json(401, {'error': 'Sign in to Galaxy'}) + return + from openpilot.starpilot.galaxy.starport_send import SendBusy, SendRejected + try: + result = starport_log_sender().start(payload['message']) + except SendRejected as error: + self.json(400, {'error': str(error)}) + except SendBusy as error: + self.json(409, {'error': str(error)}) + else: + if self.settings_session() == identity: + self.json(202, result) + else: + self.json(401, {'error': 'Sign in to Galaxy'}) + return if path == '/api/starport/enable': if type(payload) is not dict or set(payload) not in ({'enabled'}, {'enabled', 'enableBluetooth'}) or \ type(payload['enabled']) is not bool or type(payload.get('enableBluetooth', False)) is not bool or \ diff --git a/openpilot/starpilot/galaxy/starport_logs.py b/openpilot/starpilot/galaxy/starport_logs.py index 0da2ecb800..eab7fc67e1 100644 --- a/openpilot/starpilot/galaxy/starport_logs.py +++ b/openpilot/starpilot/galaxy/starport_logs.py @@ -71,9 +71,9 @@ class StarPortLogs: except OSError: raise LogsUnavailable from None - def bundle(self, now: datetime | None = None) -> tuple[str, bytes]: + def bundle(self, now: datetime | None = None, *, system_snapshot: bool = True) -> tuple[str, bytes]: try: - data = compat_report.bundle(self.directory, self.config_path) + data = compat_report.bundle(self.directory, self.config_path, **({} if system_snapshot else {"snapshot": None})) except OSError: raise LogsUnavailable from None stamp = (now or datetime.now()).strftime("%Y%m%d-%H%M%S") diff --git a/openpilot/starpilot/galaxy/starport_send.py b/openpilot/starpilot/galaxy/starport_send.py new file mode 100644 index 0000000000..332bee4fba --- /dev/null +++ b/openpilot/starpilot/galaxy/starport_send.py @@ -0,0 +1,261 @@ +"""Send StarPort Logs: post the "Download all logs" zip to the developer's Discord, from Logs & Diagnostics. + +Only sent when the user presses Send; there are no automatic reports. Each device gets its own forum +thread, titled with its car and a short hashed device tag, and later sends from the same device go to +that thread. The webhook URL is built in, encoded so repository scanners don't pick it up; a URL in +WEBHOOK_OVERRIDE_PATH on the device replaces it without a code change. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import re +import tempfile +import threading +import time +from collections.abc import Callable +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +from openpilot.starpilot.system.starport import identity as identity_store + +MESSAGE_LIMIT = 60 +WEBHOOK_OVERRIDE_PATH = identity_store.DATA_DIR / "webhook_url" +THREAD_DIR = identity_store.DATA_DIR / "discord_threads" +# The Discord webhook URL that receives StarPort logs, XORed with _WEBHOOK_KEY and base64-encoded so it is not +# stored in plain text ("" = sending not set up). Obfuscation only: the device has to be able to decode it. +_WEBHOOK_KEY = b"starport-logs" +_WEBHOOK_DATA = "".join(("GwAVAgNVXVtJBRwEHAEQTxEfAl0VXQVAEBYRHA4dGxxdRRhaWlJLREBYRkZcS0UcXl1QQVwAEEIzAydBSA", + "4KIjQjDShfJj0ZRkA6OjEbMi4JRzE6Rh1sOxo/HCVCDRMlCzRBHyIhUgQFTDAfChwdAGkpWR0HES0ZJw==")) +DISCORD_FILE_LIMIT = 10 * 1024 * 1024 - 64 * 1024 # 10 MiB attachment limit, with room for the form fields +THREAD_CACHE_WARNING = "The forum thread could not be remembered on this device. The next upload may create another post." +CONTROL = re.compile(r"[\x00-\x1f\x7f]+") + + +class SendRejected(Exception): + pass + + +class SendBusy(Exception): + pass + + +def webhook_url(override_path: Path = WEBHOOK_OVERRIDE_PATH) -> str: + try: + override = override_path.read_text().strip() + except OSError: + override = "" + if override: + return override + if not _WEBHOOK_DATA: + return "" + data = base64.b64decode(_WEBHOOK_DATA) + return bytes(byte ^ _WEBHOOK_KEY[index % len(_WEBHOOK_KEY)] for index, byte in enumerate(data)).decode() + + +def clean_message(value) -> str: + if not isinstance(value, str): + raise SendRejected("The message must be text.") + message = CONTROL.sub(" ", value).strip() + if len(message) > MESSAGE_LIMIT: + raise SendRejected(f"Keep the message to {MESSAGE_LIMIT} characters.") + return message + + +def _car_fingerprint() -> str: + from openpilot.common.params import Params + from openpilot.starpilot.saved_source import read_saved + raw, readable = read_saved(Params(), "CarParamsPersistent", 1 << 20) + if not readable or not raw: + return "" + from openpilot.cereal import messaging + from opendbc.car.structs import car + from openpilot.starpilot.schema_cache import inspect_cache + inspected = inspect_cache("CarParamsPersistent", raw) + if inspected.status != "valid" or inspected.payload is None: + return "" + fingerprint = str(messaging.log_from_bytes(inspected.payload, car.CarParams).carFingerprint or "").strip() + return "" if fingerprint == "MOCK" else fingerprint + + +def _head_unit(log_dir: Path | None, logs_to_check: int = 3) -> str: + """The car's head unit from the newest session log that identified it, or ''.""" + from openpilot.starpilot.system.starport import compat_report + for path in compat_report.session_logs(log_dir or identity_store.LOG_DIR)[:logs_to_check]: + car = compat_report.summarize(compat_report.load_events(path))["car"] + name = " ".join(part for part in (car.get("head_unit_make"), car.get("head_unit_model")) if part) + name = name or car.get("display_name") or car.get("bluetooth_name") or "" + if name: + return name + return "" + + +def device_summary(log_dir: Path | None = None) -> dict: + from openpilot.starpilot.system.starport.bugsink_transport import device_metadata + summary = device_metadata() + for key, read in (("car_fingerprint", _car_fingerprint), ("head_unit", lambda: _head_unit(log_dir))): + try: + summary[key] = read() + except Exception as error: + summary[f"{key}_error"] = str(error) + return summary + + +def device_tag(summary: dict) -> str: + """8 characters of a hash of the dongle ID: tells devices apart without showing the ID.""" + dongle_id = summary.get("device_id") or "" + return hashlib.sha256(f"starpilot-auto:{dongle_id}".encode()).hexdigest()[:8] + + +def thread_title(summary: dict) -> str: + return f"{summary.get('car_fingerprint') or 'Device'} {device_tag(summary)}"[:100] + + +def _response_json(response) -> dict: + try: + value = response.json() + except (AttributeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + +def _save_thread_id(path: Path, key: str, thread_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + # A power loss during a write must not truncate the previous cache entry. + with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as temporary: + temporary_path = Path(temporary.name) + try: + json.dump({"key": key, "thread_id": thread_id}, temporary) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, path) + finally: + temporary_path.unlink(missing_ok=True) + + +def send(data: bytes, name: str, note: str, summary: dict, url: str | None = None, post=None, + thread_dir: Path | None = None) -> str | None: + """Post the zip; return a cache warning after success, or raise SendRejected on an upload failure.""" + url = webhook_url() if url is None else url + if not url: + raise SendRejected("Sending isn't set up on this build. Use Download all logs and send the file instead.") + if len(data) > DISCORD_FILE_LIMIT: + raise SendRejected(f"The logs are {len(data) / 1e6:.1f} MB, over Discord's 10 MB limit. Use Download all logs instead.") + if post is None: + import requests + post = requests.post + dongle_id = summary.get("device_id") or "device" + lines = [f"**StarPort logs from {device_tag(summary)}**", + f"Car: {summary.get('car_fingerprint') or 'unknown'}", + f"Head unit: {summary.get('head_unit') or 'unknown'}", + f"Branch {summary.get('branch', '?')} @ {str(summary.get('commit', '?'))[:10]}", + f"Note: {note.strip() or '(none)'}"] + payload = {"content": "\n".join(lines)[:1900], "username": "StarPilot diagnostics", "allowed_mentions": {"parse": []}} + + parts = urlsplit(url) + query = dict(parse_qsl(parts.query, keep_blank_values=True)) + explicit_thread = bool(query.get("thread_id")) + # Ignore query options when identifying a webhook, and never store its token in the cache. + cache_key = hashlib.sha256(json.dumps([parts.scheme, parts.netloc, parts.path.rstrip("/"), dongle_id]).encode()).hexdigest() + thread_path = (THREAD_DIR if thread_dir is None else thread_dir) / f"{cache_key}.json" + cached_id = None + if not explicit_thread: + query.pop("thread_id", None) + try: + entry = json.loads(thread_path.read_text()) + if isinstance(entry, dict) and entry.get("key") == cache_key: + candidate = entry.get("thread_id") + if isinstance(candidate, str) and candidate.isascii() and candidate.isdigit(): + cached_id = candidate + except (OSError, ValueError): + pass + + # Always wait for confirmation, including uploads to an existing thread. + query["wait"] = "true" + for attempt in range(2): + if not explicit_thread and cached_id: + query["thread_id"] = cached_id + elif not explicit_thread: + query.pop("thread_id", None) + payload["thread_name"] = thread_title(summary) + post_url = urlunsplit(parts._replace(query=urlencode(query))) + + try: + response = post(post_url, data={"payload_json": json.dumps(payload)}, files={"files[0]": (name, data, "application/zip")}, timeout=120) + except Exception as error: + raise SendRejected(f"Could not send the logs: {type(error).__name__}. Check the internet connection, or use Download all logs.") from error + + # Unknown Channel means the cached thread was deleted (Discord answers 400 for webhooks, 404 elsewhere). + # Unknown Webhook does not. + if attempt == 0 and cached_id and response.status_code in (400, 404) and _response_json(response).get("code") == 10003: + cached_id = None # Retry from memory even if the cache cannot be removed. + try: + thread_path.unlink(missing_ok=True) + except OSError: + pass + continue + break + + if response.status_code not in (200, 204): + reason = str(_response_json(response).get("message") or "")[:120] + detail = f"HTTP {response.status_code}{f': {reason}' if reason else ''}" + raise SendRejected(f"Discord refused the logs ({detail}). Use Download all logs instead.") + + if not explicit_thread and not cached_id: + new_thread_id = _response_json(response).get("channel_id") + if not isinstance(new_thread_id, str) or not new_thread_id.isascii() or not new_thread_id.isdigit(): + return THREAD_CACHE_WARNING + try: + _save_thread_id(thread_path, cache_key, new_thread_id) + except OSError: + return THREAD_CACHE_WARNING + return None + + +class StarPortLogSender: + """One send at a time, off the request thread; the page polls status().""" + + def __init__(self, bundle: Callable[..., tuple[str, bytes]], *, sender: Callable[..., str | None] = send, + summary: Callable[[], dict] = device_summary, url: Callable[[], str] = webhook_url): + self._bundle, self._sender, self._summary, self._url = bundle, sender, summary, url + self._lock = threading.Lock() + self._thread: threading.Thread | None = None + self._state = {"state": "idle", "message": ""} + + def status(self) -> dict: + with self._lock: + return {**self._state, "available": bool(self._url()), "messageLimit": MESSAGE_LIMIT} + + def start(self, message) -> dict: + note = clean_message(message) + with self._lock: + if self._thread is not None and self._thread.is_alive(): + raise SendBusy("StarPort logs are already being sent.") + self._state = {"state": "sending", "message": "Collecting logs"} + self._thread = threading.Thread(target=self._run, args=(note,), name="starport_log_send", daemon=True) + self._thread.start() + return self.status() + + def _set(self, **values) -> None: + with self._lock: + self._state.update(values) + + def _run(self, note: str) -> None: + try: + summary = self._summary() + name, data = self._bundle() + left_out = "" + if len(data) > DISCORD_FILE_LIMIT: + # The device snapshot is the optional, bulky part; the session logs are what matter. + name, data = self._bundle(system_snapshot=False) + left_out = " The device snapshot was left out to fit Discord's 10 MB limit." + self._set(message="Sending logs") + warning = self._sender(data, name, note, summary) + self._set(state="sent", message=f"Logs sent.{left_out} {warning or 'Thanks!'}".strip(), sentAt=int(time.time())) + except SendRejected as error: + self._set(state="failed", message=str(error)[:400]) + except Exception as error: + self._set(state="failed", message=f"Could not send the logs: {type(error).__name__}. Use Download all logs instead.") diff --git a/openpilot/starpilot/galaxy/tests/test_starport_send.mjs b/openpilot/starpilot/galaxy/tests/test_starport_send.mjs new file mode 100644 index 0000000000..91175e7446 --- /dev/null +++ b/openpilot/starpilot/galaxy/tests/test_starport_send.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict" +import { StarPortLogSend, SEND_URL, MESSAGE_LIMIT } from "../web/js/starport-logs.js" + +const flush = async () => { for (let i = 0; i < 8; i++) await Promise.resolve() } +function fixture() { + const requests = [], states = [], timers = [] + let unauthorized = 0 + const client = new StarPortLogSend({ publish: (state) => states.push(state), unauthorized: () => { unauthorized++ }, + fetcher: (url, options) => new Promise((resolve) => requests.push({ url, options, resolve })), + schedule: (fn) => { timers.push(fn); return timers.length }, cancel: () => {} }) + const reply = async (index, body, status = 200) => { + requests[index].resolve({ ok: status >= 200 && status < 300, status, json: async () => body }) + await flush() + } + return { client, requests, states, timers, reply, get unauthorized() { return unauthorized } } +} + +assert.equal(SEND_URL, "./api/starport/logs/send") +assert.equal(MESSAGE_LIMIT, 60) + +// Status on open, then a send that is polled until it finishes. +const normal = fixture() +normal.client.start() +assert.equal(normal.requests[0].url, SEND_URL) +await normal.reply(0, { state: "idle", message: "", available: true }) +assert.equal(normal.states.at(-1).state, "idle") +const sending = normal.client.send(" Black screen ") +assert.equal(normal.states.at(-1).state, "sending") +assert.equal(normal.requests[1].options.method, "POST") +assert.deepEqual(JSON.parse(normal.requests[1].options.body), { message: "Black screen" }) +await normal.reply(1, { state: "sending", message: "Collecting logs" }, 202) +assert.equal(await sending, true) +assert.equal(normal.timers.length, 1) +normal.timers[0]() +await normal.reply(2, { state: "sent", message: "Logs sent. Thanks!" }) +assert.equal(normal.states.at(-1).state, "sent") +assert.equal(normal.timers.length, 1) + +// Too long is refused locally; a busy device reports its error. +const limited = fixture() +limited.client.start() +await limited.reply(0, { state: "idle" }) +assert.equal(await limited.client.send("x".repeat(61)), false) +assert.equal(limited.requests.length, 1) +assert.match(limited.states.at(-1).error, /60 characters/) +const busy = limited.client.send("hi") +await limited.reply(1, { error: "StarPort logs are already being sent." }, 409) +assert.equal(await busy, false) +assert.equal(limited.states.at(-1).error, "StarPort logs are already being sent.") + +// Signed out. +const signedOut = fixture() +signedOut.client.start() +await signedOut.reply(0, {}, 401) +assert.equal(signedOut.unauthorized, 1) +assert.equal(await signedOut.client.send("hi"), false) + +// Sending unavailable on this build hides the card. +const unset = fixture() +unset.client.start() +await unset.reply(0, { state: "idle", available: false }) +assert.equal(unset.states.at(-1).available, false) diff --git a/openpilot/starpilot/galaxy/tests/test_starport_send.py b/openpilot/starpilot/galaxy/tests/test_starport_send.py new file mode 100644 index 0000000000..aceff896c0 --- /dev/null +++ b/openpilot/starpilot/galaxy/tests/test_starport_send.py @@ -0,0 +1,211 @@ +"""Send StarPort Logs: the Discord upload, its per-device thread cache, the background job and its routes.""" + +import http.client +import json +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest import mock +from urllib.parse import parse_qs, urlsplit + +from openpilot.starpilot.galaxy import starport_send +from openpilot.starpilot.galaxy.access import GalaxyAccessOwner +from openpilot.starpilot.galaxy.server import make_server +from openpilot.starpilot.galaxy.starport_send import (DISCORD_FILE_LIMIT, MESSAGE_LIMIT, SendBusy, SendRejected, StarPortLogSender, + clean_message, send, thread_title) + +URL = "https://discord.example/api/webhooks/1/token" +SUMMARY = {"device_id": "0123456789abcdef", "car_fingerprint": "HONDA_CIVIC_2022", "head_unit": "Honda Display Audio", + "branch": "Dom", "commit": "abcdef1234567890"} + + +class Response: + def __init__(self, status_code=200, body=None): + self.status_code, self.body = status_code, body if body is not None else {"channel_id": "555"} + + def json(self): + return self.body + + +class Poster: + def __init__(self, *responses): + self.responses, self.calls = list(responses), [] + + def __call__(self, url, data, files, timeout): + self.calls.append({"url": url, "payload": json.loads(data["payload_json"]), "files": files}) + return self.responses.pop(0) + + +def query(call): + return {key: values[0] for key, values in parse_qs(urlsplit(call["url"]).query).items()} + + +class SendTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.threads = Path(temporary.name) + + def send(self, post, note="Dropped at the light", data=b"zip"): + return send(data, "logs.zip", note, SUMMARY, url=URL, post=post, thread_dir=self.threads) + + def test_first_send_opens_a_device_thread_and_later_sends_reuse_it(self): + first = Poster(Response(body={"channel_id": "555"})) + self.assertIsNone(self.send(first)) + call = first.calls[0] + self.assertEqual(call["payload"]["thread_name"], thread_title(SUMMARY)) + self.assertTrue(call["payload"]["thread_name"].startswith("HONDA_CIVIC_2022 ")) + self.assertEqual(call["payload"]["allowed_mentions"], {"parse": []}) + self.assertIn("Note: Dropped at the light", call["payload"]["content"]) + self.assertIn("Head unit: Honda Display Audio", call["payload"]["content"]) + self.assertNotIn(SUMMARY["device_id"], call["payload"]["content"]) + self.assertEqual(query(call), {"wait": "true"}) + self.assertEqual(call["files"]["files[0]"], ("logs.zip", b"zip", "application/zip")) + self.assertNotIn("token", "".join(path.read_text() for path in self.threads.iterdir())) + + second = Poster(Response()) + self.assertIsNone(self.send(second)) + self.assertEqual(query(second.calls[0])["thread_id"], "555") + self.assertNotIn("thread_name", second.calls[0]["payload"]) + + def test_deleted_thread_is_forgotten_and_a_new_one_opened(self): + self.send(Poster(Response(body={"channel_id": "555"}))) + poster = Poster(Response(404, {"code": 10003}), Response(body={"channel_id": "777"})) + self.assertIsNone(self.send(poster)) + self.assertEqual(query(poster.calls[0])["thread_id"], "555") + self.assertNotIn("thread_id", query(poster.calls[1])) + self.assertEqual(poster.calls[1]["payload"]["thread_name"], thread_title(SUMMARY)) + third = Poster(Response()) + self.send(third) + self.assertEqual(query(third.calls[0])["thread_id"], "777") + + def test_refusals_and_oversize_raise_without_posting(self): + with self.assertRaisesRegex(SendRejected, "HTTP 401: Invalid Webhook Token"): + self.send(Poster(Response(401, {"message": "Invalid Webhook Token"}))) + poster = Poster() + with self.assertRaisesRegex(SendRejected, "10 MB"): + self.send(poster, data=b"x" * (DISCORD_FILE_LIMIT + 1)) + self.assertEqual(poster.calls, []) + with self.assertRaisesRegex(SendRejected, "isn't set up"): + send(b"zip", "logs.zip", "", SUMMARY, url="", post=poster, thread_dir=self.threads) + + def test_message_is_limited_and_single_line(self): + self.assertEqual(MESSAGE_LIMIT, 60) + self.assertEqual(clean_message(" car\nwent black "), "car went black") + self.assertEqual(clean_message("x" * 60), "x" * 60) + for bad in ("x" * 61, None, 5): + with self.assertRaises(SendRejected): + clean_message(bad) + + def test_built_in_webhook_and_override(self): + self.assertTrue(starport_send.webhook_url(self.threads / "absent").startswith("https://discord.com/api/webhooks/")) + override = self.threads / "webhook_url" + override.write_text(f"{URL}\n") + self.assertEqual(starport_send.webhook_url(override), URL) + + +class SenderJobTest(unittest.TestCase): + def wait(self, job): + deadline = time.monotonic() + 2 + while job.status()["state"] == "sending" and time.monotonic() < deadline: + time.sleep(0.01) + return job.status() + + def test_sends_in_the_background_and_drops_the_snapshot_when_too_big(self): + bundles, sent = [], [] + + def bundle(system_snapshot=True): + bundles.append(system_snapshot) + return "logs.zip", b"x" * (DISCORD_FILE_LIMIT + 1 if system_snapshot else 10) + + job = StarPortLogSender(bundle, sender=lambda *args: sent.append(args), summary=lambda: SUMMARY, url=lambda: URL) + self.assertEqual(job.start("hello")["state"], "sending") + status = self.wait(job) + self.assertEqual(status["state"], "sent") + self.assertIn("snapshot was left out", status["message"]) + self.assertEqual(bundles, [True, False]) + self.assertEqual(sent[0][2], "hello") + + def test_one_send_at_a_time_and_failures_are_reported(self): + release = threading.Event() + + def sender(*_): + release.wait(2) + raise SendRejected("Discord refused the logs (HTTP 500).") + + job = StarPortLogSender(lambda **_: ("logs.zip", b"zip"), sender=sender, summary=lambda: SUMMARY, url=lambda: URL) + job.start("") + with self.assertRaises(SendBusy): + job.start("again") + release.set() + self.assertEqual(self.wait(job), {"state": "failed", "message": "Discord refused the logs (HTTP 500).", "available": True, + "messageLimit": 60}) + + +class SendHTTPTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.owner = GalaxyAccessOwner(Path(temporary.name) / "access") + self.job = mock.Mock() + self.job.status.return_value = {"state": "idle", "message": "", "available": True, "messageLimit": 60} + self.job.start.return_value = {"state": "sending", "message": "Collecting logs", "available": True, "messageLimit": 60} + self.server = make_server(port=0, owner=self.owner, starport_log_sender=self.job) + self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + self.thread.start() + self.addCleanup(self.close_server) + self.assertTrue(self.owner.configure("password123", lambda: True)) + + def close_server(self): + self.server.shutdown() + self.thread.join(timeout=2) + self.server.server_close() + + def request(self, path, *, method="GET", body=None, headers=None): + connection = http.client.HTTPConnection("127.0.0.1", self.server.server_port, timeout=2) + try: + origin = {"Origin": f"http://127.0.0.1:{self.server.server_port}"} if method == "POST" else {} + connection.request(method, path, body=body, headers={"Forwarded": "for=203.0.113.8", **origin, **(headers or {})}) + response = connection.getresponse() + return response.status, response.read(), dict(response.getheaders()) + finally: + connection.close() + + def post(self, body, cookie=None): + headers = {"Content-Type": "application/json", **({"Cookie": cookie} if cookie else {})} + return self.request("/api/starport/logs/send", method="POST", body=json.dumps(body), headers=headers) + + def login(self): + status, _, headers = self.post_login() + self.assertEqual(status, 200) + return headers["Set-Cookie"].split(";", 1)[0] + + def post_login(self): + return self.request("/api/auth/login", method="POST", body=json.dumps({"password": "password123"}), + headers={"Content-Type": "application/json"}) + + def test_requires_a_session(self): + self.assertEqual(self.request("/api/starport/logs/send")[0], 401) + self.assertEqual(self.post({"message": "hi"})[0], 401) + self.job.start.assert_not_called() + + def test_status_start_and_rejections(self): + cookie = self.login() + status, body, _ = self.request("/api/starport/logs/send", headers={"Cookie": cookie}) + self.assertEqual((status, json.loads(body)["state"]), (200, "idle")) + + status, body, _ = self.post({"message": "Black screen after unlock"}, cookie) + self.assertEqual((status, json.loads(body)["state"]), (202, "sending")) + self.job.start.assert_called_once_with("Black screen after unlock") + + self.assertEqual(self.post({"message": "hi", "extra": 1}, cookie)[0], 400) + self.job.start.side_effect = SendRejected("Keep the message to 60 characters.") + self.assertEqual(self.post({"message": "x" * 61}, cookie)[0], 400) + self.job.start.side_effect = SendBusy("StarPort logs are already being sent.") + self.assertEqual(self.post({"message": "hi"}, cookie)[0], 409) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/galaxy/web/css/galaxy.css b/openpilot/starpilot/galaxy/web/css/galaxy.css index 1b167d526e..b6347847cf 100644 --- a/openpilot/starpilot/galaxy/web/css/galaxy.css +++ b/openpilot/starpilot/galaxy/web/css/galaxy.css @@ -1060,6 +1060,9 @@ button.gx-chip:hover { .gx-starport-log__download { flex: none; width: var(--touch-target); height: var(--touch-target); border: 1px solid var(--glass-border); background: var(--primary-container); color: var(--on-primary-container); } .gx-starport-log__download:hover { background: var(--primary); color: var(--on-primary); } .gx-starport-logs__section-title { margin: var(--sp-6) 0 var(--sp-3); } +.gx-starport-send__row { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3); margin-top: var(--sp-3); } +.gx-starport-send__row .gx-field { flex: 1 1 220px; min-width: 0; } +.gx-starport-send__count { font-size: var(--fs-xs); opacity: .7; font-variant-numeric: tabular-nums; } @media (max-width: 700px) { .gx-starport-log { gap: var(--sp-3); padding: var(--sp-3); } diff --git a/openpilot/starpilot/galaxy/web/js/logs.js b/openpilot/starpilot/galaxy/web/js/logs.js index e15cc72f31..9d1e0a3f94 100644 --- a/openpilot/starpilot/galaxy/web/js/logs.js +++ b/openpilot/starpilot/galaxy/web/js/logs.js @@ -5,7 +5,7 @@ import { navigate } from "./router.js" import { SystemMonitor } from "./system-monitor.js" import { CrashReportsFeed } from "./crash-reports.js" import { storedSize } from "./record-deletion.js" -import { StarPortLogsFeed, BUNDLE_URL, fileUrl, outcomeLabel } from "./starport-logs.js" +import { StarPortLogsFeed, StarPortLogSend, BUNDLE_URL, MESSAGE_LIMIT, fileUrl, outcomeLabel } from "./starport-logs.js" import { MenuTile } from "./menu-tile.js" import { BluetoothNotice, bluetoothConnection } from "./bluetooth-notice.js" @@ -15,25 +15,27 @@ export const Logs = { props: { path: { type: String, required: true }, mode: { type: String, required: true }, unauthorized: { type: Function, required: true }, tools: { type: Array, required: true } }, data: () => ({ crashes: { reports: [], scanIncomplete: false, listLimited: false, status: "idle", error: "", selected: null, preview: null, previewStatus: "idle", deleting: false, storageBytes: 0, reportCount: 0 }, search: "", - starportLogs: { sessions: [], others: [], status: "idle", error: "" }, bluetooth: bluetoothConnection() }), + starportLogs: { sessions: [], others: [], status: "idle", error: "" }, bluetooth: bluetoothConnection(), + logSend: { state: "idle", message: "", available: true, error: "" }, sendNote: "", messageLimit: MESSAGE_LIMIT }), computed: { visibleReports() { return this.crashes.reports.filter((item) => item.name.toLowerCase().includes(this.search.toLowerCase())) }, }, created() { this.crashFeed = new CrashReportsFeed({ publish: (update) => Object.assign(this.crashes, update), unauthorized: this.unauthorized }) this.starportFeed = new StarPortLogsFeed({ publish: (update) => Object.assign(this.starportLogs, update), unauthorized: this.unauthorized }) + this.sendFeed = new StarPortLogSend({ publish: (update) => Object.assign(this.logSend, update), unauthorized: this.unauthorized }) }, mounted() { if (this.path === "/logs/crashes" && this.mode === "local") this.crashFeed.start() - if (this.path === "/logs/starport" && this.mode === "local") this.starportFeed.start() + if (this.path === "/logs/starport" && this.mode === "local") { this.starportFeed.start(); this.sendFeed.start() } }, - beforeUnmount() { this.crashFeed.stop(); this.starportFeed.stop() }, + beforeUnmount() { this.crashFeed.stop(); this.starportFeed.stop(); this.sendFeed.stop() }, watch: { path(next, previous) { if (next === "/logs/crashes" && this.mode === "local") this.crashFeed.start() else if (previous === "/logs/crashes") { this.crashFeed.stop(); this.search = "" } - if (next === "/logs/starport" && this.mode === "local") this.starportFeed.start() - else if (previous === "/logs/starport") this.starportFeed.stop() + if (next === "/logs/starport" && this.mode === "local") { this.starportFeed.start(); this.sendFeed.start() } + else if (previous === "/logs/starport") { this.starportFeed.stop(); this.sendFeed.stop() } }, }, methods: { @@ -42,6 +44,9 @@ export const Logs = { bundleUrl: () => BUNDLE_URL, fileUrl, outcomeLabel, + async sendLogs() { + if (await this.sendFeed.send(this.sendNote)) this.sendNote = "" + }, reportDate(seconds) { return new Date(seconds * 1000).toLocaleString() }, reportSize(bytes) { return `${(bytes / 1024).toFixed(1)} KiB` }, async copyPreview() { @@ -94,6 +99,15 @@ export const Logs = {

The last 20 connection sessions, newest first. The archive includes every session, a readable report for each, StarPort settings, the car view's renderer logs, and a snapshot of the device (Wi-Fi, Bluetooth and kernel logs, build, CPU load). Hardware addresses are masked.

Log summaries remain available.
Download all logsZIP
+
+
Send logs to the developer
+

Sends the same archive as Download all logs straight from the device. Add a short note about what happened.

+
{{ sendNote.length }}/{{ messageLimit }}
+ {{ logSend.message || 'Sending logs' }}… + {{ logSend.message }} + {{ logSend.message }} + {{ logSend.error }} +
Loading StarPort logs… {{ starportLogs.error || 'StarPort logs are unavailable.' }} diff --git a/openpilot/starpilot/galaxy/web/js/starport-logs.js b/openpilot/starpilot/galaxy/web/js/starport-logs.js index f88355d5ff..6c2e7de660 100644 --- a/openpilot/starpilot/galaxy/web/js/starport-logs.js +++ b/openpilot/starpilot/galaxy/web/js/starport-logs.js @@ -68,3 +68,77 @@ export class StarPortLogsFeed { } } } + +export const SEND_URL = `${LOGS_URL}/send` +export const MESSAGE_LIMIT = 60 +const POLL_MS = 1500 + +// "Send logs": the device builds the same zip as Download all logs and posts it to the developer's Discord. +// Only small JSON crosses the connection, so this also works over Bluetooth. +export class StarPortLogSend { + constructor({ publish, unauthorized = () => {}, fetcher = (...args) => fetch(...args), schedule = (fn, ms) => setTimeout(fn, ms), + cancel = (id) => clearTimeout(id) }) { + Object.assign(this, { publish, unauthorized, fetcher, schedule, cancel }) + this.active = false + this.timer = null + } + + stop() { + this.active = false + if (this.timer !== null) this.cancel(this.timer) + this.timer = null + } + + start() { + this.stop() + this.active = true + return this.poll() + } + + async call(options) { + const response = await this.fetcher(SEND_URL, { cache: "no-store", credentials: "same-origin", ...options }) + if (response.status === 401) { + this.stop() + this.unauthorized() + return null + } + const body = await response.json().catch(() => null) + if (!response.ok) throw new Error(body?.error || "Sending StarPort logs is unavailable") + return body + } + + apply(body) { + if (!this.active || !body) return + this.publish({ state: body.state, message: body.message || "", available: body.available !== false, error: "" }) + if (this.timer !== null) this.cancel(this.timer) + this.timer = body.state === "sending" ? this.schedule(() => { this.timer = null; this.poll() }, POLL_MS) : null + } + + async poll() { + if (!this.active) return + try { + this.apply(await this.call({})) + } catch (error) { + if (this.active) this.publish({ error: error?.message || "Sending StarPort logs is unavailable" }) + } + } + + async send(message) { + if (!this.active) return false + const text = String(message ?? "").trim() + if (text.length > MESSAGE_LIMIT) { + this.publish({ error: `Keep the message to ${MESSAGE_LIMIT} characters.` }) + return false + } + this.publish({ state: "sending", message: "Starting", error: "" }) + try { + const body = await this.call({ method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ message: text }) }) + if (body === null) return false + this.apply(body) + return true + } catch (error) { + if (this.active) this.publish({ state: "idle", error: error?.message || "Sending StarPort logs is unavailable" }) + return false + } + } +} From 745b1e94de92311f45df18c9353358d37c49f666 Mon Sep 17 00:00:00 2001 From: dirwin31 <83434411+dirwin31@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:26:31 -0800 Subject: [PATCH 04/38] Send the comma's Bluetooth address in the head unit pairing request LeaseAwarePhone (used for every wireless session) had no adapter() method, so Supervisor._adapter_address() hit AttributeError, swallowed it, and returned "". The Bluetooth-service pairing request was therefore never sent (bluetooth_opened always logged phone_address: ""). A 2020 myChevrolet (head unit 2.46.1.6) through an AAWireless Two+ then accepted and acknowledged video but never showed it. A real phone sends the pairing request about 1 s into the session; the car shows Android Auto once its hands-free link to that address is up. Pass adapter() through to the wrapped BluezPhone. It is a read-only property lookup, so it needs no phone-role lease, like devices() and device(). _adapter_address() now logs adapter_address_unavailable instead of failing silently. Tested on the car (session-000052): phone_address sent, pairing response received, picture displayed, touch input worked, 20-29 fps fully acked. Co-authored-by: droobah <34214975+droobah@users.noreply.github.com> --- openpilot/starpilot/system/starport/bluetooth_bridge.py | 1 + openpilot/starpilot/system/starport/supervisor.py | 4 +++- .../starpilot/system/starport/tests/test_bluetooth_bridge.py | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/openpilot/starpilot/system/starport/bluetooth_bridge.py b/openpilot/starpilot/system/starport/bluetooth_bridge.py index 4a8dc08883..7d09cb748b 100644 --- a/openpilot/starpilot/system/starport/bluetooth_bridge.py +++ b/openpilot/starpilot/system/starport/bluetooth_bridge.py @@ -830,6 +830,7 @@ class LeaseAwarePhone: def devices(self): return self.phone.devices() def device(self, address): return self.phone.device(address) def snapshot(self, address): return self.phone.snapshot(address) + def adapter(self): return self.phone.adapter() # this comma's address, for the head unit's pairing request def acquire(self, phone_class: bool = True) -> None: if self.lease is None: diff --git a/openpilot/starpilot/system/starport/supervisor.py b/openpilot/starpilot/system/starport/supervisor.py index e7673e2b69..8e69a8f5c4 100644 --- a/openpilot/starpilot/system/starport/supervisor.py +++ b/openpilot/starpilot/system/starport/supervisor.py @@ -1489,7 +1489,9 @@ class Supervisor: """This comma's Bluetooth address for the head unit's pairing request; empty when BlueZ is not up.""" try: return str(self._bluez.adapter()[1].get("Address", "")).upper() if self._bluez is not None else "" - except Exception: # DBus errors vary; the pairing request is optional + except Exception as error: # DBus errors vary; the pairing request is optional + # Never silent: a missing adapter() once left every wireless session without the request (no address, no picture). + self.log("adapter_address_unavailable", kind=type(error).__name__, error=str(error)[:120]) return "" def _project(self, result, lease, ident, connect=None, fragment_acks: bool = False) -> None: diff --git a/openpilot/starpilot/system/starport/tests/test_bluetooth_bridge.py b/openpilot/starpilot/system/starport/tests/test_bluetooth_bridge.py index 893f630b30..f65e913621 100644 --- a/openpilot/starpilot/system/starport/tests/test_bluetooth_bridge.py +++ b/openpilot/starpilot/system/starport/tests/test_bluetooth_bridge.py @@ -261,8 +261,12 @@ def test_donor_phone_uses_same_lease_and_does_not_take_over_independently(tmp_pa def devices(self): return [] + def adapter(self): + return '/org/bluez/hci0', {'Address': '39:90:21:64:07:00'} + phone = FakePhone() wrapper = LeaseAwarePhone(shared, state['session'], lambda *a, **k: None, lambda log: phone) + assert wrapper.adapter()[1]['Address'] == '39:90:21:64:07:00' # read before any lease, for the pairing request with pytest.raises(BluetoothAdmissionError, match='not active'): wrapper.register_hfp() wrapper.acquire_pairing() From fd660ef6f73b1978de02948081e5e0353a88a99a Mon Sep 17 00:00:00 2001 From: dirwin31 <83434411+dirwin31@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:29:15 -0800 Subject: [PATCH 05/38] Report the chosen setup, the car's offered video modes and the working strategy Reports could not tell a built-in unit from a dongle, and offers and handshakes were only visible in failure text, so there was nothing to compare working pairings against. session_start now records the setup type (builtin/adapter/usb/app), the resolved profile and whether it was chosen or detected. Session reports carry every video mode the head unit offered, whether the comma's address reached the car in the pairing request, and, for streaming sessions, the strategy that worked (window, reply style, Wi-Fi channel type). The drive report gains low-cardinality tags (aa.setup_type, aa.adapter_model, aa.profile_source, aa.phone_address_sent); long values stay in contexts. Old logs without these fields still report. aa.schema is now 3. --- .../system/starport/bugsink_report.py | 69 +++++++++++++++++++ .../system/starport/bugsink_transport.py | 2 +- .../starpilot/system/starport/supervisor.py | 6 +- .../starport/tests/test_bugsink_report.py | 51 ++++++++++++++ 4 files changed, 126 insertions(+), 2 deletions(-) diff --git a/openpilot/starpilot/system/starport/bugsink_report.py b/openpilot/starpilot/system/starport/bugsink_report.py index 4f341c8202..c4860ff879 100644 --- a/openpilot/starpilot/system/starport/bugsink_report.py +++ b/openpilot/starpilot/system/starport/bugsink_report.py @@ -161,6 +161,61 @@ def bounded_trace(trace: list[dict], limit: int) -> dict: return {"events": kept, "omitted": len(trace) - len(kept)} +MAX_OFFERED_MODES = 16 +SETUP_TYPES = ("builtin", "adapter", "usb", "app") + + +def offered_modes(events: list[dict]) -> list[str]: + """Every video mode the head unit offered, e.g. "1280x720 H.264 margins 0x240", from its service discovery. + + Kept for working pairings too, so the choice of mode can later be replayed against real cars.""" + from openpilot.starpilot.system.starport.session import CODEC_H264_BP, CODEC_NAMES, RESOLUTION_NAMES + found = next((event for event in events if event["event"] == "discovered"), {}) + modes: list[str] = [] + for channel in found.get("channels") or []: + for config in (channel.get("video_configs") or []) if isinstance(channel, dict) else []: + if not isinstance(config, dict): + continue + + def field(number, default=None): + values = config.get(str(number), config.get(number)) + return values[0] if isinstance(values, list) and values and isinstance(values[0], int) else default + + resolution, codec = field(1), field(10, CODEC_H264_BP) + text = f"{RESOLUTION_NAMES.get(resolution, f'resolution {resolution}')} {CODEC_NAMES.get(codec, f'codec {codec}')}" + if field(3, 0) or field(4, 0): + text += f" margins {field(3, 0)}x{field(4, 0)}" + modes.append(text) + return modes[:MAX_OFFERED_MODES] + + +def session_setup(events: list[dict]) -> dict: + """The setup type and profile the owner chose, as recorded when the session started.""" + start = next((event for event in events if event["event"] == "session_start"), {}) + setup = start.get("setup") if isinstance(start.get("setup"), dict) else {} + kind = setup.get("type") + return {key: value for key, value in ( + ("type", kind if kind in SETUP_TYPES else None), + ("profile", scrub(setup.get("profile", ""), 40) or None), + ("profile_source", setup.get("profile_source") if setup.get("profile_source") in ("override", "detected", "standard") else None), + ) if value} + + +def session_strategy(events: list[dict], setup: dict) -> dict: + """What a streaming session ran with; the raw material for "this head unit usually likes this profile".""" + if not any(event["event"] == "video_acknowledged" for event in events): + return {} + ready = next((event for event in reversed(events) if event["event"] == "video_setup"), {}) + reply = next((event for event in events if event["event"] == "bootstrap_version_reply"), {}) + mode = ready.get("mode") if isinstance(ready.get("mode"), dict) else {} + # Only a wireless bootstrap has a version reply; the modern one is logged, the standard one is its absence. + style = reply.get("style") or ("standard" if any(event["event"] == "bootstrap_start_response" for event in events) else None) + strategy = {"profile": setup.get("profile"), "window_offered": ready.get("window"), "window_used": ready.get("used_window"), + "reply_style": style, "channel_type": reply.get("channel_type"), + "config_index": mode.get("config_index")} + return {key: value for key, value in strategy.items() if value is not None} + + def session_record(path: Path) -> dict: """One finished session log reduced to its outcome, stage timings, video and the car it talked to.""" from openpilot.starpilot.system.starport import compat_report @@ -184,7 +239,14 @@ def session_record(path: Path) -> dict: car = report["car"] start = next((e for e in events if e["event"] == "session_start"), {}) software, settings = start.get("software", {}), start.get("settings") + setup = session_setup(events) + opened = next((e for e in events if e["event"] == "bluetooth_opened"), None) record = { + "setup": setup, + "offered_modes": offered_modes(events), + # True when the comma's address went to the car in the pairing request; None when the car never opened that channel. + "phone_address_sent": bool(opened.get("phone_address")) if opened is not None else None, + "strategy": session_strategy(events, setup), "software": {key: str(software.get(key) or "unknown") for key in ("commit", "branch")} if isinstance(software, dict) else {}, "settings": settings if isinstance(settings, dict) else {}, "seq": _session_number(path), @@ -395,6 +457,8 @@ def drive_event(sessions: list[dict]) -> dict: head_unit = next((s["head_unit"] for s in reversed(sessions) if s["head_unit"]), {}) video = next((s["video"] for s in reversed(sessions) if s["video"].get("resolution")), {}) label = _head_unit_label(head_unit) + setup = next((s["setup"] for s in reversed(sessions) if s.get("setup")), {}) + sent = next((s["phone_address_sent"] for s in reversed(sessions) if s.get("phone_address_sent") is not None), None) failures = [failure_detail(failure) for failure in drive_failures(sessions)] failed = [session for session in sessions if session["seq"] in {failure["session"] for failure in failures}] heading = "failures" if failures else "sessions" @@ -425,6 +489,11 @@ def drive_event(sessions: list[dict]) -> dict: "aa.encoder": video.get("encoder", "unknown"), "aa.fps": str(summary["fps_avg"]) if summary["fps_avg"] is not None else "none", "aa.failures": str(len(failures)), + # Low-cardinality choices only; long values (offered modes, strategy) live in the contexts below. + **({"aa.setup_type": setup["type"]} if setup.get("type") else {}), + **({"aa.adapter_model": setup["profile"]} if setup.get("profile") else {}), + **({"aa.profile_source": setup["profile_source"]} if setup.get("profile_source") else {}), + **({"aa.phone_address_sent": "yes" if sent else "no"} if sent is not None else {}), **({"aa.attention": ",".join(reasons)[:190]} if reasons else {}), **({"aa.cause": latest["cause"], "aa.stage": latest["stage"], "aa.error_kind": latest["error_kind"]} if latest else {})}, "breadcrumbs": {"values": attempt_breadcrumbs(sessions)}, diff --git a/openpilot/starpilot/system/starport/bugsink_transport.py b/openpilot/starpilot/system/starport/bugsink_transport.py index 0f5acf7364..969ba752ef 100644 --- a/openpilot/starpilot/system/starport/bugsink_transport.py +++ b/openpilot/starpilot/system/starport/bugsink_transport.py @@ -38,7 +38,7 @@ class Sender: self.opener = opener or urllib.request.build_opener(NoRedirect()) self.active = active self.logger, self.client = logger, client - self.schema_tags = {"aa.schema": "2"} if schema_tags is None else schema_tags + self.schema_tags = {"aa.schema": "3"} if schema_tags is None else schema_tags def __call__(self, **event) -> bool: if not self.active(): diff --git a/openpilot/starpilot/system/starport/supervisor.py b/openpilot/starpilot/system/starport/supervisor.py index 8e69a8f5c4..20152d392a 100644 --- a/openpilot/starpilot/system/starport/supervisor.py +++ b/openpilot/starpilot/system/starport/supervisor.py @@ -1134,9 +1134,13 @@ class Supervisor: receiver = {"wired": "usb", "headunitpad": "headunitpad", "openheadunit": "openheadunit"}.get(connection, self.config["receiver_name"]) from openpilot.starpilot.system.starport.bugsink_transport import device_metadata software = device_metadata() + profile, profile_source = car_profiles.resolve(self.config) self.log("session_start", receiver=receiver, generation=generation, trigger=trigger, software={key: software.get(key, "unknown") for key in ("commit", "branch")}, - settings={key: self.config.get(key) for key in REPORTED_SETTINGS}) + settings={key: self.config.get(key) for key in REPORTED_SETTINGS}, + # What the owner chose, so a report can tell a built-in unit from a dongle and say which profile applied. + setup={"type": {"wired": "usb", "headunitpad": "app", "openheadunit": "app"}.get(connection) or self.car_type(), + "profile": profile, "profile_source": profile_source}) lease = self._lease() if wireless else NoLease() attempt = 0 car_first = trigger == "onroad" and wireless diff --git a/openpilot/starpilot/system/starport/tests/test_bugsink_report.py b/openpilot/starpilot/system/starport/tests/test_bugsink_report.py index d03e3e2e63..3221f8114b 100644 --- a/openpilot/starpilot/system/starport/tests/test_bugsink_report.py +++ b/openpilot/starpilot/system/starport/tests/test_bugsink_report.py @@ -441,3 +441,54 @@ def test_pairing_that_failed_carries_a_device_slice_but_success_does_not(tmp_pat assert event["contexts"]["starport_pairing_system"]["bluetooth"]["lines"] == ["bt '' up"] assert calls[0][1] - calls[0][0] == 30 assert "starport_pairing_system" not in br.pairing_event({**record, "result": "paired"}, collector)["contexts"] + + +OFFER = {"channels": [{"id": 3, "video_configs": [ + {"1": [2], "10": [3], "3": [0], "4": [240]}, + {"1": [8], "10": [7], "3": [288], "4": [1378]}, + {"1": [7], "10": [7], "3": [0], "4": [811]}]}], "head_unit": {"2": ["Honda"]}} + + +def adapter_session(): + events = [list(item) for item in STREAMED] + events[0] = (0, 0, "session_start", {"receiver": "AAWireless", "trigger": "auto", + "setup": {"type": "adapter", "profile": "aawireless", "profile_source": "override"}}) + events[5] = (0, 10, "discovered", {**OFFER, "head_unit": STREAMED[5][3]["head_unit"]}) + events.insert(6, (0, 10, "bootstrap_start_response", {})) + events.insert(7, (0, 10, "bootstrap_version_reply", {"style": "modern", "channel_type": 1})) + events.insert(8, (0, 10, "bluetooth_opened", {"channel": 6, "phone_address": "39:90:21:64:07:00"})) + events.insert(9, (0, 11, "video_setup", {"mode": {"config_index": 0}, "window": 30, "used_window": 30})) + return events + + +def test_report_keeps_setup_offered_modes_and_the_working_strategy(tmp_path): + record = br.session_record(write_session(tmp_path, 1, adapter_session())) + assert record["setup"] == {"type": "adapter", "profile": "aawireless", "profile_source": "override"} + assert record["offered_modes"] == ["1280x720 H.264 margins 0x240", "1440x2560 H.265 margins 288x1378", "1080x1920 H.265 margins 0x811"] + assert record["phone_address_sent"] is True + assert record["strategy"] == {"profile": "aawireless", "window_offered": 30, "window_used": 30, + "reply_style": "modern", "channel_type": 1, "config_index": 0} + event = br.drive_event([record]) + tags = event["tags"] + assert (tags["aa.setup_type"], tags["aa.adapter_model"], tags["aa.profile_source"], tags["aa.phone_address_sent"]) == ( + "adapter", "aawireless", "override", "yes") + context = event["contexts"]["starport_session_1"] + assert context["offered_modes"] == record["offered_modes"] and context["strategy"]["reply_style"] == "modern" + assert "39:90:21:64:07:00" not in json.dumps(event) # the comma's own address is sanitized from every field + + +def test_report_records_that_the_address_was_not_sent_and_old_logs_still_report(tmp_path): + events = [list(item) for item in STREAMED] + events.insert(6, (0, 10, "bluetooth_opened", {"channel": 6, "phone_address": ""})) + assert br.drive_event([br.session_record(write_session(tmp_path, 1, events))])["tags"]["aa.phone_address_sent"] == "no" + old = br.session_record(write_session(tmp_path, 2, STREAMED)) # a log written before these fields existed + assert old["setup"] == {} and old["offered_modes"] == [] and old["phone_address_sent"] is None and old["strategy"] == {} + tags = br.drive_event([old])["tags"] + assert not any(key in tags for key in ("aa.setup_type", "aa.adapter_model", "aa.profile_source", "aa.phone_address_sent")) + + +def test_unknown_setup_values_are_dropped(tmp_path): + events = [list(item) for item in STREAMED] + events[0] = (0, 0, "session_start", {"setup": {"type": "