From 900e312687cf48db91a263da6767f17eaf4a7e6b Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 13 Apr 2024 17:21:07 +0000 Subject: [PATCH 1/5] Add request_queued method to SunnylinkApi --- CHANGELOGS.md | 16 ++++++++-------- common/api/sunnylink.py | 4 ++++ selfdrive/athena/sunnylinkd.py | 17 ++++++++++++----- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index 83d7da1c71..18c0a9df45 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -5,17 +5,17 @@ sunnypilot - 0.9.7.0 (2024-xx-xx) ************************ * UPDATED: Synced with commaai's openpilot * master commit 56e343b (February 27, 2024) -* NEW❗: Config Backup (Alpha early access) - * Remotely back up and restore sunnypilot settings easily - * Device registration with sunnylink ensures a secure, integrated experience across services - * AES encryption derived from the device's RSA private key is used for utmost security - * Settings are encrypted on-device, transmitted securely via HTTPS, and stored encrypted on sunnylink - * Prevents loss of settings after device resets, offering peace of mind through end-to-end encryption - * Early alpha access to all current and previous GitHub Sponsors and Patreon supporters +* NEW❗: sunnylink (Alpha early access) + * NEW❗: Config Backup + * Remotely back up and restore sunnypilot settings easily + * Device registration with sunnylink ensures a secure, integrated experience across services + * AES encryption derived from the device's RSA private key is used for utmost security + * Settings are encrypted on-device, transmitted securely via HTTPS, and stored encrypted on sunnylink + * Prevents loss of settings after device resets, offering peace of mind through end-to-end encryption + * Early alpha access to all current and previous GitHub Sponsors and Patreon supporters * GitHub account pairing from device settings scanning QR code * Pairing your account will allow you to access features via our API (still WIP but accessible if you dig a little on our code 😉) * Allow inheritance of your sponsorship status, allowing you to get extra features and early access whenever applicable - * Immediate sponsor recognition works only for new sponsors. If you're an earlier sponsor, we've got you — just reach out to a moderator on Discord (https://discord.gg/sunnypilot) to get sorted! * RE-ENABLED: Map-based Turn Speed Control (M-TSC) for supported platforms * openpilot Longitudianl Control available cars * Custom Stock Longitudinal Control available cars diff --git a/common/api/sunnylink.py b/common/api/sunnylink.py index c2ac89d6dd..46f5715c88 100644 --- a/common/api/sunnylink.py +++ b/common/api/sunnylink.py @@ -28,6 +28,10 @@ class SunnylinkApi(BaseApi): return super().api_get(endpoint, method, timeout, **kwargs) + def resume_queued(self, timeout=10, **kwargs): + sunnylinkId, commaId = self._resolve_dongle_ids() + return self.api_get(f"ws/{sunnylinkId}/resume_queued", "POST", timeout, access_token=self.get_token(), **kwargs) + def get_token(self, expiry_hours=1): # Add your additional data here additional_data = {} diff --git a/selfdrive/athena/sunnylinkd.py b/selfdrive/athena/sunnylinkd.py index f08cf2797b..479b9fb311 100755 --- a/selfdrive/athena/sunnylinkd.py +++ b/selfdrive/athena/sunnylinkd.py @@ -21,7 +21,8 @@ SUNNYLINK_ATHENA_HOST = os.getenv('SUNNYLINK_ATHENA_HOST', 'wss://ws.stg.api.sun HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) LOCAL_PORT_WHITELIST = {8022} - +params = Params() +sunnylink_api = SunnylinkApi(params.get("SunnylinkDongleId", encoding='utf-8')) def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None: end_event = threading.Event() @@ -55,7 +56,16 @@ def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None: def ws_recv(ws: WebSocket, end_event: threading.Event) -> None: last_ping = int(time.monotonic() * 1e9) + resume_requested = False while not end_event.is_set(): + try: + if(not resume_requested): + sunnylink_api.resume_queued() + resume_requested = True + except Exception: + cloudlog.exception("sunnylinkd.resume_queued.exception") + resume_requested = False + try: opcode, data = ws.recv_data(control_frame=True) if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY): @@ -92,12 +102,9 @@ def main(exit_event: threading.Event = None): except Exception: cloudlog.exception("failed to set core affinity") - params = Params() - dongle_id = params.get("SunnylinkDongleId", encoding='utf-8') UploadQueueCache.initialize(upload_queue) ws_uri = SUNNYLINK_ATHENA_HOST - api = SunnylinkApi(dongle_id) conn_start = None conn_retries = 0 while exit_event is None or not exit_event.is_set(): @@ -107,7 +114,7 @@ def main(exit_event: threading.Event = None): cloudlog.event("sunnylinkd.main.connecting_ws", ws_uri=ws_uri, retries=conn_retries) ws = create_connection(ws_uri, - cookie="jwt=" + api.get_token(), + cookie="jwt=" + sunnylink_api.get_token(), enable_multithread=True, timeout=30.0) cloudlog.event("sunnylinkd.main.connected_ws", ws_uri=ws_uri, retries=conn_retries, From 0e237f459c420fec14e50beb639c30bfc9ebdd26 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Sat, 13 Apr 2024 17:22:33 +0000 Subject: [PATCH 2/5] sunnylink: Sync Settings --- common/params_pyx.pyx | 5 +++++ selfdrive/athena/sunnylinkd.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/common/params_pyx.pyx b/common/params_pyx.pyx index 535514e521..29fe354ca1 100644 --- a/common/params_pyx.pyx +++ b/common/params_pyx.pyx @@ -3,6 +3,7 @@ from libcpp cimport bool from libcpp.string cimport string from libcpp.vector cimport vector +from libcpp.map cimport map cdef extern from "common/params.h": cpdef enum ParamKeyType: @@ -26,6 +27,7 @@ cdef extern from "common/params.h": string getParamPath(string) nogil void clearAll(ParamKeyType) vector[string] allKeys() + map[string, string] readAll() def ensure_bytes(v): @@ -116,3 +118,6 @@ cdef class Params: def all_keys(self): return self.p.allKeys() + + def read_all(self): + return self.p.readAll() diff --git a/selfdrive/athena/sunnylinkd.py b/selfdrive/athena/sunnylinkd.py index 479b9fb311..bb9ae2ecae 100755 --- a/selfdrive/athena/sunnylinkd.py +++ b/selfdrive/athena/sunnylinkd.py @@ -3,12 +3,16 @@ from __future__ import annotations +import base64 +import gzip +import json import os import threading import time from openpilot.selfdrive.athena.athenad import ws_send, jsonrpc_handler, \ recv_queue, RECONNECT_TIMEOUT_S, UploadQueueCache, upload_queue, cur_upload_items, backoff, ws_manage +from jsonrpc import dispatcher from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException, create_connection) @@ -96,6 +100,43 @@ def ws_ping(ws: WebSocket, end_event: threading.Event) -> None: time.sleep(RECONNECT_TIMEOUT_S * 0.8) # Sleep about 80% before a timeout +@dispatcher.add_method +def getParamsAllKeys() -> list[str]: + keys: list[str] = [k.decode('utf-8') for k in Params().all_keys()] + return keys + + +@dispatcher.add_method +def getParams(params_keys: list[str], compression: bool = False) -> str | dict[str, str]: + try: + params = Params() + params_dict: dict[str, bytes] = {key: params.get(key) or b'' for key in params_keys} + + # Compress the values before encoding to base64 as output from params.get is bytes and same for compression + if compression: + params_dict = {key: gzip.compress(value) for key, value in params_dict.items()} + + # Last step is to encode the values to base64 and decode to utf-8 for JSON serialization + return {key: base64.b64encode(value).decode('utf-8') for key, value in params_dict.items()} + + except Exception as e: + return cloudlog.exception("sunnylinkd.getParams.exception", e) + +@dispatcher.add_method +def saveParams(params_to_update: dict[str, str], compression: bool = False) -> None: + params = Params() + try: + params_dict = {key: base64.b64decode(value) for key, value in params_to_update.items()} + + if compression: + params_dict = {key: gzip.decompress(value) for key, value in params_dict.items()} + + for key, value in params_dict.items(): + params.put(key, value) + except Exception as e: + return cloudlog.exception("sunnylinkd.saveParams.exception", e) + + def main(exit_event: threading.Event = None): try: set_core_affinity([0, 1, 2, 3]) From 53f50e53ef2805b4ad8d4e4874d7fbde7b075dc2 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 13 Apr 2024 17:40:17 +0000 Subject: [PATCH 3/5] Add easter egg for specific Sunnylink device ID --- selfdrive/ui/qt/offroad/sunnypilot/sunnylink_settings.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/selfdrive/ui/qt/offroad/sunnypilot/sunnylink_settings.cc b/selfdrive/ui/qt/offroad/sunnypilot/sunnylink_settings.cc index 935ed3fc56..e765792b4d 100644 --- a/selfdrive/ui/qt/offroad/sunnypilot/sunnylink_settings.cc +++ b/selfdrive/ui/qt/offroad/sunnypilot/sunnylink_settings.cc @@ -200,6 +200,11 @@ void SunnylinkPanel::updateLabels() { bool is_paired = uiState()->isSunnylinkPaired(); auto paired_users = uiState()->sunnylinkDeviceUsers(); + //little easter egg for Panda :D + if(sunnylinkDongleId == "d689627422cefcbc") { + role_name = "Panda 🐼"; + } + sunnylinkEnabledBtn->setEnabled(!is_onroad); sunnylinkEnabledBtn->setValue(tr("Device ID ")+ sunnylinkDongleId); From fe023d946f8708eca85ce828641412d7ed01e1d3 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sun, 14 Apr 2024 10:11:08 +0000 Subject: [PATCH 4/5] Add new thread for handling sunnylink queue --- selfdrive/athena/sunnylinkd.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/selfdrive/athena/sunnylinkd.py b/selfdrive/athena/sunnylinkd.py index bb9ae2ecae..ca924abeb0 100755 --- a/selfdrive/athena/sunnylinkd.py +++ b/selfdrive/athena/sunnylinkd.py @@ -35,6 +35,7 @@ def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None: threading.Thread(target=ws_recv, args=(ws, end_event), name='ws_recv'), threading.Thread(target=ws_send, args=(ws, end_event), name='ws_send'), threading.Thread(target=ws_ping, args=(ws, end_event), name='ws_ping'), + threading.Thread(target=ws_queue, args=(end_event,), name='ws_queue'), # threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler'), # threading.Thread(target=log_handler, args=(end_event,), name='log_handler'), # threading.Thread(target=stat_handler, args=(end_event,), name='stat_handler'), @@ -60,16 +61,7 @@ def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None: def ws_recv(ws: WebSocket, end_event: threading.Event) -> None: last_ping = int(time.monotonic() * 1e9) - resume_requested = False while not end_event.is_set(): - try: - if(not resume_requested): - sunnylink_api.resume_queued() - resume_requested = True - except Exception: - cloudlog.exception("sunnylinkd.resume_queued.exception") - resume_requested = False - try: opcode, data = ws.recv_data(control_frame=True) if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY): @@ -99,6 +91,22 @@ def ws_ping(ws: WebSocket, end_event: threading.Event) -> None: end_event.set() time.sleep(RECONNECT_TIMEOUT_S * 0.8) # Sleep about 80% before a timeout +def ws_queue(end_event: threading.Event) -> None: + resume_requested = False + backoff_time = 1 # Start with a delay of 1 second + max_backoff_time = 60 # Maximum delay of 60 seconds + + while not end_event.is_set(): + try: + if not resume_requested: + sunnylink_api.resume_queued(timeout=29) + resume_requested = True + backoff_time = 1 # Reset backoff time after a successful request + except Exception: + cloudlog.exception("sunnylinkd.ws_queue.resume_queued.exception") + resume_requested = False + time.sleep(backoff_time) # Wait for the backoff time before the next attempt + backoff_time = min(backoff_time * 2, max_backoff_time) # Double the backoff time for the next attempt, up to a maximum @dispatcher.add_method def getParamsAllKeys() -> list[str]: From 39bcc41f9f0fe9a32e7b3a820a00a79b9894c8ea Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Mon, 15 Apr 2024 19:24:40 +0000 Subject: [PATCH 5/5] [sunnylink] Integrate debug logging and revise retry mechanism in sunnylinkd.py --- selfdrive/athena/sunnylinkd.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/selfdrive/athena/sunnylinkd.py b/selfdrive/athena/sunnylinkd.py index ca924abeb0..9c26dd42a1 100755 --- a/selfdrive/athena/sunnylinkd.py +++ b/selfdrive/athena/sunnylinkd.py @@ -68,7 +68,9 @@ def ws_recv(ws: WebSocket, end_event: threading.Event) -> None: if opcode == ABNF.OPCODE_TEXT: data = data.decode("utf-8") recv_queue.put_nowait(data) + cloudlog.debug(f"sunnylinkd.ws_recv.recv {data}") elif opcode in (ABNF.OPCODE_PING, ABNF.OPCODE_PONG): + cloudlog.debug(f"sunnylinkd.ws_recv.pong {opcode}") last_ping = int(time.monotonic() * 1e9) Params().put("LastSunnylinkPingTime", str(last_ping)) except WebSocketTimeoutException: @@ -86,6 +88,7 @@ def ws_ping(ws: WebSocket, end_event: threading.Event) -> None: while not end_event.is_set(): try: ws.ping() + cloudlog.debug(f"sunnylinkd.ws_recv.ws_ping: Pinging") except Exception: cloudlog.exception("sunnylinkd.ws_ping.exception") end_event.set() @@ -93,20 +96,22 @@ def ws_ping(ws: WebSocket, end_event: threading.Event) -> None: def ws_queue(end_event: threading.Event) -> None: resume_requested = False - backoff_time = 1 # Start with a delay of 1 second - max_backoff_time = 60 # Maximum delay of 60 seconds + tries = 0 - while not end_event.is_set(): + while not end_event.is_set() and not resume_requested: try: if not resume_requested: + cloudlog.debug(f"sunnylinkd.ws_queue.resume_queued") sunnylink_api.resume_queued(timeout=29) resume_requested = True - backoff_time = 1 # Reset backoff time after a successful request + tries = 0 except Exception: cloudlog.exception("sunnylinkd.ws_queue.resume_queued.exception") resume_requested = False - time.sleep(backoff_time) # Wait for the backoff time before the next attempt - backoff_time = min(backoff_time * 2, max_backoff_time) # Double the backoff time for the next attempt, up to a maximum + tries += 1 + time.sleep(backoff(tries)) # Wait for the backoff time before the next attempt + cloudlog.debug("Resume requested or end_event is set, exiting ws_queue thread") + @dispatcher.add_method def getParamsAllKeys() -> list[str]: