mirror of
https://github.com/dragonpilot/dragonpilot.git
synced 2026-10-01 03:43:42 +08:00
Merge branch 'master' of https://github.com/commaai/openpilot into master3
This commit is contained in:
Rick Lan
Executable
+826
@@ -0,0 +1,826 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import bz2
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import random
|
||||
import select
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from queue import Queue
|
||||
from typing import cast
|
||||
from collections.abc import Callable
|
||||
|
||||
import requests
|
||||
from jsonrpc import JSONRPCResponseManager, dispatcher
|
||||
from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException,
|
||||
create_connection)
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from cereal import log
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.api import Api
|
||||
from openpilot.common.file_helpers import CallbackReader
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import set_core_affinity
|
||||
from openpilot.system.hardware import HARDWARE, PC
|
||||
from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.version import get_build_metadata
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
|
||||
ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai')
|
||||
HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4"))
|
||||
LOCAL_PORT_WHITELIST = {8022}
|
||||
|
||||
LOG_ATTR_NAME = 'user.upload'
|
||||
LOG_ATTR_VALUE_MAX_UNIX_TIME = int.to_bytes(2147483647, 4, sys.byteorder)
|
||||
RECONNECT_TIMEOUT_S = 70
|
||||
|
||||
RETRY_DELAY = 10 # seconds
|
||||
MAX_RETRY_COUNT = 30 # Try for at most 5 minutes if upload fails immediately
|
||||
MAX_AGE = 31 * 24 * 3600 # seconds
|
||||
WS_FRAME_SIZE = 4096
|
||||
|
||||
NetworkType = log.DeviceState.NetworkType
|
||||
|
||||
UploadFileDict = dict[str, str | int | float | bool]
|
||||
UploadItemDict = dict[str, str | bool | int | float | dict[str, str]]
|
||||
|
||||
UploadFilesToUrlResponse = dict[str, int | list[UploadItemDict] | list[str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class UploadFile:
|
||||
fn: str
|
||||
url: str
|
||||
headers: dict[str, str]
|
||||
allow_cellular: bool
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> UploadFile:
|
||||
return cls(d.get("fn", ""), d.get("url", ""), d.get("headers", {}), d.get("allow_cellular", False))
|
||||
|
||||
|
||||
@dataclass
|
||||
class UploadItem:
|
||||
path: str
|
||||
url: str
|
||||
headers: dict[str, str]
|
||||
created_at: int
|
||||
id: str | None
|
||||
retry_count: int = 0
|
||||
current: bool = False
|
||||
progress: float = 0
|
||||
allow_cellular: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> UploadItem:
|
||||
return cls(d["path"], d["url"], d["headers"], d["created_at"], d["id"], d["retry_count"], d["current"],
|
||||
d["progress"], d["allow_cellular"])
|
||||
|
||||
|
||||
dispatcher["echo"] = lambda s: s
|
||||
recv_queue: Queue[str] = queue.Queue()
|
||||
send_queue: Queue[str] = queue.Queue()
|
||||
upload_queue: Queue[UploadItem] = queue.Queue()
|
||||
low_priority_send_queue: Queue[str] = queue.Queue()
|
||||
log_recv_queue: Queue[str] = queue.Queue()
|
||||
cancelled_uploads: set[str] = set()
|
||||
|
||||
cur_upload_items: dict[int, UploadItem | None] = {}
|
||||
|
||||
|
||||
def strip_bz2_extension(fn: str) -> str:
|
||||
if fn.endswith('.bz2'):
|
||||
return fn[:-4]
|
||||
return fn
|
||||
|
||||
|
||||
class AbortTransferException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UploadQueueCache:
|
||||
|
||||
@staticmethod
|
||||
def initialize(upload_queue: Queue[UploadItem]) -> None:
|
||||
try:
|
||||
upload_queue_json = Params().get("AthenadUploadQueue")
|
||||
if upload_queue_json is not None:
|
||||
for item in json.loads(upload_queue_json):
|
||||
upload_queue.put(UploadItem.from_dict(item))
|
||||
except Exception:
|
||||
cloudlog.exception("athena.UploadQueueCache.initialize.exception")
|
||||
|
||||
@staticmethod
|
||||
def cache(upload_queue: Queue[UploadItem]) -> None:
|
||||
try:
|
||||
queue: list[UploadItem | None] = list(upload_queue.queue)
|
||||
items = [asdict(i) for i in queue if i is not None and (i.id not in cancelled_uploads)]
|
||||
Params().put("AthenadUploadQueue", json.dumps(items))
|
||||
except Exception:
|
||||
cloudlog.exception("athena.UploadQueueCache.cache.exception")
|
||||
|
||||
|
||||
def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None:
|
||||
end_event = threading.Event()
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=ws_manage, args=(ws, end_event), name='ws_manage'),
|
||||
threading.Thread(target=ws_recv, args=(ws, end_event), name='ws_recv'),
|
||||
threading.Thread(target=ws_send, args=(ws, end_event), name='ws_send'),
|
||||
threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler'),
|
||||
threading.Thread(target=log_handler, args=(end_event,), name='log_handler'),
|
||||
threading.Thread(target=stat_handler, args=(end_event,), name='stat_handler'),
|
||||
] + [
|
||||
threading.Thread(target=jsonrpc_handler, args=(end_event,), name=f'worker_{x}')
|
||||
for x in range(HANDLER_THREADS)
|
||||
]
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
try:
|
||||
while not end_event.wait(0.1):
|
||||
if exit_event is not None and exit_event.is_set():
|
||||
end_event.set()
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
end_event.set()
|
||||
raise
|
||||
finally:
|
||||
for thread in threads:
|
||||
cloudlog.debug(f"athena.joining {thread.name}")
|
||||
thread.join()
|
||||
|
||||
|
||||
def jsonrpc_handler(end_event: threading.Event) -> None:
|
||||
dispatcher["startLocalProxy"] = partial(startLocalProxy, end_event)
|
||||
while not end_event.is_set():
|
||||
try:
|
||||
data = recv_queue.get(timeout=1)
|
||||
if "method" in data:
|
||||
cloudlog.event("athena.jsonrpc_handler.call_method", data=data)
|
||||
response = JSONRPCResponseManager.handle(data, dispatcher)
|
||||
send_queue.put_nowait(response.json)
|
||||
elif "id" in data and ("result" in data or "error" in data):
|
||||
log_recv_queue.put_nowait(data)
|
||||
else:
|
||||
raise Exception("not a valid request or response")
|
||||
except queue.Empty:
|
||||
pass
|
||||
except Exception as e:
|
||||
cloudlog.exception("athena jsonrpc handler failed")
|
||||
send_queue.put_nowait(json.dumps({"error": str(e)}))
|
||||
|
||||
|
||||
def retry_upload(tid: int, end_event: threading.Event, increase_count: bool = True) -> None:
|
||||
item = cur_upload_items[tid]
|
||||
if item is not None and item.retry_count < MAX_RETRY_COUNT:
|
||||
new_retry_count = item.retry_count + 1 if increase_count else item.retry_count
|
||||
|
||||
item = replace(
|
||||
item,
|
||||
retry_count=new_retry_count,
|
||||
progress=0,
|
||||
current=False
|
||||
)
|
||||
upload_queue.put_nowait(item)
|
||||
UploadQueueCache.cache(upload_queue)
|
||||
|
||||
cur_upload_items[tid] = None
|
||||
|
||||
for _ in range(RETRY_DELAY):
|
||||
time.sleep(1)
|
||||
if end_event.is_set():
|
||||
break
|
||||
|
||||
|
||||
def cb(sm, item, tid, end_event: threading.Event, sz: int, cur: int) -> None:
|
||||
# Abort transfer if connection changed to metered after starting upload
|
||||
# or if athenad is shutting down to re-connect the websocket
|
||||
sm.update(0)
|
||||
metered = sm['deviceState'].networkMetered
|
||||
if metered and (not item.allow_cellular):
|
||||
raise AbortTransferException
|
||||
|
||||
if end_event.is_set():
|
||||
raise AbortTransferException
|
||||
|
||||
cur_upload_items[tid] = replace(item, progress=cur / sz if sz else 1)
|
||||
|
||||
|
||||
def upload_handler(end_event: threading.Event) -> None:
|
||||
sm = messaging.SubMaster(['deviceState'])
|
||||
tid = threading.get_ident()
|
||||
|
||||
while not end_event.is_set():
|
||||
cur_upload_items[tid] = None
|
||||
|
||||
try:
|
||||
cur_upload_items[tid] = item = replace(upload_queue.get(timeout=1), current=True)
|
||||
|
||||
if item.id in cancelled_uploads:
|
||||
cancelled_uploads.remove(item.id)
|
||||
continue
|
||||
|
||||
# Remove item if too old
|
||||
age = datetime.now() - datetime.fromtimestamp(item.created_at / 1000)
|
||||
if age.total_seconds() > MAX_AGE:
|
||||
cloudlog.event("athena.upload_handler.expired", item=item, error=True)
|
||||
continue
|
||||
|
||||
# Check if uploading over metered connection is allowed
|
||||
sm.update(0)
|
||||
metered = sm['deviceState'].networkMetered
|
||||
network_type = sm['deviceState'].networkType.raw
|
||||
if metered and (not item.allow_cellular):
|
||||
retry_upload(tid, end_event, False)
|
||||
continue
|
||||
|
||||
try:
|
||||
fn = item.path
|
||||
try:
|
||||
sz = os.path.getsize(fn)
|
||||
except OSError:
|
||||
sz = -1
|
||||
|
||||
cloudlog.event("athena.upload_handler.upload_start", fn=fn, sz=sz, network_type=network_type, metered=metered, retry_count=item.retry_count)
|
||||
response = _do_upload(item, partial(cb, sm, item, tid, end_event))
|
||||
|
||||
if response.status_code not in (200, 201, 401, 403, 412):
|
||||
cloudlog.event("athena.upload_handler.retry", status_code=response.status_code, fn=fn, sz=sz, network_type=network_type, metered=metered)
|
||||
retry_upload(tid, end_event)
|
||||
else:
|
||||
cloudlog.event("athena.upload_handler.success", fn=fn, sz=sz, network_type=network_type, metered=metered)
|
||||
|
||||
UploadQueueCache.cache(upload_queue)
|
||||
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.SSLError):
|
||||
cloudlog.event("athena.upload_handler.timeout", fn=fn, sz=sz, network_type=network_type, metered=metered)
|
||||
retry_upload(tid, end_event)
|
||||
except AbortTransferException:
|
||||
cloudlog.event("athena.upload_handler.abort", fn=fn, sz=sz, network_type=network_type, metered=metered)
|
||||
retry_upload(tid, end_event, False)
|
||||
|
||||
except queue.Empty:
|
||||
pass
|
||||
except Exception:
|
||||
cloudlog.exception("athena.upload_handler.exception")
|
||||
|
||||
|
||||
def _do_upload(upload_item: UploadItem, callback: Callable = None) -> requests.Response:
|
||||
path = upload_item.path
|
||||
compress = False
|
||||
|
||||
# If file does not exist, but does exist without the .bz2 extension we will compress on the fly
|
||||
if not os.path.exists(path) and os.path.exists(strip_bz2_extension(path)):
|
||||
path = strip_bz2_extension(path)
|
||||
compress = True
|
||||
|
||||
with open(path, "rb") as f:
|
||||
content = f.read()
|
||||
if compress:
|
||||
cloudlog.event("athena.upload_handler.compress", fn=path, fn_orig=upload_item.path)
|
||||
content = bz2.compress(content)
|
||||
|
||||
with io.BytesIO(content) as data:
|
||||
return requests.put(upload_item.url,
|
||||
data=CallbackReader(data, callback, len(content)) if callback else data,
|
||||
headers={**upload_item.headers, 'Content-Length': str(len(content))},
|
||||
timeout=30)
|
||||
|
||||
|
||||
# security: user should be able to request any message from their car
|
||||
@dispatcher.add_method
|
||||
def getMessage(service: str, timeout: int = 1000) -> dict:
|
||||
if service is None or service not in SERVICE_LIST:
|
||||
raise Exception("invalid service")
|
||||
|
||||
socket = messaging.sub_sock(service, timeout=timeout)
|
||||
ret = messaging.recv_one(socket)
|
||||
|
||||
if ret is None:
|
||||
raise TimeoutError
|
||||
|
||||
# this is because capnp._DynamicStructReader doesn't have typing information
|
||||
return cast(dict, ret.to_dict())
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getVersion() -> dict[str, str]:
|
||||
build_metadata = get_build_metadata()
|
||||
return {
|
||||
"version": build_metadata.openpilot.version,
|
||||
"remote": build_metadata.openpilot.git_normalized_origin,
|
||||
"branch": build_metadata.channel,
|
||||
"commit": build_metadata.openpilot.git_commit,
|
||||
}
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def setNavDestination(latitude: int = 0, longitude: int = 0, place_name: str = None, place_details: str = None) -> dict[str, int]:
|
||||
destination = {
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"place_name": place_name,
|
||||
"place_details": place_details,
|
||||
}
|
||||
Params().put("NavDestination", json.dumps(destination))
|
||||
|
||||
return {"success": 1}
|
||||
|
||||
|
||||
def scan_dir(path: str, prefix: str) -> list[str]:
|
||||
files = []
|
||||
# only walk directories that match the prefix
|
||||
# (glob and friends traverse entire dir tree)
|
||||
with os.scandir(path) as i:
|
||||
for e in i:
|
||||
rel_path = os.path.relpath(e.path, Paths.log_root())
|
||||
if e.is_dir(follow_symlinks=False):
|
||||
# add trailing slash
|
||||
rel_path = os.path.join(rel_path, '')
|
||||
# if prefix is a partial dir name, current dir will start with prefix
|
||||
# if prefix is a partial file name, prefix with start with dir name
|
||||
if rel_path.startswith(prefix) or prefix.startswith(rel_path):
|
||||
files.extend(scan_dir(e.path, prefix))
|
||||
else:
|
||||
if rel_path.startswith(prefix):
|
||||
files.append(rel_path)
|
||||
return files
|
||||
|
||||
@dispatcher.add_method
|
||||
def listDataDirectory(prefix='') -> list[str]:
|
||||
return scan_dir(Paths.log_root(), prefix)
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def uploadFileToUrl(fn: str, url: str, headers: dict[str, str]) -> UploadFilesToUrlResponse:
|
||||
# this is because mypy doesn't understand that the decorator doesn't change the return type
|
||||
response: UploadFilesToUrlResponse = uploadFilesToUrls([{
|
||||
"fn": fn,
|
||||
"url": url,
|
||||
"headers": headers,
|
||||
}])
|
||||
return response
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def uploadFilesToUrls(files_data: list[UploadFileDict]) -> UploadFilesToUrlResponse:
|
||||
files = map(UploadFile.from_dict, files_data)
|
||||
|
||||
items: list[UploadItemDict] = []
|
||||
failed: list[str] = []
|
||||
for file in files:
|
||||
if len(file.fn) == 0 or file.fn[0] == '/' or '..' in file.fn or len(file.url) == 0:
|
||||
failed.append(file.fn)
|
||||
continue
|
||||
|
||||
path = os.path.join(Paths.log_root(), file.fn)
|
||||
if not os.path.exists(path) and not os.path.exists(strip_bz2_extension(path)):
|
||||
failed.append(file.fn)
|
||||
continue
|
||||
|
||||
# Skip item if already in queue
|
||||
url = file.url.split('?')[0]
|
||||
if any(url == item['url'].split('?')[0] for item in listUploadQueue()):
|
||||
continue
|
||||
|
||||
item = UploadItem(
|
||||
path=path,
|
||||
url=file.url,
|
||||
headers=file.headers,
|
||||
created_at=int(time.time() * 1000),
|
||||
id=None,
|
||||
allow_cellular=file.allow_cellular,
|
||||
)
|
||||
upload_id = hashlib.sha1(str(item).encode()).hexdigest()
|
||||
item = replace(item, id=upload_id)
|
||||
upload_queue.put_nowait(item)
|
||||
items.append(asdict(item))
|
||||
|
||||
UploadQueueCache.cache(upload_queue)
|
||||
|
||||
resp: UploadFilesToUrlResponse = {"enqueued": len(items), "items": items}
|
||||
if failed:
|
||||
resp["failed"] = failed
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def listUploadQueue() -> list[UploadItemDict]:
|
||||
items = list(upload_queue.queue) + list(cur_upload_items.values())
|
||||
return [asdict(i) for i in items if (i is not None) and (i.id not in cancelled_uploads)]
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def cancelUpload(upload_id: str | list[str]) -> dict[str, int | str]:
|
||||
if not isinstance(upload_id, list):
|
||||
upload_id = [upload_id]
|
||||
|
||||
uploading_ids = {item.id for item in list(upload_queue.queue)}
|
||||
cancelled_ids = uploading_ids.intersection(upload_id)
|
||||
if len(cancelled_ids) == 0:
|
||||
return {"success": 0, "error": "not found"}
|
||||
|
||||
cancelled_uploads.update(cancelled_ids)
|
||||
return {"success": 1}
|
||||
|
||||
@dispatcher.add_method
|
||||
def setRouteViewed(route: str) -> dict[str, int | str]:
|
||||
# maintain a list of the last 10 routes viewed in connect
|
||||
params = Params()
|
||||
|
||||
r = params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
|
||||
routes = [] if r is None else r.split(",")
|
||||
routes.append(route)
|
||||
|
||||
# remove duplicates
|
||||
routes = list(dict.fromkeys(routes))
|
||||
|
||||
params.put("AthenadRecentlyViewedRoutes", ",".join(routes[-10:]))
|
||||
return {"success": 1}
|
||||
|
||||
|
||||
def startLocalProxy(global_end_event: threading.Event, remote_ws_uri: str, local_port: int) -> dict[str, int]:
|
||||
try:
|
||||
if local_port not in LOCAL_PORT_WHITELIST:
|
||||
raise Exception("Requested local port not whitelisted")
|
||||
|
||||
cloudlog.debug("athena.startLocalProxy.starting")
|
||||
|
||||
dongle_id = Params().get("DongleId").decode('utf8')
|
||||
identity_token = Api(dongle_id).get_token()
|
||||
ws = create_connection(remote_ws_uri,
|
||||
cookie="jwt=" + identity_token,
|
||||
enable_multithread=True)
|
||||
|
||||
# Set TOS to keep connection responsive while under load.
|
||||
# DSCP of 36/HDD_LINUX_AC_VI with the minimum delay flag
|
||||
ws.sock.setsockopt(socket.IPPROTO_IP, socket.IP_TOS, 0x90)
|
||||
|
||||
ssock, csock = socket.socketpair()
|
||||
local_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
local_sock.connect(('127.0.0.1', local_port))
|
||||
local_sock.setblocking(False)
|
||||
|
||||
proxy_end_event = threading.Event()
|
||||
threads = [
|
||||
threading.Thread(target=ws_proxy_recv, args=(ws, local_sock, ssock, proxy_end_event, global_end_event)),
|
||||
threading.Thread(target=ws_proxy_send, args=(ws, local_sock, csock, proxy_end_event))
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
cloudlog.debug("athena.startLocalProxy.started")
|
||||
return {"success": 1}
|
||||
except Exception as e:
|
||||
cloudlog.exception("athenad.startLocalProxy.exception")
|
||||
raise e
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getPublicKey() -> str | None:
|
||||
if not os.path.isfile(Paths.persist_root() + '/comma/id_rsa.pub'):
|
||||
return None
|
||||
|
||||
with open(Paths.persist_root() + '/comma/id_rsa.pub') as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getSshAuthorizedKeys() -> str:
|
||||
return Params().get("GithubSshKeys", encoding='utf8') or ''
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getGithubUsername() -> str:
|
||||
return Params().get("GithubUsername", encoding='utf8') or ''
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getSimInfo():
|
||||
return HARDWARE.get_sim_info()
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getNetworkType():
|
||||
return HARDWARE.get_network_type()
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getNetworkMetered() -> bool:
|
||||
network_type = HARDWARE.get_network_type()
|
||||
return HARDWARE.get_network_metered(network_type)
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def getNetworks():
|
||||
return HARDWARE.get_networks()
|
||||
|
||||
|
||||
@dispatcher.add_method
|
||||
def takeSnapshot() -> str | dict[str, str] | None:
|
||||
from openpilot.system.camerad.snapshot.snapshot import jpeg_write, snapshot
|
||||
ret = snapshot()
|
||||
if ret is not None:
|
||||
def b64jpeg(x):
|
||||
if x is not None:
|
||||
f = io.BytesIO()
|
||||
jpeg_write(f, x)
|
||||
return base64.b64encode(f.getvalue()).decode("utf-8")
|
||||
else:
|
||||
return None
|
||||
return {'jpegBack': b64jpeg(ret[0]),
|
||||
'jpegFront': b64jpeg(ret[1])}
|
||||
else:
|
||||
raise Exception("not available while camerad is started")
|
||||
|
||||
|
||||
def get_logs_to_send_sorted() -> list[str]:
|
||||
# TODO: scan once then use inotify to detect file creation/deletion
|
||||
curr_time = int(time.time())
|
||||
logs = []
|
||||
for log_entry in os.listdir(Paths.swaglog_root()):
|
||||
log_path = os.path.join(Paths.swaglog_root(), log_entry)
|
||||
time_sent = 0
|
||||
try:
|
||||
value = getxattr(log_path, LOG_ATTR_NAME)
|
||||
if value is not None:
|
||||
time_sent = int.from_bytes(value, sys.byteorder)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
# assume send failed and we lost the response if sent more than one hour ago
|
||||
if not time_sent or curr_time - time_sent > 3600:
|
||||
logs.append(log_entry)
|
||||
# excluding most recent (active) log file
|
||||
return sorted(logs)[:-1]
|
||||
|
||||
|
||||
def log_handler(end_event: threading.Event) -> None:
|
||||
if PC:
|
||||
return
|
||||
|
||||
log_files = []
|
||||
last_scan = 0.
|
||||
while not end_event.is_set():
|
||||
try:
|
||||
curr_scan = time.monotonic()
|
||||
if curr_scan - last_scan > 10:
|
||||
log_files = get_logs_to_send_sorted()
|
||||
last_scan = curr_scan
|
||||
|
||||
# send one log
|
||||
curr_log = None
|
||||
if len(log_files) > 0:
|
||||
log_entry = log_files.pop() # newest log file
|
||||
cloudlog.debug(f"athena.log_handler.forward_request {log_entry}")
|
||||
try:
|
||||
curr_time = int(time.time())
|
||||
log_path = os.path.join(Paths.swaglog_root(), log_entry)
|
||||
setxattr(log_path, LOG_ATTR_NAME, int.to_bytes(curr_time, 4, sys.byteorder))
|
||||
with open(log_path) as f:
|
||||
jsonrpc = {
|
||||
"method": "forwardLogs",
|
||||
"params": {
|
||||
"logs": f.read()
|
||||
},
|
||||
"jsonrpc": "2.0",
|
||||
"id": log_entry
|
||||
}
|
||||
low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
|
||||
curr_log = log_entry
|
||||
except OSError:
|
||||
pass # file could be deleted by log rotation
|
||||
|
||||
# wait for response up to ~100 seconds
|
||||
# always read queue at least once to process any old responses that arrive
|
||||
for _ in range(100):
|
||||
if end_event.is_set():
|
||||
break
|
||||
try:
|
||||
log_resp = json.loads(log_recv_queue.get(timeout=1))
|
||||
log_entry = log_resp.get("id")
|
||||
log_success = "result" in log_resp and log_resp["result"].get("success")
|
||||
cloudlog.debug(f"athena.log_handler.forward_response {log_entry} {log_success}")
|
||||
if log_entry and log_success:
|
||||
log_path = os.path.join(Paths.swaglog_root(), log_entry)
|
||||
try:
|
||||
setxattr(log_path, LOG_ATTR_NAME, LOG_ATTR_VALUE_MAX_UNIX_TIME)
|
||||
except OSError:
|
||||
pass # file could be deleted by log rotation
|
||||
if curr_log == log_entry:
|
||||
break
|
||||
except queue.Empty:
|
||||
if curr_log is None:
|
||||
break
|
||||
|
||||
except Exception:
|
||||
cloudlog.exception("athena.log_handler.exception")
|
||||
|
||||
|
||||
def stat_handler(end_event: threading.Event) -> None:
|
||||
STATS_DIR = Paths.stats_root()
|
||||
while not end_event.is_set():
|
||||
last_scan = 0.
|
||||
curr_scan = time.monotonic()
|
||||
try:
|
||||
if curr_scan - last_scan > 10:
|
||||
stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(STATS_DIR)))
|
||||
if len(stat_filenames) > 0:
|
||||
stat_path = os.path.join(STATS_DIR, stat_filenames[0])
|
||||
with open(stat_path) as f:
|
||||
jsonrpc = {
|
||||
"method": "storeStats",
|
||||
"params": {
|
||||
"stats": f.read()
|
||||
},
|
||||
"jsonrpc": "2.0",
|
||||
"id": stat_filenames[0]
|
||||
}
|
||||
low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
|
||||
os.remove(stat_path)
|
||||
last_scan = curr_scan
|
||||
except Exception:
|
||||
cloudlog.exception("athena.stat_handler.exception")
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def ws_proxy_recv(ws: WebSocket, local_sock: socket.socket, ssock: socket.socket, end_event: threading.Event, global_end_event: threading.Event) -> None:
|
||||
while not (end_event.is_set() or global_end_event.is_set()):
|
||||
try:
|
||||
r = select.select((ws.sock,), (), (), 30)
|
||||
if r[0]:
|
||||
data = ws.recv()
|
||||
if isinstance(data, str):
|
||||
data = data.encode("utf-8")
|
||||
local_sock.sendall(data)
|
||||
except WebSocketTimeoutException:
|
||||
pass
|
||||
except Exception:
|
||||
cloudlog.exception("athenad.ws_proxy_recv.exception")
|
||||
break
|
||||
|
||||
cloudlog.debug("athena.ws_proxy_recv closing sockets")
|
||||
ssock.close()
|
||||
local_sock.close()
|
||||
ws.close()
|
||||
cloudlog.debug("athena.ws_proxy_recv done closing sockets")
|
||||
|
||||
end_event.set()
|
||||
|
||||
|
||||
def ws_proxy_send(ws: WebSocket, local_sock: socket.socket, signal_sock: socket.socket, end_event: threading.Event) -> None:
|
||||
while not end_event.is_set():
|
||||
try:
|
||||
r, _, _ = select.select((local_sock, signal_sock), (), ())
|
||||
if r:
|
||||
if r[0].fileno() == signal_sock.fileno():
|
||||
# got end signal from ws_proxy_recv
|
||||
end_event.set()
|
||||
break
|
||||
data = local_sock.recv(4096)
|
||||
if not data:
|
||||
# local_sock is dead
|
||||
end_event.set()
|
||||
break
|
||||
|
||||
ws.send(data, ABNF.OPCODE_BINARY)
|
||||
except Exception:
|
||||
cloudlog.exception("athenad.ws_proxy_send.exception")
|
||||
end_event.set()
|
||||
|
||||
cloudlog.debug("athena.ws_proxy_send closing sockets")
|
||||
signal_sock.close()
|
||||
cloudlog.debug("athena.ws_proxy_send done closing sockets")
|
||||
|
||||
|
||||
def ws_recv(ws: WebSocket, end_event: threading.Event) -> None:
|
||||
last_ping = int(time.monotonic() * 1e9)
|
||||
while not end_event.is_set():
|
||||
try:
|
||||
opcode, data = ws.recv_data(control_frame=True)
|
||||
if opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
|
||||
if opcode == ABNF.OPCODE_TEXT:
|
||||
data = data.decode("utf-8")
|
||||
recv_queue.put_nowait(data)
|
||||
elif opcode == ABNF.OPCODE_PING:
|
||||
last_ping = int(time.monotonic() * 1e9)
|
||||
Params().put("LastAthenaPingTime", str(last_ping))
|
||||
except WebSocketTimeoutException:
|
||||
ns_since_last_ping = int(time.monotonic() * 1e9) - last_ping
|
||||
if ns_since_last_ping > RECONNECT_TIMEOUT_S * 1e9:
|
||||
cloudlog.exception("athenad.ws_recv.timeout")
|
||||
end_event.set()
|
||||
except Exception:
|
||||
cloudlog.exception("athenad.ws_recv.exception")
|
||||
end_event.set()
|
||||
|
||||
|
||||
def ws_send(ws: WebSocket, end_event: threading.Event) -> None:
|
||||
while not end_event.is_set():
|
||||
try:
|
||||
try:
|
||||
data = send_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
data = low_priority_send_queue.get(timeout=1)
|
||||
for i in range(0, len(data), WS_FRAME_SIZE):
|
||||
frame = data[i:i+WS_FRAME_SIZE]
|
||||
last = i + WS_FRAME_SIZE >= len(data)
|
||||
opcode = ABNF.OPCODE_TEXT if i == 0 else ABNF.OPCODE_CONT
|
||||
ws.send_frame(ABNF.create_frame(frame, opcode, last))
|
||||
except queue.Empty:
|
||||
pass
|
||||
except Exception:
|
||||
cloudlog.exception("athenad.ws_send.exception")
|
||||
end_event.set()
|
||||
|
||||
|
||||
def ws_manage(ws: WebSocket, end_event: threading.Event) -> None:
|
||||
params = Params()
|
||||
onroad_prev = None
|
||||
sock = ws.sock
|
||||
|
||||
while True:
|
||||
onroad = params.get_bool("IsOnroad")
|
||||
if onroad != onroad_prev:
|
||||
onroad_prev = onroad
|
||||
|
||||
if sock is not None:
|
||||
# While not sending data, onroad, we can expect to time out in 7 + (7 * 2) = 21s
|
||||
# offroad, we can expect to time out in 30 + (10 * 3) = 60s
|
||||
# FIXME: TCP_USER_TIMEOUT is effectively 2x for some reason (32s), so it's mostly unused
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, 16000 if onroad else 0)
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 7 if onroad else 30)
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 7 if onroad else 10)
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 2 if onroad else 3)
|
||||
|
||||
if end_event.wait(5):
|
||||
break
|
||||
|
||||
|
||||
def backoff(retries: int) -> int:
|
||||
return random.randrange(0, min(128, int(2 ** retries)))
|
||||
|
||||
|
||||
def main(exit_event: threading.Event = None):
|
||||
try:
|
||||
set_core_affinity([0, 1, 2, 3])
|
||||
except Exception:
|
||||
cloudlog.exception("failed to set core affinity")
|
||||
|
||||
params = Params()
|
||||
dongle_id = params.get("DongleId", encoding='utf-8')
|
||||
UploadQueueCache.initialize(upload_queue)
|
||||
|
||||
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
|
||||
api = Api(dongle_id)
|
||||
|
||||
conn_start = None
|
||||
conn_retries = 0
|
||||
while exit_event is None or not exit_event.is_set():
|
||||
try:
|
||||
if conn_start is None:
|
||||
conn_start = time.monotonic()
|
||||
|
||||
cloudlog.event("athenad.main.connecting_ws", ws_uri=ws_uri, retries=conn_retries)
|
||||
ws = create_connection(ws_uri,
|
||||
cookie="jwt=" + api.get_token(),
|
||||
enable_multithread=True,
|
||||
timeout=30.0)
|
||||
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri, retries=conn_retries,
|
||||
duration=time.monotonic() - conn_start)
|
||||
conn_start = None
|
||||
|
||||
conn_retries = 0
|
||||
cur_upload_items.clear()
|
||||
|
||||
handle_long_poll(ws, exit_event)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
break
|
||||
except (ConnectionError, TimeoutError, WebSocketException):
|
||||
conn_retries += 1
|
||||
params.remove("LastAthenaPingTime")
|
||||
except Exception:
|
||||
cloudlog.exception("athenad.main.exception")
|
||||
|
||||
conn_retries += 1
|
||||
params.remove("LastAthenaPingTime")
|
||||
|
||||
time.sleep(backoff(conn_retries))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import time
|
||||
from multiprocessing import Process
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.manager.process import launcher
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.hardware import HARDWARE
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
ATHENA_MGR_PID_PARAM = "AthenadPid"
|
||||
|
||||
|
||||
def main():
|
||||
params = Params()
|
||||
dongle_id = params.get("DongleId").decode('utf-8')
|
||||
build_metadata = get_build_metadata()
|
||||
|
||||
cloudlog.bind_global(dongle_id=dongle_id,
|
||||
version=build_metadata.openpilot.version,
|
||||
origin=build_metadata.openpilot.git_normalized_origin,
|
||||
branch=build_metadata.channel,
|
||||
commit=build_metadata.openpilot.git_commit,
|
||||
dirty=build_metadata.openpilot.is_dirty,
|
||||
device=HARDWARE.get_device_type())
|
||||
|
||||
try:
|
||||
while 1:
|
||||
cloudlog.info("starting athena daemon")
|
||||
proc = Process(name='athenad', target=launcher, args=('system.athena.athenad', 'athenad'))
|
||||
proc.start()
|
||||
proc.join()
|
||||
cloudlog.event("athenad exited", exitcode=proc.exitcode)
|
||||
time.sleep(5)
|
||||
except Exception:
|
||||
cloudlog.exception("manage_athenad.exception")
|
||||
finally:
|
||||
params.remove(ATHENA_MGR_PID_PARAM)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
import time
|
||||
import json
|
||||
import jwt
|
||||
from pathlib import Path
|
||||
|
||||
from datetime import datetime, timedelta, UTC
|
||||
from openpilot.common.api import api_get
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.spinner import Spinner
|
||||
from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
|
||||
from openpilot.system.hardware import HARDWARE, PC
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
|
||||
|
||||
UNREGISTERED_DONGLE_ID = "UnregisteredDevice"
|
||||
|
||||
|
||||
def is_registered_device() -> bool:
|
||||
dongle = Params().get("DongleId", encoding='utf-8')
|
||||
return dongle not in (None, UNREGISTERED_DONGLE_ID)
|
||||
|
||||
|
||||
def register(show_spinner=False) -> str | None:
|
||||
params = Params()
|
||||
|
||||
IMEI = params.get("IMEI", encoding='utf8')
|
||||
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
|
||||
dongle_id: str | None = params.get("DongleId", encoding='utf8')
|
||||
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
|
||||
|
||||
pubkey = Path(Paths.persist_root()+"/comma/id_rsa.pub")
|
||||
if not pubkey.is_file():
|
||||
dongle_id = UNREGISTERED_DONGLE_ID
|
||||
cloudlog.warning(f"missing public key: {pubkey}")
|
||||
elif needs_registration:
|
||||
if show_spinner:
|
||||
spinner = Spinner()
|
||||
spinner.update("registering device")
|
||||
|
||||
# Create registration token, in the future, this key will make JWTs directly
|
||||
with open(Paths.persist_root()+"/comma/id_rsa.pub") as f1, open(Paths.persist_root()+"/comma/id_rsa") as f2:
|
||||
public_key = f1.read()
|
||||
private_key = f2.read()
|
||||
|
||||
# Block until we get the imei
|
||||
serial = HARDWARE.get_serial()
|
||||
start_time = time.monotonic()
|
||||
imei1: str | None = None
|
||||
imei2: str | None = None
|
||||
while imei1 is None and imei2 is None:
|
||||
try:
|
||||
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
|
||||
except Exception:
|
||||
cloudlog.exception("Error getting imei, trying again...")
|
||||
time.sleep(1)
|
||||
|
||||
if time.monotonic() - start_time > 60 and show_spinner:
|
||||
spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})")
|
||||
|
||||
params.put("IMEI", imei1)
|
||||
params.put("HardwareSerial", serial)
|
||||
|
||||
backoff = 0
|
||||
start_time = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
register_token = jwt.encode({'register': True, 'exp': datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=1)}, private_key, algorithm='RS256')
|
||||
cloudlog.info("getting pilotauth")
|
||||
resp = api_get("v2/pilotauth/", method='POST', timeout=15,
|
||||
imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token)
|
||||
|
||||
if resp.status_code in (402, 403):
|
||||
cloudlog.info(f"Unable to register device, got {resp.status_code}")
|
||||
dongle_id = UNREGISTERED_DONGLE_ID
|
||||
else:
|
||||
dongleauth = json.loads(resp.text)
|
||||
dongle_id = dongleauth["dongle_id"]
|
||||
break
|
||||
except Exception:
|
||||
cloudlog.exception("failed to authenticate")
|
||||
backoff = min(backoff + 1, 15)
|
||||
time.sleep(backoff)
|
||||
|
||||
if time.monotonic() - start_time > 60 and show_spinner:
|
||||
spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})")
|
||||
|
||||
if show_spinner:
|
||||
spinner.close()
|
||||
|
||||
if dongle_id:
|
||||
params.put("DongleId", dongle_id)
|
||||
set_offroad_alert("Offroad_UnofficialHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC)
|
||||
return dongle_id
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(register())
|
||||
@@ -0,0 +1,70 @@
|
||||
import http.server
|
||||
import socket
|
||||
|
||||
|
||||
class MockResponse:
|
||||
def __init__(self, json, status_code):
|
||||
self.json = json
|
||||
self.text = json
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class EchoSocket:
|
||||
def __init__(self, port):
|
||||
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.socket.bind(('127.0.0.1', port))
|
||||
self.socket.listen(1)
|
||||
|
||||
def run(self):
|
||||
conn, _ = self.socket.accept()
|
||||
conn.settimeout(5.0)
|
||||
|
||||
try:
|
||||
while True:
|
||||
data = conn.recv(4096)
|
||||
if data:
|
||||
print(f'EchoSocket got {data}')
|
||||
conn.sendall(data)
|
||||
else:
|
||||
break
|
||||
finally:
|
||||
conn.shutdown(0)
|
||||
conn.close()
|
||||
self.socket.shutdown(0)
|
||||
self.socket.close()
|
||||
|
||||
|
||||
class MockApi:
|
||||
def __init__(self, dongle_id):
|
||||
pass
|
||||
|
||||
def get_token(self):
|
||||
return "fake-token"
|
||||
|
||||
|
||||
class MockWebsocket:
|
||||
sock = socket.socket()
|
||||
|
||||
def __init__(self, recv_queue, send_queue):
|
||||
self.recv_queue = recv_queue
|
||||
self.send_queue = send_queue
|
||||
|
||||
def recv(self):
|
||||
data = self.recv_queue.get()
|
||||
if isinstance(data, Exception):
|
||||
raise data
|
||||
return data
|
||||
|
||||
def send(self, data, opcode):
|
||||
self.send_queue.put_nowait((data, opcode))
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def do_PUT(self):
|
||||
length = int(self.headers['Content-Length'])
|
||||
self.rfile.read(length)
|
||||
self.send_response(201, "Created")
|
||||
self.end_headers()
|
||||
@@ -0,0 +1,423 @@
|
||||
import pytest
|
||||
from functools import wraps
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import requests
|
||||
import shutil
|
||||
import time
|
||||
import threading
|
||||
import queue
|
||||
from dataclasses import asdict, replace
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from websocket import ABNF
|
||||
from websocket._exceptions import WebSocketConnectionClosedException
|
||||
|
||||
from cereal import messaging
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.timeout import Timeout
|
||||
from openpilot.system.athena import athenad
|
||||
from openpilot.system.athena.athenad import MAX_RETRY_COUNT, dispatcher
|
||||
from openpilot.system.athena.tests.helpers import HTTPRequestHandler, MockWebsocket, MockApi, EchoSocket
|
||||
from openpilot.selfdrive.test.helpers import http_server_context
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
|
||||
def seed_athena_server(host, port):
|
||||
with Timeout(2, 'HTTP Server seeding failed'):
|
||||
while True:
|
||||
try:
|
||||
requests.put(f'http://{host}:{port}/qlog.bz2', data='', timeout=10)
|
||||
break
|
||||
except requests.exceptions.ConnectionError:
|
||||
time.sleep(0.1)
|
||||
|
||||
def with_upload_handler(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
end_event = threading.Event()
|
||||
thread = threading.Thread(target=athenad.upload_handler, args=(end_event,))
|
||||
thread.start()
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
end_event.set()
|
||||
thread.join()
|
||||
return wrapper
|
||||
|
||||
@pytest.fixture
|
||||
def mock_create_connection(mocker):
|
||||
return mocker.patch('openpilot.system.athena.athenad.create_connection')
|
||||
|
||||
@pytest.fixture
|
||||
def host():
|
||||
with http_server_context(handler=HTTPRequestHandler, setup=seed_athena_server) as (host, port):
|
||||
yield f"http://{host}:{port}"
|
||||
|
||||
class TestAthenadMethods:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.SOCKET_PORT = 45454
|
||||
athenad.Api = MockApi
|
||||
athenad.LOCAL_PORT_WHITELIST = {cls.SOCKET_PORT}
|
||||
|
||||
def setup_method(self):
|
||||
self.default_params = {
|
||||
"DongleId": "0000000000000000",
|
||||
"GithubSshKeys": b"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501
|
||||
"GithubUsername": b"commaci",
|
||||
"AthenadUploadQueue": '[]',
|
||||
}
|
||||
|
||||
self.params = Params()
|
||||
for k, v in self.default_params.items():
|
||||
self.params.put(k, v)
|
||||
self.params.put_bool("GsmMetered", True)
|
||||
|
||||
athenad.upload_queue = queue.Queue()
|
||||
athenad.cur_upload_items.clear()
|
||||
athenad.cancelled_uploads.clear()
|
||||
|
||||
for i in os.listdir(Paths.log_root()):
|
||||
p = os.path.join(Paths.log_root(), i)
|
||||
if os.path.isdir(p):
|
||||
shutil.rmtree(p)
|
||||
else:
|
||||
os.unlink(p)
|
||||
|
||||
# *** test helpers ***
|
||||
|
||||
@staticmethod
|
||||
def _wait_for_upload():
|
||||
now = time.time()
|
||||
while time.time() - now < 5:
|
||||
if athenad.upload_queue.qsize() == 0:
|
||||
break
|
||||
|
||||
@staticmethod
|
||||
def _create_file(file: str, parent: str = None, data: bytes = b'') -> str:
|
||||
fn = os.path.join(Paths.log_root() if parent is None else parent, file)
|
||||
os.makedirs(os.path.dirname(fn), exist_ok=True)
|
||||
with open(fn, 'wb') as f:
|
||||
f.write(data)
|
||||
return fn
|
||||
|
||||
|
||||
# *** test cases ***
|
||||
|
||||
def test_echo(self):
|
||||
assert dispatcher["echo"]("bob") == "bob"
|
||||
|
||||
def test_get_message(self):
|
||||
with pytest.raises(TimeoutError) as _:
|
||||
dispatcher["getMessage"]("controlsState")
|
||||
|
||||
end_event = multiprocessing.Event()
|
||||
|
||||
pub_sock = messaging.pub_sock("deviceState")
|
||||
|
||||
def send_deviceState():
|
||||
while not end_event.is_set():
|
||||
msg = messaging.new_message('deviceState')
|
||||
pub_sock.send(msg.to_bytes())
|
||||
time.sleep(0.01)
|
||||
|
||||
p = multiprocessing.Process(target=send_deviceState)
|
||||
p.start()
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
deviceState = dispatcher["getMessage"]("deviceState")
|
||||
assert deviceState['deviceState']
|
||||
finally:
|
||||
end_event.set()
|
||||
p.join()
|
||||
|
||||
def test_list_data_directory(self):
|
||||
route = '2021-03-29--13-32-47'
|
||||
segments = [0, 1, 2, 3, 11]
|
||||
|
||||
filenames = ['qlog', 'qcamera.ts', 'rlog', 'fcamera.hevc', 'ecamera.hevc', 'dcamera.hevc']
|
||||
files = [f'{route}--{s}/{f}' for s in segments for f in filenames]
|
||||
for file in files:
|
||||
self._create_file(file)
|
||||
|
||||
resp = dispatcher["listDataDirectory"]()
|
||||
assert resp, 'list empty!'
|
||||
assert len(resp) == len(files)
|
||||
|
||||
resp = dispatcher["listDataDirectory"](f'{route}--123')
|
||||
assert len(resp) == 0
|
||||
|
||||
prefix = f'{route}'
|
||||
expected = list(filter(lambda f: f.startswith(prefix), files))
|
||||
resp = dispatcher["listDataDirectory"](prefix)
|
||||
assert resp, 'list empty!'
|
||||
assert len(resp) == len(expected)
|
||||
|
||||
prefix = f'{route}--1'
|
||||
expected = list(filter(lambda f: f.startswith(prefix), files))
|
||||
resp = dispatcher["listDataDirectory"](prefix)
|
||||
assert resp, 'list empty!'
|
||||
assert len(resp) == len(expected)
|
||||
|
||||
prefix = f'{route}--1/'
|
||||
expected = list(filter(lambda f: f.startswith(prefix), files))
|
||||
resp = dispatcher["listDataDirectory"](prefix)
|
||||
assert resp, 'list empty!'
|
||||
assert len(resp) == len(expected)
|
||||
|
||||
prefix = f'{route}--1/q'
|
||||
expected = list(filter(lambda f: f.startswith(prefix), files))
|
||||
resp = dispatcher["listDataDirectory"](prefix)
|
||||
assert resp, 'list empty!'
|
||||
assert len(resp) == len(expected)
|
||||
|
||||
def test_strip_bz2_extension(self):
|
||||
fn = self._create_file('qlog.bz2')
|
||||
if fn.endswith('.bz2'):
|
||||
assert athenad.strip_bz2_extension(fn) == fn[:-4]
|
||||
|
||||
@pytest.mark.parametrize("compress", [True, False])
|
||||
def test_do_upload(self, host, compress):
|
||||
# random bytes to ensure rather large object post-compression
|
||||
fn = self._create_file('qlog', data=os.urandom(10000 * 1024))
|
||||
|
||||
upload_fn = fn + ('.bz2' if compress else '')
|
||||
item = athenad.UploadItem(path=upload_fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='')
|
||||
with pytest.raises(requests.exceptions.ConnectionError):
|
||||
athenad._do_upload(item)
|
||||
|
||||
item = athenad.UploadItem(path=upload_fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='')
|
||||
resp = athenad._do_upload(item)
|
||||
assert resp.status_code == 201
|
||||
|
||||
def test_upload_file_to_url(self, host):
|
||||
fn = self._create_file('qlog.bz2')
|
||||
|
||||
resp = dispatcher["uploadFileToUrl"]("qlog.bz2", f"{host}/qlog.bz2", {})
|
||||
assert resp['enqueued'] == 1
|
||||
assert 'failed' not in resp
|
||||
assert {"path": fn, "url": f"{host}/qlog.bz2", "headers": {}}.items() <= resp['items'][0].items()
|
||||
assert resp['items'][0].get('id') is not None
|
||||
assert athenad.upload_queue.qsize() == 1
|
||||
|
||||
def test_upload_file_to_url_duplicate(self, host):
|
||||
self._create_file('qlog.bz2')
|
||||
|
||||
url1 = f"{host}/qlog.bz2?sig=sig1"
|
||||
dispatcher["uploadFileToUrl"]("qlog.bz2", url1, {})
|
||||
|
||||
# Upload same file again, but with different signature
|
||||
url2 = f"{host}/qlog.bz2?sig=sig2"
|
||||
resp = dispatcher["uploadFileToUrl"]("qlog.bz2", url2, {})
|
||||
assert resp == {'enqueued': 0, 'items': []}
|
||||
|
||||
def test_upload_file_to_url_does_not_exist(self, host):
|
||||
not_exists_resp = dispatcher["uploadFileToUrl"]("does_not_exist.bz2", "http://localhost:1238", {})
|
||||
assert not_exists_resp == {'enqueued': 0, 'items': [], 'failed': ['does_not_exist.bz2']}
|
||||
|
||||
@with_upload_handler
|
||||
def test_upload_handler(self, host):
|
||||
fn = self._create_file('qlog.bz2')
|
||||
item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
|
||||
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
self._wait_for_upload()
|
||||
time.sleep(0.1)
|
||||
|
||||
# TODO: verify that upload actually succeeded
|
||||
# TODO: also check that end_event and metered network raises AbortTransferException
|
||||
assert athenad.upload_queue.qsize() == 0
|
||||
|
||||
@pytest.mark.parametrize("status,retry", [(500,True), (412,False)])
|
||||
@with_upload_handler
|
||||
def test_upload_handler_retry(self, mocker, host, status, retry):
|
||||
mock_put = mocker.patch('requests.put')
|
||||
mock_put.return_value.status_code = status
|
||||
fn = self._create_file('qlog.bz2')
|
||||
item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
|
||||
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
self._wait_for_upload()
|
||||
time.sleep(0.1)
|
||||
|
||||
assert athenad.upload_queue.qsize() == (1 if retry else 0)
|
||||
|
||||
if retry:
|
||||
assert athenad.upload_queue.get().retry_count == 1
|
||||
|
||||
@with_upload_handler
|
||||
def test_upload_handler_timeout(self):
|
||||
"""When an upload times out or fails to connect it should be placed back in the queue"""
|
||||
fn = self._create_file('qlog.bz2')
|
||||
item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
|
||||
item_no_retry = replace(item, retry_count=MAX_RETRY_COUNT)
|
||||
|
||||
athenad.upload_queue.put_nowait(item_no_retry)
|
||||
self._wait_for_upload()
|
||||
time.sleep(0.1)
|
||||
|
||||
# Check that upload with retry count exceeded is not put back
|
||||
assert athenad.upload_queue.qsize() == 0
|
||||
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
self._wait_for_upload()
|
||||
time.sleep(0.1)
|
||||
|
||||
# Check that upload item was put back in the queue with incremented retry count
|
||||
assert athenad.upload_queue.qsize() == 1
|
||||
assert athenad.upload_queue.get().retry_count == 1
|
||||
|
||||
@with_upload_handler
|
||||
def test_cancel_upload(self):
|
||||
item = athenad.UploadItem(path="qlog.bz2", url="http://localhost:44444/qlog.bz2", headers={},
|
||||
created_at=int(time.time()*1000), id='id', allow_cellular=True)
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
dispatcher["cancelUpload"](item.id)
|
||||
|
||||
assert item.id in athenad.cancelled_uploads
|
||||
|
||||
self._wait_for_upload()
|
||||
time.sleep(0.1)
|
||||
|
||||
assert athenad.upload_queue.qsize() == 0
|
||||
assert len(athenad.cancelled_uploads) == 0
|
||||
|
||||
@with_upload_handler
|
||||
def test_cancel_expiry(self):
|
||||
t_future = datetime.now() - timedelta(days=40)
|
||||
ts = int(t_future.strftime("%s")) * 1000
|
||||
|
||||
# Item that would time out if actually uploaded
|
||||
fn = self._create_file('qlog.bz2')
|
||||
item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.bz2", headers={}, created_at=ts, id='', allow_cellular=True)
|
||||
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
self._wait_for_upload()
|
||||
time.sleep(0.1)
|
||||
|
||||
assert athenad.upload_queue.qsize() == 0
|
||||
|
||||
def test_list_upload_queue_empty(self):
|
||||
items = dispatcher["listUploadQueue"]()
|
||||
assert len(items) == 0
|
||||
|
||||
@with_upload_handler
|
||||
def test_list_upload_queue_current(self, host: str):
|
||||
fn = self._create_file('qlog.bz2')
|
||||
item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
|
||||
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
self._wait_for_upload()
|
||||
|
||||
items = dispatcher["listUploadQueue"]()
|
||||
assert len(items) == 1
|
||||
assert items[0]['current']
|
||||
|
||||
def test_list_upload_queue(self):
|
||||
item = athenad.UploadItem(path="qlog.bz2", url="http://localhost:44444/qlog.bz2", headers={},
|
||||
created_at=int(time.time()*1000), id='id', allow_cellular=True)
|
||||
athenad.upload_queue.put_nowait(item)
|
||||
|
||||
items = dispatcher["listUploadQueue"]()
|
||||
assert len(items) == 1
|
||||
assert items[0] == asdict(item)
|
||||
assert not items[0]['current']
|
||||
|
||||
athenad.cancelled_uploads.add(item.id)
|
||||
items = dispatcher["listUploadQueue"]()
|
||||
assert len(items) == 0
|
||||
|
||||
def test_upload_queue_persistence(self):
|
||||
item1 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id1')
|
||||
item2 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id2')
|
||||
|
||||
athenad.upload_queue.put_nowait(item1)
|
||||
athenad.upload_queue.put_nowait(item2)
|
||||
|
||||
# Ensure cancelled items are not persisted
|
||||
athenad.cancelled_uploads.add(item2.id)
|
||||
|
||||
# serialize item
|
||||
athenad.UploadQueueCache.cache(athenad.upload_queue)
|
||||
|
||||
# deserialize item
|
||||
athenad.upload_queue.queue.clear()
|
||||
athenad.UploadQueueCache.initialize(athenad.upload_queue)
|
||||
|
||||
assert athenad.upload_queue.qsize() == 1
|
||||
assert asdict(athenad.upload_queue.queue[-1]) == asdict(item1)
|
||||
|
||||
def test_start_local_proxy(self, mock_create_connection):
|
||||
end_event = threading.Event()
|
||||
|
||||
ws_recv = queue.Queue()
|
||||
ws_send = queue.Queue()
|
||||
mock_ws = MockWebsocket(ws_recv, ws_send)
|
||||
mock_create_connection.return_value = mock_ws
|
||||
|
||||
echo_socket = EchoSocket(self.SOCKET_PORT)
|
||||
socket_thread = threading.Thread(target=echo_socket.run)
|
||||
socket_thread.start()
|
||||
|
||||
athenad.startLocalProxy(end_event, 'ws://localhost:1234', self.SOCKET_PORT)
|
||||
|
||||
ws_recv.put_nowait(b'ping')
|
||||
try:
|
||||
recv = ws_send.get(timeout=5)
|
||||
assert recv == (b'ping', ABNF.OPCODE_BINARY), recv
|
||||
finally:
|
||||
# signal websocket close to athenad.ws_proxy_recv
|
||||
ws_recv.put_nowait(WebSocketConnectionClosedException())
|
||||
socket_thread.join()
|
||||
|
||||
def test_get_ssh_authorized_keys(self):
|
||||
keys = dispatcher["getSshAuthorizedKeys"]()
|
||||
assert keys == self.default_params["GithubSshKeys"].decode('utf-8')
|
||||
|
||||
def test_get_github_username(self):
|
||||
keys = dispatcher["getGithubUsername"]()
|
||||
assert keys == self.default_params["GithubUsername"].decode('utf-8')
|
||||
|
||||
def test_get_version(self):
|
||||
resp = dispatcher["getVersion"]()
|
||||
keys = ["version", "remote", "branch", "commit"]
|
||||
assert list(resp.keys()) == keys
|
||||
for k in keys:
|
||||
assert isinstance(resp[k], str), f"{k} is not a string"
|
||||
assert len(resp[k]) > 0, f"{k} has no value"
|
||||
|
||||
def test_jsonrpc_handler(self):
|
||||
end_event = threading.Event()
|
||||
thread = threading.Thread(target=athenad.jsonrpc_handler, args=(end_event,))
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
try:
|
||||
# with params
|
||||
athenad.recv_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0}))
|
||||
resp = athenad.send_queue.get(timeout=3)
|
||||
assert json.loads(resp) == {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'}
|
||||
# without params
|
||||
athenad.recv_queue.put_nowait(json.dumps({"method": "getNetworkType", "jsonrpc": "2.0", "id": 0}))
|
||||
resp = athenad.send_queue.get(timeout=3)
|
||||
assert json.loads(resp) == {'result': 1, 'id': 0, 'jsonrpc': '2.0'}
|
||||
# log forwarding
|
||||
athenad.recv_queue.put_nowait(json.dumps({'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}))
|
||||
resp = athenad.log_recv_queue.get(timeout=3)
|
||||
assert json.loads(resp) == {'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}
|
||||
finally:
|
||||
end_event.set()
|
||||
thread.join()
|
||||
|
||||
def test_get_logs_to_send_sorted(self):
|
||||
fl = list()
|
||||
for i in range(10):
|
||||
file = f'swaglog.{i:010}'
|
||||
self._create_file(file, Paths.swaglog_root())
|
||||
fl.append(file)
|
||||
|
||||
# ensure the list is all logs except most recent
|
||||
sl = athenad.get_logs_to_send_sorted()
|
||||
assert sl == fl[:-1]
|
||||
@@ -0,0 +1,102 @@
|
||||
import pytest
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from typing import cast
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.timeout import Timeout
|
||||
from openpilot.system.athena import athenad
|
||||
from openpilot.system.manager.helpers import write_onroad_params
|
||||
from openpilot.system.hardware import TICI
|
||||
|
||||
TIMEOUT_TOLERANCE = 20 # seconds
|
||||
|
||||
|
||||
def wifi_radio(on: bool) -> None:
|
||||
if not TICI:
|
||||
return
|
||||
print(f"wifi {'on' if on else 'off'}")
|
||||
subprocess.run(["nmcli", "radio", "wifi", "on" if on else "off"], check=True)
|
||||
|
||||
|
||||
class TestAthenadPing:
|
||||
params: Params
|
||||
dongle_id: str
|
||||
|
||||
athenad: threading.Thread
|
||||
exit_event: threading.Event
|
||||
|
||||
def _get_ping_time(self) -> str | None:
|
||||
return cast(str | None, self.params.get("LastAthenaPingTime", encoding="utf-8"))
|
||||
|
||||
def _clear_ping_time(self) -> None:
|
||||
self.params.remove("LastAthenaPingTime")
|
||||
|
||||
def _received_ping(self) -> bool:
|
||||
return self._get_ping_time() is not None
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls) -> None:
|
||||
wifi_radio(True)
|
||||
|
||||
def setup_method(self) -> None:
|
||||
self.params = Params()
|
||||
self.dongle_id = self.params.get("DongleId", encoding="utf-8")
|
||||
|
||||
wifi_radio(True)
|
||||
self._clear_ping_time()
|
||||
|
||||
self.exit_event = threading.Event()
|
||||
self.athenad = threading.Thread(target=athenad.main, args=(self.exit_event,))
|
||||
|
||||
def teardown_method(self) -> None:
|
||||
if self.athenad.is_alive():
|
||||
self.exit_event.set()
|
||||
self.athenad.join()
|
||||
|
||||
def assertTimeout(self, reconnect_time: float, subtests, mocker) -> None:
|
||||
self.athenad.start()
|
||||
|
||||
mock_create_connection = mocker.patch('openpilot.system.athena.athenad.create_connection',
|
||||
new_callable=lambda: mocker.MagicMock(wraps=athenad.create_connection))
|
||||
|
||||
time.sleep(1)
|
||||
mock_create_connection.assert_called_once()
|
||||
mock_create_connection.reset_mock()
|
||||
|
||||
# check normal behaviour, server pings on connection
|
||||
with subtests.test("Wi-Fi: receives ping"), Timeout(70, "no ping received"):
|
||||
while not self._received_ping():
|
||||
time.sleep(0.1)
|
||||
print("ping received")
|
||||
|
||||
mock_create_connection.assert_not_called()
|
||||
|
||||
# websocket should attempt reconnect after short time
|
||||
with subtests.test("LTE: attempt reconnect"):
|
||||
wifi_radio(False)
|
||||
print("waiting for reconnect attempt")
|
||||
start_time = time.monotonic()
|
||||
with Timeout(reconnect_time, "no reconnect attempt"):
|
||||
while not mock_create_connection.called:
|
||||
time.sleep(0.1)
|
||||
print(f"reconnect attempt after {time.monotonic() - start_time:.2f}s")
|
||||
|
||||
self._clear_ping_time()
|
||||
|
||||
# check ping received after reconnect
|
||||
with subtests.test("LTE: receives ping"), Timeout(70, "no ping received"):
|
||||
while not self._received_ping():
|
||||
time.sleep(0.1)
|
||||
print("ping received")
|
||||
|
||||
@pytest.mark.skipif(not TICI, reason="only run on desk")
|
||||
def test_offroad(self, subtests, mocker) -> None:
|
||||
write_onroad_params(False, self.params)
|
||||
self.assertTimeout(60 + TIMEOUT_TOLERANCE, subtests, mocker) # based using TCP keepalive settings
|
||||
|
||||
@pytest.mark.skipif(not TICI, reason="only run on desk")
|
||||
def test_onroad(self, subtests, mocker) -> None:
|
||||
write_onroad_params(True, self.params)
|
||||
self.assertTimeout(21 + TIMEOUT_TOLERANCE, subtests, mocker)
|
||||
@@ -0,0 +1,74 @@
|
||||
import json
|
||||
from Crypto.PublicKey import RSA
|
||||
from pathlib import Path
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
|
||||
from openpilot.system.athena.tests.helpers import MockResponse
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
|
||||
class TestRegistration:
|
||||
|
||||
def setup_method(self):
|
||||
# clear params and setup key paths
|
||||
self.params = Params()
|
||||
self.params.clear_all()
|
||||
|
||||
persist_dir = Path(Paths.persist_root()) / "comma"
|
||||
persist_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.priv_key = persist_dir / "id_rsa"
|
||||
self.pub_key = persist_dir / "id_rsa.pub"
|
||||
|
||||
def _generate_keys(self):
|
||||
self.pub_key.touch()
|
||||
k = RSA.generate(2048)
|
||||
with open(self.priv_key, "wb") as f:
|
||||
f.write(k.export_key())
|
||||
with open(self.pub_key, "wb") as f:
|
||||
f.write(k.publickey().export_key())
|
||||
|
||||
def test_valid_cache(self, mocker):
|
||||
# if all params are written, return the cached dongle id
|
||||
self.params.put("IMEI", "imei")
|
||||
self.params.put("HardwareSerial", "serial")
|
||||
self._generate_keys()
|
||||
|
||||
m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
|
||||
dongle = "DONGLE_ID_123"
|
||||
self.params.put("DongleId", dongle)
|
||||
assert register() == dongle
|
||||
assert not m.called
|
||||
|
||||
def test_no_keys(self, mocker):
|
||||
# missing pubkey
|
||||
m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
|
||||
dongle = register()
|
||||
assert m.call_count == 0
|
||||
assert dongle == UNREGISTERED_DONGLE_ID
|
||||
assert self.params.get("DongleId", encoding='utf-8') == dongle
|
||||
|
||||
def test_missing_cache(self, mocker):
|
||||
# keys exist but no dongle id
|
||||
self._generate_keys()
|
||||
m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
|
||||
dongle = "DONGLE_ID_123"
|
||||
m.return_value = MockResponse(json.dumps({'dongle_id': dongle}), 200)
|
||||
assert register() == dongle
|
||||
assert m.call_count == 1
|
||||
|
||||
# call again, shouldn't hit the API this time
|
||||
assert register() == dongle
|
||||
assert m.call_count == 1
|
||||
assert self.params.get("DongleId", encoding='utf-8') == dongle
|
||||
|
||||
def test_unregistered(self, mocker):
|
||||
# keys exist, but unregistered
|
||||
self._generate_keys()
|
||||
m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
|
||||
m.return_value = MockResponse(None, 402)
|
||||
dongle = register()
|
||||
assert m.call_count == 1
|
||||
assert dongle == UNREGISTERED_DONGLE_ID
|
||||
assert self.params.get("DongleId", encoding='utf-8') == dongle
|
||||
@@ -1,6 +1,6 @@
|
||||
Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc')
|
||||
Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc')
|
||||
|
||||
libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon, 'atomic']
|
||||
libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', messaging, visionipc, gpucommon, 'atomic']
|
||||
|
||||
camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/camera_util.cc',
|
||||
'sensors/ar0231.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc'])
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
#include <thread>
|
||||
|
||||
#include "cereal/messaging/messaging.h"
|
||||
#include "cereal/visionipc/visionipc_server.h"
|
||||
#include "msgq/visionipc/visionipc_server.h"
|
||||
#include "common/queue.h"
|
||||
#include "common/util.h"
|
||||
|
||||
|
||||
@@ -6,12 +6,12 @@ import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from cereal.visionipc import VisionIpcClient, VisionStreamType
|
||||
from msgq.visionipc import VisionIpcClient, VisionStreamType
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import DT_MDL
|
||||
from openpilot.system.hardware import PC
|
||||
from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
|
||||
|
||||
VISION_STREAMS = {
|
||||
@@ -83,7 +83,7 @@ def snapshot():
|
||||
front_camera_allowed = params.get_bool("RecordFront")
|
||||
params.put_bool("IsTakingSnapshot", True)
|
||||
set_offroad_alert("Offroad_IsTakingSnapshot", True)
|
||||
time.sleep(2.0) # Give thermald time to read the param, or if just started give camerad time to start
|
||||
time.sleep(2.0) # Give hardwared time to read the param, or if just started give camerad time to start
|
||||
|
||||
# Check if camerad is already started
|
||||
try:
|
||||
|
||||
Executable → Regular
+1
-2
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
import pytest
|
||||
import time
|
||||
import numpy as np
|
||||
@@ -8,7 +7,7 @@ from collections import defaultdict
|
||||
import cereal.messaging as messaging
|
||||
from cereal import log
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
|
||||
TEST_TIMESPAN = 30
|
||||
LAG_FRAME_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 0.5, # ARs use synced pulses for frame starts
|
||||
|
||||
Executable → Regular
+3
-8
@@ -1,6 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
import time
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
from openpilot.selfdrive.test.helpers import with_processes, phone_only
|
||||
@@ -9,9 +7,9 @@ from openpilot.system.camerad.snapshot.snapshot import get_snapshots
|
||||
TEST_TIME = 45
|
||||
REPEAT = 5
|
||||
|
||||
class TestCamerad(unittest.TestCase):
|
||||
class TestCamerad:
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
def setup_class(cls):
|
||||
pass
|
||||
|
||||
def _numpy_rgb2gray(self, im):
|
||||
@@ -49,7 +47,4 @@ class TestCamerad(unittest.TestCase):
|
||||
|
||||
passed += int(res)
|
||||
time.sleep(2)
|
||||
self.assertGreaterEqual(passed, REPEAT)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
assert passed >= REPEAT
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "cereal/messaging/messaging.h"
|
||||
#include "cereal/gen/cpp/log.capnp.h"
|
||||
|
||||
// no-op base hw class
|
||||
class HardwareNone {
|
||||
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from openpilot.common.realtime import DT_HW
|
||||
from openpilot.common.numpy_fast import interp
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.selfdrive.controls.lib.pid import PIDController
|
||||
|
||||
class BaseFanController(ABC):
|
||||
@abstractmethod
|
||||
def update(self, cur_temp: float, ignition: bool) -> int:
|
||||
pass
|
||||
|
||||
|
||||
class TiciFanController(BaseFanController):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
cloudlog.info("Setting up TICI fan handler")
|
||||
|
||||
self.last_ignition = False
|
||||
self.controller = PIDController(k_p=0, k_i=4e-3, k_f=1, rate=(1 / DT_HW))
|
||||
|
||||
def update(self, cur_temp: float, ignition: bool) -> int:
|
||||
self.controller.neg_limit = -(100 if ignition else 30)
|
||||
self.controller.pos_limit = -(30 if ignition else 0)
|
||||
|
||||
if ignition != self.last_ignition:
|
||||
self.controller.reset()
|
||||
|
||||
error = 70 - cur_temp
|
||||
fan_pwr_out = -int(self.controller.update(
|
||||
error=error,
|
||||
feedforward=interp(cur_temp, [60.0, 100.0], [0, -100])
|
||||
))
|
||||
|
||||
self.last_ignition = ignition
|
||||
return fan_pwr_out
|
||||
|
||||
Executable
+474
@@ -0,0 +1,474 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict, namedtuple
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from cereal import log
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.dict_helpers import strip_deprecated_keys
|
||||
from openpilot.common.filter_simple import FirstOrderFilter
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.realtime import DT_HW
|
||||
from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
|
||||
from openpilot.system.hardware import HARDWARE, TICI, AGNOS
|
||||
from openpilot.system.loggerd.config import get_available_percent
|
||||
from openpilot.system.statsd import statlog
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.hardware.power_monitoring import PowerMonitoring
|
||||
from openpilot.system.hardware.fan_controller import TiciFanController
|
||||
from openpilot.system.version import terms_version, training_version
|
||||
|
||||
ThermalStatus = log.DeviceState.ThermalStatus
|
||||
NetworkType = log.DeviceState.NetworkType
|
||||
NetworkStrength = log.DeviceState.NetworkStrength
|
||||
CURRENT_TAU = 15. # 15s time constant
|
||||
TEMP_TAU = 5. # 5s time constant
|
||||
DISCONNECT_TIMEOUT = 5. # wait 5 seconds before going offroad after disconnect so you get an alert
|
||||
PANDA_STATES_TIMEOUT = round(1000 / SERVICE_LIST['pandaStates'].frequency * 1.5) # 1.5x the expected pandaState frequency
|
||||
|
||||
ThermalBand = namedtuple("ThermalBand", ['min_temp', 'max_temp'])
|
||||
HardwareState = namedtuple("HardwareState", ['network_type', 'network_info', 'network_strength', 'network_stats',
|
||||
'network_metered', 'nvme_temps', 'modem_temps'])
|
||||
|
||||
# List of thermal bands. We will stay within this region as long as we are within the bounds.
|
||||
# When exiting the bounds, we'll jump to the lower or higher band. Bands are ordered in the dict.
|
||||
THERMAL_BANDS = OrderedDict({
|
||||
ThermalStatus.green: ThermalBand(None, 80.0),
|
||||
ThermalStatus.yellow: ThermalBand(75.0, 96.0),
|
||||
ThermalStatus.red: ThermalBand(88.0, 107.),
|
||||
ThermalStatus.danger: ThermalBand(94.0, None),
|
||||
})
|
||||
|
||||
# Override to highest thermal band when offroad and above this temp
|
||||
OFFROAD_DANGER_TEMP = 75
|
||||
|
||||
prev_offroad_states: dict[str, tuple[bool, str | None]] = {}
|
||||
|
||||
tz_by_type: dict[str, int] | None = None
|
||||
def populate_tz_by_type():
|
||||
global tz_by_type
|
||||
tz_by_type = {}
|
||||
for n in os.listdir("/sys/devices/virtual/thermal"):
|
||||
if not n.startswith("thermal_zone"):
|
||||
continue
|
||||
with open(os.path.join("/sys/devices/virtual/thermal", n, "type")) as f:
|
||||
tz_by_type[f.read().strip()] = int(n.removeprefix("thermal_zone"))
|
||||
|
||||
def read_tz(x):
|
||||
if x is None:
|
||||
return 0
|
||||
|
||||
if isinstance(x, str):
|
||||
if tz_by_type is None:
|
||||
populate_tz_by_type()
|
||||
x = tz_by_type[x]
|
||||
|
||||
try:
|
||||
with open(f"/sys/devices/virtual/thermal/thermal_zone{x}/temp") as f:
|
||||
return int(f.read())
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
|
||||
|
||||
def read_thermal(thermal_config):
|
||||
dat = messaging.new_message('deviceState', valid=True)
|
||||
dat.deviceState.cpuTempC = [read_tz(z) / thermal_config.cpu[1] for z in thermal_config.cpu[0]]
|
||||
dat.deviceState.gpuTempC = [read_tz(z) / thermal_config.gpu[1] for z in thermal_config.gpu[0]]
|
||||
dat.deviceState.memoryTempC = read_tz(thermal_config.mem[0]) / thermal_config.mem[1]
|
||||
dat.deviceState.pmicTempC = [read_tz(z) / thermal_config.pmic[1] for z in thermal_config.pmic[0]]
|
||||
return dat
|
||||
|
||||
|
||||
def set_offroad_alert_if_changed(offroad_alert: str, show_alert: bool, extra_text: str | None=None):
|
||||
if prev_offroad_states.get(offroad_alert, None) == (show_alert, extra_text):
|
||||
return
|
||||
prev_offroad_states[offroad_alert] = (show_alert, extra_text)
|
||||
set_offroad_alert(offroad_alert, show_alert, extra_text)
|
||||
|
||||
|
||||
def hw_state_thread(end_event, hw_queue):
|
||||
"""Handles non critical hardware state, and sends over queue"""
|
||||
count = 0
|
||||
prev_hw_state = None
|
||||
|
||||
modem_version = None
|
||||
modem_nv = None
|
||||
modem_configured = False
|
||||
modem_restarted = False
|
||||
modem_missing_count = 0
|
||||
|
||||
while not end_event.is_set():
|
||||
# these are expensive calls. update every 10s
|
||||
if (count % int(10. / DT_HW)) == 0:
|
||||
try:
|
||||
network_type = HARDWARE.get_network_type()
|
||||
modem_temps = HARDWARE.get_modem_temperatures()
|
||||
if len(modem_temps) == 0 and prev_hw_state is not None:
|
||||
modem_temps = prev_hw_state.modem_temps
|
||||
|
||||
# Log modem version once
|
||||
if AGNOS and ((modem_version is None) or (modem_nv is None)):
|
||||
modem_version = HARDWARE.get_modem_version()
|
||||
modem_nv = HARDWARE.get_modem_nv()
|
||||
|
||||
if (modem_version is not None) and (modem_nv is not None):
|
||||
cloudlog.event("modem version", version=modem_version, nv=modem_nv)
|
||||
else:
|
||||
if not modem_restarted:
|
||||
# TODO: we may be able to remove this with a MM update
|
||||
# ModemManager's probing on startup can fail
|
||||
# rarely, restart the service to probe again.
|
||||
modem_missing_count += 1
|
||||
if modem_missing_count > 3:
|
||||
modem_restarted = True
|
||||
cloudlog.event("restarting ModemManager")
|
||||
os.system("sudo systemctl restart --no-block ModemManager")
|
||||
|
||||
tx, rx = HARDWARE.get_modem_data_usage()
|
||||
|
||||
hw_state = HardwareState(
|
||||
network_type=network_type,
|
||||
network_info=HARDWARE.get_network_info(),
|
||||
network_strength=HARDWARE.get_network_strength(network_type),
|
||||
network_stats={'wwanTx': tx, 'wwanRx': rx},
|
||||
network_metered=HARDWARE.get_network_metered(network_type),
|
||||
nvme_temps=HARDWARE.get_nvme_temperatures(),
|
||||
modem_temps=modem_temps,
|
||||
)
|
||||
|
||||
try:
|
||||
hw_queue.put_nowait(hw_state)
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
# TODO: remove this once the config is in AGNOS
|
||||
if not modem_configured and len(HARDWARE.get_sim_info().get('sim_id', '')) > 0:
|
||||
cloudlog.warning("configuring modem")
|
||||
HARDWARE.configure_modem()
|
||||
modem_configured = True
|
||||
|
||||
prev_hw_state = hw_state
|
||||
except Exception:
|
||||
cloudlog.exception("Error getting hardware state")
|
||||
|
||||
count += 1
|
||||
time.sleep(DT_HW)
|
||||
|
||||
|
||||
def hardware_thread(end_event, hw_queue) -> None:
|
||||
pm = messaging.PubMaster(['deviceState'])
|
||||
sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "controlsState", "pandaStates"], poll="pandaStates")
|
||||
|
||||
count = 0
|
||||
|
||||
onroad_conditions: dict[str, bool] = {
|
||||
"ignition": False,
|
||||
}
|
||||
startup_conditions: dict[str, bool] = {}
|
||||
startup_conditions_prev: dict[str, bool] = {}
|
||||
|
||||
off_ts: float | None = None
|
||||
started_ts: float | None = None
|
||||
started_seen = False
|
||||
startup_blocked_ts: float | None = None
|
||||
thermal_status = ThermalStatus.yellow
|
||||
|
||||
last_hw_state = HardwareState(
|
||||
network_type=NetworkType.none,
|
||||
network_info=None,
|
||||
network_metered=False,
|
||||
network_strength=NetworkStrength.unknown,
|
||||
network_stats={'wwanTx': -1, 'wwanRx': -1},
|
||||
nvme_temps=[],
|
||||
modem_temps=[],
|
||||
)
|
||||
|
||||
all_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
|
||||
offroad_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
|
||||
should_start_prev = False
|
||||
in_car = False
|
||||
engaged_prev = False
|
||||
|
||||
params = Params()
|
||||
power_monitor = PowerMonitoring()
|
||||
|
||||
HARDWARE.initialize_hardware()
|
||||
thermal_config = HARDWARE.get_thermal_config()
|
||||
|
||||
fan_controller = None
|
||||
|
||||
while not end_event.is_set():
|
||||
sm.update(PANDA_STATES_TIMEOUT)
|
||||
|
||||
pandaStates = sm['pandaStates']
|
||||
peripheralState = sm['peripheralState']
|
||||
peripheral_panda_present = peripheralState.pandaType != log.PandaState.PandaType.unknown
|
||||
|
||||
if sm.updated['pandaStates'] and len(pandaStates) > 0:
|
||||
|
||||
# Set ignition based on any panda connected
|
||||
onroad_conditions["ignition"] = any(ps.ignitionLine or ps.ignitionCan for ps in pandaStates if ps.pandaType != log.PandaState.PandaType.unknown)
|
||||
|
||||
pandaState = pandaStates[0]
|
||||
|
||||
in_car = pandaState.harnessStatus != log.PandaState.HarnessStatus.notConnected
|
||||
|
||||
# Setup fan handler on first connect to panda
|
||||
if fan_controller is None and peripheral_panda_present:
|
||||
if TICI:
|
||||
fan_controller = TiciFanController()
|
||||
|
||||
elif (time.monotonic() - sm.recv_time['pandaStates']) > DISCONNECT_TIMEOUT:
|
||||
if onroad_conditions["ignition"]:
|
||||
onroad_conditions["ignition"] = False
|
||||
cloudlog.error("panda timed out onroad")
|
||||
|
||||
# Run at 2Hz, plus rising edge of ignition
|
||||
ign_edge = started_ts is None and onroad_conditions["ignition"]
|
||||
if (sm.frame % round(SERVICE_LIST['pandaStates'].frequency * DT_HW) != 0) and not ign_edge:
|
||||
continue
|
||||
|
||||
msg = read_thermal(thermal_config)
|
||||
msg.deviceState.deviceType = HARDWARE.get_device_type()
|
||||
|
||||
try:
|
||||
last_hw_state = hw_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
msg.deviceState.freeSpacePercent = get_available_percent(default=100.0)
|
||||
msg.deviceState.memoryUsagePercent = int(round(psutil.virtual_memory().percent))
|
||||
msg.deviceState.gpuUsagePercent = int(round(HARDWARE.get_gpu_usage_percent()))
|
||||
online_cpu_usage = [int(round(n)) for n in psutil.cpu_percent(percpu=True)]
|
||||
offline_cpu_usage = [0., ] * (len(msg.deviceState.cpuTempC) - len(online_cpu_usage))
|
||||
msg.deviceState.cpuUsagePercent = online_cpu_usage + offline_cpu_usage
|
||||
|
||||
msg.deviceState.networkType = last_hw_state.network_type
|
||||
msg.deviceState.networkMetered = last_hw_state.network_metered
|
||||
msg.deviceState.networkStrength = last_hw_state.network_strength
|
||||
msg.deviceState.networkStats = last_hw_state.network_stats
|
||||
if last_hw_state.network_info is not None:
|
||||
msg.deviceState.networkInfo = last_hw_state.network_info
|
||||
|
||||
msg.deviceState.nvmeTempC = last_hw_state.nvme_temps
|
||||
msg.deviceState.modemTempC = last_hw_state.modem_temps
|
||||
|
||||
msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness()
|
||||
|
||||
# this subset is only used for offroad
|
||||
temp_sources = [
|
||||
msg.deviceState.memoryTempC,
|
||||
max(msg.deviceState.cpuTempC),
|
||||
max(msg.deviceState.gpuTempC),
|
||||
]
|
||||
offroad_comp_temp = offroad_temp_filter.update(max(temp_sources))
|
||||
|
||||
# this drives the thermal status while onroad
|
||||
temp_sources.append(max(msg.deviceState.pmicTempC))
|
||||
all_comp_temp = all_temp_filter.update(max(temp_sources))
|
||||
msg.deviceState.maxTempC = all_comp_temp
|
||||
|
||||
if fan_controller is not None:
|
||||
msg.deviceState.fanSpeedPercentDesired = fan_controller.update(all_comp_temp, onroad_conditions["ignition"])
|
||||
|
||||
is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (time.monotonic() - off_ts > 60 * 5))
|
||||
if is_offroad_for_5_min and offroad_comp_temp > OFFROAD_DANGER_TEMP:
|
||||
# if device is offroad and already hot without the extra onroad load,
|
||||
# we want to cool down first before increasing load
|
||||
thermal_status = ThermalStatus.danger
|
||||
else:
|
||||
current_band = THERMAL_BANDS[thermal_status]
|
||||
band_idx = list(THERMAL_BANDS.keys()).index(thermal_status)
|
||||
if current_band.min_temp is not None and all_comp_temp < current_band.min_temp:
|
||||
thermal_status = list(THERMAL_BANDS.keys())[band_idx - 1]
|
||||
elif current_band.max_temp is not None and all_comp_temp > current_band.max_temp:
|
||||
thermal_status = list(THERMAL_BANDS.keys())[band_idx + 1]
|
||||
|
||||
# **** starting logic ****
|
||||
|
||||
startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate")
|
||||
startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall")
|
||||
startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version
|
||||
|
||||
# with 2% left, we killall, otherwise the phone will take a long time to boot
|
||||
startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2
|
||||
startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version
|
||||
startup_conditions["not_driver_view"] = not params.get_bool("IsDriverViewEnabled")
|
||||
startup_conditions["not_taking_snapshot"] = not params.get_bool("IsTakingSnapshot")
|
||||
|
||||
# must be at an engageable thermal band to go onroad
|
||||
startup_conditions["device_temp_engageable"] = thermal_status < ThermalStatus.red
|
||||
|
||||
# ensure device is fully booted
|
||||
startup_conditions["device_booted"] = startup_conditions.get("device_booted", False) or HARDWARE.booted()
|
||||
|
||||
# if the temperature enters the danger zone, go offroad to cool down
|
||||
onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger
|
||||
extra_text = f"{offroad_comp_temp:.1f}C"
|
||||
show_alert = (not onroad_conditions["device_temp_good"] or not startup_conditions["device_temp_engageable"]) and onroad_conditions["ignition"]
|
||||
set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", show_alert, extra_text=extra_text)
|
||||
|
||||
# TODO: this should move to TICI.initialize_hardware, but we currently can't import params there
|
||||
if TICI:
|
||||
if not os.path.isfile("/persist/comma/living-in-the-moment"):
|
||||
if not Path("/data/media").is_mount():
|
||||
set_offroad_alert_if_changed("Offroad_StorageMissing", True)
|
||||
else:
|
||||
# check for bad NVMe
|
||||
try:
|
||||
with open("/sys/block/nvme0n1/device/model") as f:
|
||||
model = f.read().strip()
|
||||
if not model.startswith("Samsung SSD 980") and params.get("Offroad_BadNvme") is None:
|
||||
set_offroad_alert_if_changed("Offroad_BadNvme", True)
|
||||
cloudlog.event("Unsupported NVMe", model=model, error=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Handle offroad/onroad transition
|
||||
should_start = all(onroad_conditions.values())
|
||||
if started_ts is None:
|
||||
should_start = should_start and all(startup_conditions.values())
|
||||
|
||||
if should_start != should_start_prev or (count == 0):
|
||||
params.put_bool("IsEngaged", False)
|
||||
engaged_prev = False
|
||||
HARDWARE.set_power_save(not should_start)
|
||||
|
||||
if sm.updated['controlsState']:
|
||||
engaged = sm['controlsState'].enabled
|
||||
if engaged != engaged_prev:
|
||||
params.put_bool("IsEngaged", engaged)
|
||||
engaged_prev = engaged
|
||||
|
||||
try:
|
||||
with open('/dev/kmsg', 'w') as kmsg:
|
||||
kmsg.write(f"<3>[hardware] engaged: {engaged}\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if should_start:
|
||||
off_ts = None
|
||||
if started_ts is None:
|
||||
started_ts = time.monotonic()
|
||||
started_seen = True
|
||||
if startup_blocked_ts is not None:
|
||||
cloudlog.event("Startup after block", block_duration=(time.monotonic() - startup_blocked_ts),
|
||||
startup_conditions=startup_conditions, onroad_conditions=onroad_conditions,
|
||||
startup_conditions_prev=startup_conditions_prev, error=True)
|
||||
startup_blocked_ts = None
|
||||
else:
|
||||
if onroad_conditions["ignition"] and (startup_conditions != startup_conditions_prev):
|
||||
cloudlog.event("Startup blocked", startup_conditions=startup_conditions, onroad_conditions=onroad_conditions, error=True)
|
||||
startup_conditions_prev = startup_conditions.copy()
|
||||
startup_blocked_ts = time.monotonic()
|
||||
|
||||
started_ts = None
|
||||
if off_ts is None:
|
||||
off_ts = time.monotonic()
|
||||
|
||||
# Offroad power monitoring
|
||||
voltage = None if peripheralState.pandaType == log.PandaState.PandaType.unknown else peripheralState.voltage
|
||||
power_monitor.calculate(voltage, onroad_conditions["ignition"])
|
||||
msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used()
|
||||
msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity())
|
||||
current_power_draw = HARDWARE.get_current_power_draw()
|
||||
statlog.sample("power_draw", current_power_draw)
|
||||
msg.deviceState.powerDrawW = current_power_draw
|
||||
|
||||
som_power_draw = HARDWARE.get_som_power_draw()
|
||||
statlog.sample("som_power_draw", som_power_draw)
|
||||
msg.deviceState.somPowerDrawW = som_power_draw
|
||||
|
||||
# Check if we need to shut down
|
||||
if power_monitor.should_shutdown(onroad_conditions["ignition"], in_car, off_ts, started_seen):
|
||||
cloudlog.warning(f"shutting device down, offroad since {off_ts}")
|
||||
params.put_bool("DoShutdown", True)
|
||||
|
||||
msg.deviceState.started = started_ts is not None
|
||||
msg.deviceState.startedMonoTime = int(1e9*(started_ts or 0))
|
||||
|
||||
last_ping = params.get("LastAthenaPingTime")
|
||||
if last_ping is not None:
|
||||
msg.deviceState.lastAthenaPingTime = int(last_ping)
|
||||
|
||||
msg.deviceState.thermalStatus = thermal_status
|
||||
pm.send("deviceState", msg)
|
||||
|
||||
# Log to statsd
|
||||
statlog.gauge("free_space_percent", msg.deviceState.freeSpacePercent)
|
||||
statlog.gauge("gpu_usage_percent", msg.deviceState.gpuUsagePercent)
|
||||
statlog.gauge("memory_usage_percent", msg.deviceState.memoryUsagePercent)
|
||||
for i, usage in enumerate(msg.deviceState.cpuUsagePercent):
|
||||
statlog.gauge(f"cpu{i}_usage_percent", usage)
|
||||
for i, temp in enumerate(msg.deviceState.cpuTempC):
|
||||
statlog.gauge(f"cpu{i}_temperature", temp)
|
||||
for i, temp in enumerate(msg.deviceState.gpuTempC):
|
||||
statlog.gauge(f"gpu{i}_temperature", temp)
|
||||
statlog.gauge("memory_temperature", msg.deviceState.memoryTempC)
|
||||
for i, temp in enumerate(msg.deviceState.pmicTempC):
|
||||
statlog.gauge(f"pmic{i}_temperature", temp)
|
||||
for i, temp in enumerate(last_hw_state.nvme_temps):
|
||||
statlog.gauge(f"nvme_temperature{i}", temp)
|
||||
for i, temp in enumerate(last_hw_state.modem_temps):
|
||||
statlog.gauge(f"modem_temperature{i}", temp)
|
||||
statlog.gauge("fan_speed_percent_desired", msg.deviceState.fanSpeedPercentDesired)
|
||||
statlog.gauge("screen_brightness_percent", msg.deviceState.screenBrightnessPercent)
|
||||
|
||||
# report to server once every 10 minutes
|
||||
rising_edge_started = should_start and not should_start_prev
|
||||
if rising_edge_started or (count % int(600. / DT_HW)) == 0:
|
||||
dat = {
|
||||
'count': count,
|
||||
'pandaStates': [strip_deprecated_keys(p.to_dict()) for p in pandaStates],
|
||||
'peripheralState': strip_deprecated_keys(peripheralState.to_dict()),
|
||||
'location': (strip_deprecated_keys(sm["gpsLocationExternal"].to_dict()) if sm.alive["gpsLocationExternal"] else None),
|
||||
'deviceState': strip_deprecated_keys(msg.to_dict())
|
||||
}
|
||||
cloudlog.event("STATUS_PACKET", **dat)
|
||||
|
||||
# save last one before going onroad
|
||||
if rising_edge_started:
|
||||
try:
|
||||
params.put("LastOffroadStatusPacket", json.dumps(dat))
|
||||
except Exception:
|
||||
cloudlog.exception("failed to save offroad status")
|
||||
|
||||
params.put_bool_nonblocking("NetworkMetered", msg.deviceState.networkMetered)
|
||||
|
||||
count += 1
|
||||
should_start_prev = should_start
|
||||
|
||||
|
||||
def main():
|
||||
hw_queue = queue.Queue(maxsize=1)
|
||||
end_event = threading.Event()
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=hw_state_thread, args=(end_event, hw_queue)),
|
||||
threading.Thread(target=hardware_thread, args=(end_event, hw_queue)),
|
||||
]
|
||||
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
if not all(t.is_alive() for t in threads):
|
||||
break
|
||||
finally:
|
||||
end_event.set()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
import time
|
||||
import threading
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.hardware import HARDWARE
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.statsd import statlog
|
||||
|
||||
CAR_VOLTAGE_LOW_PASS_K = 0.011 # LPF gain for 45s tau (dt/tau / (dt/tau + 1))
|
||||
|
||||
# While driving, a battery charges completely in about 30-60 minutes
|
||||
CAR_BATTERY_CAPACITY_uWh = 30e6
|
||||
CAR_CHARGING_RATE_W = 45
|
||||
|
||||
VBATT_PAUSE_CHARGING = 11.8 # Lower limit on the LPF car battery voltage
|
||||
MAX_TIME_OFFROAD_S = 30*3600
|
||||
MIN_ON_TIME_S = 3600
|
||||
DELAY_SHUTDOWN_TIME_S = 300 # Wait at least DELAY_SHUTDOWN_TIME_S seconds after offroad_time to shutdown.
|
||||
VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 60
|
||||
|
||||
class PowerMonitoring:
|
||||
def __init__(self):
|
||||
self.params = Params()
|
||||
self.last_measurement_time = None # Used for integration delta
|
||||
self.last_save_time = 0 # Used for saving current value in a param
|
||||
self.power_used_uWh = 0 # Integrated power usage in uWh since going into offroad
|
||||
self.next_pulsed_measurement_time = None
|
||||
self.car_voltage_mV = 12e3 # Low-passed version of peripheralState voltage
|
||||
self.car_voltage_instant_mV = 12e3 # Last value of peripheralState voltage
|
||||
self.integration_lock = threading.Lock()
|
||||
|
||||
car_battery_capacity_uWh = self.params.get("CarBatteryCapacity")
|
||||
if car_battery_capacity_uWh is None:
|
||||
car_battery_capacity_uWh = 0
|
||||
|
||||
# Reset capacity if it's low
|
||||
self.car_battery_capacity_uWh = max((CAR_BATTERY_CAPACITY_uWh / 10), int(car_battery_capacity_uWh))
|
||||
|
||||
# Calculation tick
|
||||
def calculate(self, voltage: int | None, ignition: bool):
|
||||
try:
|
||||
now = time.monotonic()
|
||||
|
||||
# If peripheralState is None, we're probably not in a car, so we don't care
|
||||
if voltage is None:
|
||||
with self.integration_lock:
|
||||
self.last_measurement_time = None
|
||||
self.next_pulsed_measurement_time = None
|
||||
self.power_used_uWh = 0
|
||||
return
|
||||
|
||||
# Low-pass battery voltage
|
||||
self.car_voltage_instant_mV = voltage
|
||||
self.car_voltage_mV = ((voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K)))
|
||||
statlog.gauge("car_voltage", self.car_voltage_mV / 1e3)
|
||||
|
||||
# Cap the car battery power and save it in a param every 10-ish seconds
|
||||
self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0)
|
||||
self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh)
|
||||
if now - self.last_save_time >= 10:
|
||||
self.params.put_nonblocking("CarBatteryCapacity", str(int(self.car_battery_capacity_uWh)))
|
||||
self.last_save_time = now
|
||||
|
||||
# First measurement, set integration time
|
||||
with self.integration_lock:
|
||||
if self.last_measurement_time is None:
|
||||
self.last_measurement_time = now
|
||||
return
|
||||
|
||||
if ignition:
|
||||
# If there is ignition, we integrate the charging rate of the car
|
||||
with self.integration_lock:
|
||||
self.power_used_uWh = 0
|
||||
integration_time_h = (now - self.last_measurement_time) / 3600
|
||||
if integration_time_h < 0:
|
||||
raise ValueError(f"Negative integration time: {integration_time_h}h")
|
||||
self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W * 1e6 * integration_time_h)
|
||||
self.last_measurement_time = now
|
||||
else:
|
||||
# Get current power draw somehow
|
||||
current_power = HARDWARE.get_current_power_draw()
|
||||
|
||||
# Do the integration
|
||||
self._perform_integration(now, current_power)
|
||||
except Exception:
|
||||
cloudlog.exception("Power monitoring calculation failed")
|
||||
|
||||
def _perform_integration(self, t: float, current_power: float) -> None:
|
||||
with self.integration_lock:
|
||||
try:
|
||||
if self.last_measurement_time:
|
||||
integration_time_h = (t - self.last_measurement_time) / 3600
|
||||
power_used = (current_power * 1000000) * integration_time_h
|
||||
if power_used < 0:
|
||||
raise ValueError(f"Negative power used! Integration time: {integration_time_h} h Current Power: {power_used} uWh")
|
||||
self.power_used_uWh += power_used
|
||||
self.car_battery_capacity_uWh -= power_used
|
||||
self.last_measurement_time = t
|
||||
except Exception:
|
||||
cloudlog.exception("Integration failed")
|
||||
|
||||
# Get the power usage
|
||||
def get_power_used(self) -> int:
|
||||
return int(self.power_used_uWh)
|
||||
|
||||
def get_car_battery_capacity(self) -> int:
|
||||
return int(self.car_battery_capacity_uWh)
|
||||
|
||||
# See if we need to shutdown
|
||||
def should_shutdown(self, ignition: bool, in_car: bool, offroad_timestamp: float | None, started_seen: bool):
|
||||
if offroad_timestamp is None:
|
||||
return False
|
||||
|
||||
now = time.monotonic()
|
||||
should_shutdown = False
|
||||
offroad_time = (now - offroad_timestamp)
|
||||
low_voltage_shutdown = (self.car_voltage_mV < (VBATT_PAUSE_CHARGING * 1e3) and
|
||||
offroad_time > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S)
|
||||
should_shutdown |= offroad_time > MAX_TIME_OFFROAD_S
|
||||
should_shutdown |= low_voltage_shutdown
|
||||
should_shutdown |= (self.car_battery_capacity_uWh <= 0)
|
||||
should_shutdown &= not ignition
|
||||
should_shutdown &= (not self.params.get_bool("DisablePowerDown"))
|
||||
should_shutdown &= in_car
|
||||
should_shutdown &= offroad_time > DELAY_SHUTDOWN_TIME_S
|
||||
should_shutdown |= self.params.get_bool("ForcePowerDown")
|
||||
should_shutdown &= started_seen or (now > MIN_ON_TIME_S)
|
||||
return should_shutdown
|
||||
@@ -0,0 +1,50 @@
|
||||
import pytest
|
||||
|
||||
from openpilot.system.hardware.fan_controller import TiciFanController
|
||||
|
||||
ALL_CONTROLLERS = [TiciFanController]
|
||||
|
||||
def patched_controller(mocker, controller_class):
|
||||
mocker.patch("os.system", new=mocker.Mock())
|
||||
return controller_class()
|
||||
|
||||
class TestFanController:
|
||||
def wind_up(self, controller, ignition=True):
|
||||
for _ in range(1000):
|
||||
controller.update(100, ignition)
|
||||
|
||||
def wind_down(self, controller, ignition=False):
|
||||
for _ in range(1000):
|
||||
controller.update(10, ignition)
|
||||
|
||||
@pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
|
||||
def test_hot_onroad(self, mocker, controller_class):
|
||||
controller = patched_controller(mocker, controller_class)
|
||||
self.wind_up(controller)
|
||||
assert controller.update(100, True) >= 70
|
||||
|
||||
@pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
|
||||
def test_offroad_limits(self, mocker, controller_class):
|
||||
controller = patched_controller(mocker, controller_class)
|
||||
self.wind_up(controller)
|
||||
assert controller.update(100, False) <= 30
|
||||
|
||||
@pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
|
||||
def test_no_fan_wear(self, mocker, controller_class):
|
||||
controller = patched_controller(mocker, controller_class)
|
||||
self.wind_down(controller)
|
||||
assert controller.update(10, False) == 0
|
||||
|
||||
@pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
|
||||
def test_limited(self, mocker, controller_class):
|
||||
controller = patched_controller(mocker, controller_class)
|
||||
self.wind_up(controller, True)
|
||||
assert controller.update(100, True) == 100
|
||||
|
||||
@pytest.mark.parametrize("controller_class", ALL_CONTROLLERS)
|
||||
def test_windup_speed(self, mocker, controller_class):
|
||||
controller = patched_controller(mocker, controller_class)
|
||||
self.wind_down(controller, True)
|
||||
for _ in range(10):
|
||||
controller.update(90, True)
|
||||
assert controller.update(90, True) >= 60
|
||||
@@ -0,0 +1,199 @@
|
||||
import pytest
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.hardware.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \
|
||||
CAR_CHARGING_RATE_W, VBATT_PAUSE_CHARGING, DELAY_SHUTDOWN_TIME_S
|
||||
|
||||
# Create fake time
|
||||
ssb = 0.
|
||||
def mock_time_monotonic():
|
||||
global ssb
|
||||
ssb += 1.
|
||||
return ssb
|
||||
|
||||
TEST_DURATION_S = 50
|
||||
GOOD_VOLTAGE = 12 * 1e3
|
||||
VOLTAGE_BELOW_PAUSE_CHARGING = (VBATT_PAUSE_CHARGING - 1) * 1e3
|
||||
|
||||
def pm_patch(mocker, name, value, constant=False):
|
||||
if constant:
|
||||
mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", value)
|
||||
else:
|
||||
mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", return_value=value)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_time(mocker):
|
||||
mocker.patch("time.monotonic", mock_time_monotonic)
|
||||
|
||||
|
||||
class TestPowerMonitoring:
|
||||
def setup_method(self):
|
||||
self.params = Params()
|
||||
|
||||
# Test to see that it doesn't do anything when pandaState is None
|
||||
def test_panda_state_present(self):
|
||||
pm = PowerMonitoring()
|
||||
for _ in range(10):
|
||||
pm.calculate(None, None)
|
||||
assert pm.get_power_used() == 0
|
||||
assert pm.get_car_battery_capacity() == (CAR_BATTERY_CAPACITY_uWh / 10)
|
||||
|
||||
# Test to see that it doesn't integrate offroad when ignition is True
|
||||
def test_offroad_ignition(self):
|
||||
pm = PowerMonitoring()
|
||||
for _ in range(10):
|
||||
pm.calculate(GOOD_VOLTAGE, True)
|
||||
assert pm.get_power_used() == 0
|
||||
|
||||
# Test to see that it integrates with discharging battery
|
||||
def test_offroad_integration_discharging(self, mocker):
|
||||
POWER_DRAW = 4
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
for _ in range(TEST_DURATION_S + 1):
|
||||
pm.calculate(GOOD_VOLTAGE, False)
|
||||
expected_power_usage = ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
|
||||
assert abs(pm.get_power_used() - expected_power_usage) < 10
|
||||
|
||||
# Test to check positive integration of car_battery_capacity
|
||||
def test_car_battery_integration_onroad(self, mocker):
|
||||
POWER_DRAW = 4
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = 0
|
||||
for _ in range(TEST_DURATION_S + 1):
|
||||
pm.calculate(GOOD_VOLTAGE, True)
|
||||
expected_capacity = ((TEST_DURATION_S/3600) * CAR_CHARGING_RATE_W * 1e6)
|
||||
assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
|
||||
|
||||
# Test to check positive integration upper limit
|
||||
def test_car_battery_integration_upper_limit(self, mocker):
|
||||
POWER_DRAW = 4
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh - 1000
|
||||
for _ in range(TEST_DURATION_S + 1):
|
||||
pm.calculate(GOOD_VOLTAGE, True)
|
||||
estimated_capacity = CAR_BATTERY_CAPACITY_uWh + (CAR_CHARGING_RATE_W / 3600 * 1e6)
|
||||
assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
|
||||
|
||||
# Test to check negative integration of car_battery_capacity
|
||||
def test_car_battery_integration_offroad(self, mocker):
|
||||
POWER_DRAW = 4
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
|
||||
for _ in range(TEST_DURATION_S + 1):
|
||||
pm.calculate(GOOD_VOLTAGE, False)
|
||||
expected_capacity = CAR_BATTERY_CAPACITY_uWh - ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
|
||||
assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
|
||||
|
||||
# Test to check negative integration lower limit
|
||||
def test_car_battery_integration_lower_limit(self, mocker):
|
||||
POWER_DRAW = 4
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = 1000
|
||||
for _ in range(TEST_DURATION_S + 1):
|
||||
pm.calculate(GOOD_VOLTAGE, False)
|
||||
estimated_capacity = 0 - ((1/3600) * POWER_DRAW * 1e6)
|
||||
assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
|
||||
|
||||
# Test to check policy of stopping charging after MAX_TIME_OFFROAD_S
|
||||
def test_max_time_offroad(self, mocker):
|
||||
MOCKED_MAX_OFFROAD_TIME = 3600
|
||||
POWER_DRAW = 0 # To stop shutting down for other reasons
|
||||
pm_patch(mocker, "MAX_TIME_OFFROAD_S", MOCKED_MAX_OFFROAD_TIME, constant=True)
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
|
||||
start_time = ssb
|
||||
ignition = False
|
||||
while ssb <= start_time + MOCKED_MAX_OFFROAD_TIME:
|
||||
pm.calculate(GOOD_VOLTAGE, ignition)
|
||||
if (ssb - start_time) % 1000 == 0 and ssb < start_time + MOCKED_MAX_OFFROAD_TIME:
|
||||
assert not pm.should_shutdown(ignition, True, start_time, False)
|
||||
assert pm.should_shutdown(ignition, True, start_time, False)
|
||||
|
||||
def test_car_voltage(self, mocker):
|
||||
POWER_DRAW = 0 # To stop shutting down for other reasons
|
||||
TEST_TIME = 350
|
||||
VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 50
|
||||
pm_patch(mocker, "VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S", VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S, constant=True)
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
|
||||
ignition = False
|
||||
start_time = ssb
|
||||
for i in range(TEST_TIME):
|
||||
pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
|
||||
if i % 10 == 0:
|
||||
assert pm.should_shutdown(ignition, True, start_time, True) == \
|
||||
(pm.car_voltage_mV < VBATT_PAUSE_CHARGING * 1e3 and \
|
||||
(ssb - start_time) > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S and \
|
||||
(ssb - start_time) > DELAY_SHUTDOWN_TIME_S)
|
||||
assert pm.should_shutdown(ignition, True, start_time, True)
|
||||
|
||||
# Test to check policy of not stopping charging when DisablePowerDown is set
|
||||
def test_disable_power_down(self, mocker):
|
||||
POWER_DRAW = 0 # To stop shutting down for other reasons
|
||||
TEST_TIME = 100
|
||||
self.params.put_bool("DisablePowerDown", True)
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
|
||||
ignition = False
|
||||
for i in range(TEST_TIME):
|
||||
pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
|
||||
if i % 10 == 0:
|
||||
assert not pm.should_shutdown(ignition, True, ssb, False)
|
||||
assert not pm.should_shutdown(ignition, True, ssb, False)
|
||||
|
||||
# Test to check policy of not stopping charging when ignition
|
||||
def test_ignition(self, mocker):
|
||||
POWER_DRAW = 0 # To stop shutting down for other reasons
|
||||
TEST_TIME = 100
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
|
||||
ignition = True
|
||||
for i in range(TEST_TIME):
|
||||
pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
|
||||
if i % 10 == 0:
|
||||
assert not pm.should_shutdown(ignition, True, ssb, False)
|
||||
assert not pm.should_shutdown(ignition, True, ssb, False)
|
||||
|
||||
# Test to check policy of not stopping charging when harness is not connected
|
||||
def test_harness_connection(self, mocker):
|
||||
POWER_DRAW = 0 # To stop shutting down for other reasons
|
||||
TEST_TIME = 100
|
||||
pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
|
||||
|
||||
ignition = False
|
||||
for i in range(TEST_TIME):
|
||||
pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
|
||||
if i % 10 == 0:
|
||||
assert not pm.should_shutdown(ignition, False, ssb, False)
|
||||
assert not pm.should_shutdown(ignition, False, ssb, False)
|
||||
|
||||
def test_delay_shutdown_time(self):
|
||||
pm = PowerMonitoring()
|
||||
pm.car_battery_capacity_uWh = 0
|
||||
ignition = False
|
||||
in_car = True
|
||||
offroad_timestamp = ssb
|
||||
started_seen = True
|
||||
pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
|
||||
|
||||
while ssb < offroad_timestamp + DELAY_SHUTDOWN_TIME_S:
|
||||
assert not pm.should_shutdown(ignition, in_car,
|
||||
offroad_timestamp,
|
||||
started_seen), \
|
||||
f"Should not shutdown before {DELAY_SHUTDOWN_TIME_S} seconds offroad time"
|
||||
assert pm.should_shutdown(ignition, in_car,
|
||||
offroad_timestamp,
|
||||
started_seen), \
|
||||
f"Should shutdown after {DELAY_SHUTDOWN_TIME_S} seconds offroad time"
|
||||
@@ -379,23 +379,19 @@ class Tici(HardwareBase):
|
||||
|
||||
# *** CPU config ***
|
||||
|
||||
# offline big cluster, leave core 4 online for boardd
|
||||
for i in range(5, 8):
|
||||
# offline big cluster, leave core 4 online for pandad
|
||||
for i in range(4, 8):
|
||||
val = '0' if powersave_enabled else '1'
|
||||
sudo_write(val, f'/sys/devices/system/cpu/cpu{i}/online')
|
||||
|
||||
for n in ('0', '4'):
|
||||
if powersave_enabled and n == '4':
|
||||
continue
|
||||
gov = 'ondemand' if powersave_enabled else 'performance'
|
||||
sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
|
||||
|
||||
# *** IRQ config ***
|
||||
|
||||
# boardd core
|
||||
affine_irq(4, "spi_geni") # SPI
|
||||
affine_irq(4, "xhci-hcd:usb3") # aux panda USB (or potentially anything else on USB)
|
||||
if "tici" in self.get_device_type():
|
||||
affine_irq(4, "xhci-hcd:usb1") # internal panda USB (also modem)
|
||||
|
||||
# GPU
|
||||
affine_irq(5, "kgsl-3d0")
|
||||
|
||||
@@ -416,7 +412,7 @@ class Tici(HardwareBase):
|
||||
if self.amplifier is not None:
|
||||
self.amplifier.initialize_configuration(self.get_device_type())
|
||||
|
||||
# Allow thermald to write engagement status to kmsg
|
||||
# Allow hardwared to write engagement status to kmsg
|
||||
os.system("sudo chmod a+w /dev/kmsg")
|
||||
|
||||
# Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
|
||||
@@ -453,6 +449,18 @@ class Tici(HardwareBase):
|
||||
sudo_write("N", "/sys/kernel/debug/msm_vidc/clock_scaling")
|
||||
sudo_write("Y", "/sys/kernel/debug/msm_vidc/disable_thermal_mitigation")
|
||||
|
||||
# pandad core
|
||||
affine_irq(3, "spi_geni") # SPI
|
||||
if "tici" in self.get_device_type():
|
||||
affine_irq(3, "xhci-hcd:usb3") # aux panda USB (or potentially anything else on USB)
|
||||
affine_irq(3, "xhci-hcd:usb1") # internal panda USB (also modem)
|
||||
try:
|
||||
pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
|
||||
subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
|
||||
subprocess.call(["sudo", "taskset", "-pc", "3", pid])
|
||||
except subprocess.CalledProcessException as e:
|
||||
print(str(e))
|
||||
|
||||
def configure_modem(self):
|
||||
sim_id = self.get_sim_info().get('sim_id', '')
|
||||
|
||||
|
||||
Executable → Regular
+2
-8
@@ -1,14 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
import requests
|
||||
|
||||
TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
||||
MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
|
||||
|
||||
|
||||
class TestAgnosUpdater(unittest.TestCase):
|
||||
class TestAgnosUpdater:
|
||||
|
||||
def test_manifest(self):
|
||||
with open(MANIFEST) as f:
|
||||
@@ -17,10 +15,6 @@ class TestAgnosUpdater(unittest.TestCase):
|
||||
for img in m:
|
||||
r = requests.head(img['url'], timeout=10)
|
||||
r.raise_for_status()
|
||||
self.assertEqual(r.headers['Content-Type'], "application/x-xz")
|
||||
assert r.headers['Content-Type'] == "application/x-xz"
|
||||
if not img['sparse']:
|
||||
assert img['hash'] == img['hash_raw']
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Executable → Regular
+7
-12
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import pytest
|
||||
import time
|
||||
import random
|
||||
import unittest
|
||||
import subprocess
|
||||
|
||||
from panda import Panda
|
||||
@@ -10,14 +9,14 @@ from openpilot.system.hardware.tici.hardware import Tici
|
||||
from openpilot.system.hardware.tici.amplifier import Amplifier
|
||||
|
||||
|
||||
class TestAmplifier(unittest.TestCase):
|
||||
class TestAmplifier:
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
def setup_class(cls):
|
||||
if not TICI:
|
||||
raise unittest.SkipTest
|
||||
pytest.skip()
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
# clear dmesg
|
||||
subprocess.check_call("sudo dmesg -C", shell=True)
|
||||
|
||||
@@ -25,7 +24,7 @@ class TestAmplifier(unittest.TestCase):
|
||||
Panda.wait_for_panda(None, 30)
|
||||
self.panda = Panda()
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
HARDWARE.reset_internal_panda()
|
||||
|
||||
def _check_for_i2c_errors(self, expected):
|
||||
@@ -68,8 +67,4 @@ class TestAmplifier(unittest.TestCase):
|
||||
if self._check_for_i2c_errors(True):
|
||||
break
|
||||
else:
|
||||
self.fail("didn't hit any i2c errors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
pytest.fail("didn't hit any i2c errors")
|
||||
|
||||
Executable → Regular
+8
-10
@@ -1,7 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
import pytest
|
||||
import time
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
from openpilot.system.hardware.tici.hardware import Tici
|
||||
@@ -10,19 +8,19 @@ HARDWARE = Tici()
|
||||
|
||||
|
||||
@pytest.mark.tici
|
||||
class TestHardware(unittest.TestCase):
|
||||
class TestHardware:
|
||||
|
||||
def test_power_save_time(self):
|
||||
ts = []
|
||||
ts = {True: [], False: []}
|
||||
for _ in range(5):
|
||||
for on in (True, False):
|
||||
st = time.monotonic()
|
||||
HARDWARE.set_power_save(on)
|
||||
ts.append(time.monotonic() - st)
|
||||
ts[on].append(time.monotonic() - st)
|
||||
|
||||
assert 0.1 < np.mean(ts) < 0.25
|
||||
assert max(ts) < 0.3
|
||||
# disabling power save is the main time-critical one
|
||||
assert 0.1 < np.mean(ts[False]) < 0.15
|
||||
assert max(ts[False]) < 0.2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
assert 0.1 < np.mean(ts[True]) < 0.35
|
||||
assert max(ts[True]) < 0.4
|
||||
|
||||
Executable → Regular
+9
-15
@@ -1,7 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
from collections import defaultdict, deque
|
||||
import pytest
|
||||
import unittest
|
||||
import time
|
||||
import numpy as np
|
||||
from dataclasses import dataclass
|
||||
@@ -12,8 +10,8 @@ from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.mock import mock_messages
|
||||
from openpilot.selfdrive.car.car_helpers import write_car_param
|
||||
from openpilot.system.hardware.tici.power_monitor import get_power
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.selfdrive.manager.manager import manager_cleanup
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.manager import manager_cleanup
|
||||
|
||||
SAMPLE_TIME = 8 # seconds to sample power
|
||||
MAX_WARMUP_TIME = 30 # seconds to wait for SAMPLE_TIME consecutive valid samples
|
||||
@@ -40,15 +38,15 @@ PROCS = [
|
||||
|
||||
|
||||
@pytest.mark.tici
|
||||
class TestPowerDraw(unittest.TestCase):
|
||||
class TestPowerDraw:
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
write_car_param()
|
||||
|
||||
# wait a bit for power save to disable
|
||||
time.sleep(5)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
manager_cleanup()
|
||||
|
||||
def get_expected_messages(self, proc):
|
||||
@@ -97,7 +95,7 @@ class TestPowerDraw(unittest.TestCase):
|
||||
return now, msg_counts, time.monotonic() - start_time - SAMPLE_TIME
|
||||
|
||||
@mock_messages(['liveLocationKalman'])
|
||||
def test_camera_procs(self):
|
||||
def test_camera_procs(self, subtests):
|
||||
baseline = get_power()
|
||||
|
||||
prev = baseline
|
||||
@@ -122,12 +120,8 @@ class TestPowerDraw(unittest.TestCase):
|
||||
expected = proc.power
|
||||
msgs_received = sum(msg_counts[msg] for msg in proc.msgs)
|
||||
tab.append([proc.name, round(expected, 2), round(cur, 2), self.get_expected_messages(proc), msgs_received, round(warmup_time[proc.name], 2)])
|
||||
with self.subTest(proc=proc.name):
|
||||
self.assertTrue(self.valid_msg_count(proc, msg_counts), f"expected {self.get_expected_messages(proc)} msgs, got {msgs_received} msgs")
|
||||
self.assertTrue(self.valid_power_draw(proc, cur), f"expected {expected:.2f}W, got {cur:.2f}W")
|
||||
with subtests.test(proc=proc.name):
|
||||
assert self.valid_msg_count(proc, msg_counts), f"expected {self.get_expected_messages(proc)} msgs, got {msgs_received} msgs"
|
||||
assert self.valid_power_draw(proc, cur), f"expected {expected:.2f}W, got {cur:.2f}W"
|
||||
print(tabulate(tab))
|
||||
print(f"Baseline {baseline:.2f}W\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
Import('env', 'cereal', 'messaging', 'common')
|
||||
Import('env', 'messaging', 'common')
|
||||
|
||||
env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11'])
|
||||
env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[messaging, common, 'systemd', 'json11'])
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
Import('env', 'arch', 'cereal', 'messaging', 'common', 'visionipc')
|
||||
Import('env', 'arch', 'messaging', 'common', 'visionipc')
|
||||
|
||||
libs = [common, cereal, messaging, visionipc,
|
||||
'zmq', 'capnp', 'kj', 'z',
|
||||
'avformat', 'avcodec', 'swscale', 'avutil',
|
||||
'yuv', 'OpenCL', 'pthread']
|
||||
libs = [common, messaging, visionipc,
|
||||
'z', 'avformat', 'avcodec', 'swscale',
|
||||
'avutil', 'yuv', 'OpenCL', 'pthread']
|
||||
|
||||
src = ['logger.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/v4l_encoder.cc']
|
||||
if arch != "larch64":
|
||||
|
||||
@@ -6,7 +6,12 @@ VideoEncoder::VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in
|
||||
out_width = encoder_info.frame_width > 0 ? encoder_info.frame_width : in_width;
|
||||
out_height = encoder_info.frame_height > 0 ? encoder_info.frame_height : in_height;
|
||||
|
||||
pm.reset(new PubMaster({encoder_info.publish_name}));
|
||||
|
||||
std::vector pubs = {encoder_info.publish_name};
|
||||
if (encoder_info.thumbnail_name != NULL) {
|
||||
pubs.push_back(encoder_info.thumbnail_name);
|
||||
}
|
||||
pm.reset(new PubMaster(pubs));
|
||||
}
|
||||
|
||||
void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra,
|
||||
@@ -40,4 +45,15 @@ void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t
|
||||
kj::ArrayOutputStream output_stream(kj::ArrayPtr<capnp::byte>(e->msg_cache.data(), bytes_size));
|
||||
capnp::writeMessage(output_stream, msg);
|
||||
e->pm->send(e->encoder_info.publish_name, e->msg_cache.data(), bytes_size);
|
||||
}
|
||||
|
||||
// Publish keyframe thumbnail
|
||||
if ((flags & V4L2_BUF_FLAG_KEYFRAME) && e->encoder_info.thumbnail_name != NULL) {
|
||||
MessageBuilder tm;
|
||||
auto thumbnail = tm.initEvent().initThumbnail();
|
||||
thumbnail.setFrameId(extra.frame_id);
|
||||
thumbnail.setTimestampEof(extra.timestamp_eof);
|
||||
thumbnail.setThumbnail(dat);
|
||||
thumbnail.setEncoding(cereal::Thumbnail::Encoding::KEYFRAME);
|
||||
pm->send(e->encoder_info.thumbnail_name, tm);
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "cereal/messaging/messaging.h"
|
||||
#include "cereal/visionipc/visionipc.h"
|
||||
#include "msgq/visionipc/visionipc.h"
|
||||
#include "common/queue.h"
|
||||
#include "system/camerad/cameras/camera_common.h"
|
||||
#include "system/loggerd/loggerd.h"
|
||||
@@ -25,6 +25,8 @@ public:
|
||||
void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
|
||||
|
||||
protected:
|
||||
void publish_thumbnail(uint32_t frame_id, uint64_t timestamp_eof, kj::ArrayPtr<capnp::byte> dat);
|
||||
|
||||
int in_width, in_height;
|
||||
int out_width, out_height;
|
||||
const EncoderInfo encoder_info;
|
||||
|
||||
@@ -215,7 +215,7 @@ void loggerd_thread() {
|
||||
const bool encoder = util::ends_with(it.name, "EncodeData");
|
||||
const bool livestream_encoder = util::starts_with(it.name, "livestream");
|
||||
if (!it.should_log && (!encoder || livestream_encoder)) continue;
|
||||
LOGD("logging %s (on port %d)", it.name.c_str(), it.port);
|
||||
LOGD("logging %s", it.name.c_str());
|
||||
|
||||
SubSocket * sock = SubSocket::create(ctx.get(), it.name);
|
||||
assert(sock != NULL);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#include "cereal/messaging/messaging.h"
|
||||
#include "cereal/services.h"
|
||||
#include "cereal/visionipc/visionipc_client.h"
|
||||
#include "msgq/visionipc/visionipc_client.h"
|
||||
#include "system/camerad/cameras/camera_common.h"
|
||||
#include "system/hardware/hw.h"
|
||||
#include "common/params.h"
|
||||
@@ -33,6 +33,7 @@ constexpr char PRESERVE_ATTR_VALUE = '1';
|
||||
class EncoderInfo {
|
||||
public:
|
||||
const char *publish_name;
|
||||
const char *thumbnail_name = NULL;
|
||||
const char *filename = NULL;
|
||||
bool record = true;
|
||||
int frame_width = -1;
|
||||
@@ -76,6 +77,7 @@ const EncoderInfo main_driver_encoder_info = {
|
||||
|
||||
const EncoderInfo stream_road_encoder_info = {
|
||||
.publish_name = "livestreamRoadEncodeData",
|
||||
//.thumbnail_name = "thumbnail",
|
||||
.encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
|
||||
.record = false,
|
||||
.bitrate = LIVESTREAM_BITRATE,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import random
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -29,12 +28,12 @@ def create_random_file(file_path: Path, size_mb: float, lock: bool = False, uplo
|
||||
if upload_xattr is not None:
|
||||
setxattr(str(file_path), uploader.UPLOAD_ATTR_NAME, upload_xattr)
|
||||
|
||||
class MockResponse():
|
||||
class MockResponse:
|
||||
def __init__(self, text, status_code):
|
||||
self.text = text
|
||||
self.status_code = status_code
|
||||
|
||||
class MockApi():
|
||||
class MockApi:
|
||||
def __init__(self, dongle_id):
|
||||
pass
|
||||
|
||||
@@ -44,7 +43,7 @@ class MockApi():
|
||||
def get_token(self):
|
||||
return "fake-token"
|
||||
|
||||
class MockApiIgnore():
|
||||
class MockApiIgnore:
|
||||
def __init__(self, dongle_id):
|
||||
pass
|
||||
|
||||
@@ -54,7 +53,7 @@ class MockApiIgnore():
|
||||
def get_token(self):
|
||||
return "fake-token"
|
||||
|
||||
class UploaderTestCase(unittest.TestCase):
|
||||
class UploaderTestCase:
|
||||
f_type = "UNKNOWN"
|
||||
|
||||
root: Path
|
||||
@@ -66,7 +65,7 @@ class UploaderTestCase(unittest.TestCase):
|
||||
def set_ignore(self):
|
||||
uploader.Api = MockApiIgnore
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
uploader.Api = MockApi
|
||||
uploader.fake_upload = True
|
||||
uploader.force_wifi = True
|
||||
|
||||
Executable → Regular
+5
-11
@@ -1,7 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
import time
|
||||
import threading
|
||||
import unittest
|
||||
from collections import namedtuple
|
||||
from pathlib import Path
|
||||
from collections.abc import Sequence
|
||||
@@ -17,9 +15,9 @@ class TestDeleter(UploaderTestCase):
|
||||
def fake_statvfs(self, d):
|
||||
return self.fake_stats
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.f_type = "fcamera.hevc"
|
||||
super().setUp()
|
||||
super().setup_method()
|
||||
self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096)
|
||||
deleter.os.statvfs = self.fake_statvfs
|
||||
|
||||
@@ -64,7 +62,7 @@ class TestDeleter(UploaderTestCase):
|
||||
finally:
|
||||
self.join_thread()
|
||||
|
||||
self.assertEqual(deleted_order, f_paths, "Files not deleted in expected order")
|
||||
assert deleted_order == f_paths, "Files not deleted in expected order"
|
||||
|
||||
def test_delete_order(self):
|
||||
self.assertDeleteOrder([
|
||||
@@ -105,7 +103,7 @@ class TestDeleter(UploaderTestCase):
|
||||
time.sleep(0.01)
|
||||
self.join_thread()
|
||||
|
||||
self.assertTrue(f_path.exists(), "File deleted with available space")
|
||||
assert f_path.exists(), "File deleted with available space"
|
||||
|
||||
def test_no_delete_with_lock_file(self):
|
||||
f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True)
|
||||
@@ -116,8 +114,4 @@ class TestDeleter(UploaderTestCase):
|
||||
time.sleep(0.01)
|
||||
self.join_thread()
|
||||
|
||||
self.assertTrue(f_path.exists(), "File deleted when locked")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
assert f_path.exists(), "File deleted when locked"
|
||||
|
||||
Executable → Regular
+19
-25
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
import math
|
||||
import os
|
||||
import pytest
|
||||
@@ -6,7 +5,6 @@ import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from parameterized import parameterized
|
||||
@@ -15,7 +13,7 @@ from tqdm import trange
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.timeout import Timeout
|
||||
from openpilot.system.hardware import TICI
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.tools.lib.logreader import LogReader
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
|
||||
@@ -29,18 +27,18 @@ CAMERAS = [
|
||||
]
|
||||
|
||||
# we check frame count, so we don't have to be too strict on size
|
||||
FILE_SIZE_TOLERANCE = 0.5
|
||||
FILE_SIZE_TOLERANCE = 0.7
|
||||
|
||||
|
||||
@pytest.mark.tici # TODO: all of loggerd should work on PC
|
||||
class TestEncoder(unittest.TestCase):
|
||||
class TestEncoder:
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self._clear_logs()
|
||||
os.environ["LOGGERD_TEST"] = "1"
|
||||
os.environ["LOGGERD_SEGMENT_LENGTH"] = str(SEGMENT_LENGTH)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
self._clear_logs()
|
||||
|
||||
def _clear_logs(self):
|
||||
@@ -85,7 +83,7 @@ class TestEncoder(unittest.TestCase):
|
||||
file_path = f"{route_prefix_path}--{i}/{camera}"
|
||||
|
||||
# check file exists
|
||||
self.assertTrue(os.path.exists(file_path), f"segment #{i}: '{file_path}' missing")
|
||||
assert os.path.exists(file_path), f"segment #{i}: '{file_path}' missing"
|
||||
|
||||
# TODO: this ffprobe call is really slow
|
||||
# check frame count
|
||||
@@ -98,13 +96,13 @@ class TestEncoder(unittest.TestCase):
|
||||
frame_count = int(probe.split('\n')[0].strip())
|
||||
counts.append(frame_count)
|
||||
|
||||
self.assertEqual(frame_count, expected_frames,
|
||||
f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}")
|
||||
assert frame_count == expected_frames, \
|
||||
f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}"
|
||||
|
||||
# sanity check file size
|
||||
file_size = os.path.getsize(file_path)
|
||||
self.assertTrue(math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE),
|
||||
f"{file_path} size {file_size} isn't close to target size {size}")
|
||||
assert math.isclose(file_size, size, rel_tol=FILE_SIZE_TOLERANCE), \
|
||||
f"{file_path} size {file_size} isn't close to target size {size}"
|
||||
|
||||
# Check encodeIdx
|
||||
if encode_idx_name is not None:
|
||||
@@ -118,24 +116,24 @@ class TestEncoder(unittest.TestCase):
|
||||
frame_idxs = [m.frameId for m in encode_msgs]
|
||||
|
||||
# Check frame count
|
||||
self.assertEqual(frame_count, len(segment_idxs))
|
||||
self.assertEqual(frame_count, len(encode_idxs))
|
||||
assert frame_count == len(segment_idxs)
|
||||
assert frame_count == len(encode_idxs)
|
||||
|
||||
# Check for duplicates or skips
|
||||
self.assertEqual(0, segment_idxs[0])
|
||||
self.assertEqual(len(set(segment_idxs)), len(segment_idxs))
|
||||
assert 0 == segment_idxs[0]
|
||||
assert len(set(segment_idxs)) == len(segment_idxs)
|
||||
|
||||
self.assertTrue(all(valid))
|
||||
assert all(valid)
|
||||
|
||||
self.assertEqual(expected_frames * i, encode_idxs[0])
|
||||
assert expected_frames * i == encode_idxs[0]
|
||||
first_frames.append(frame_idxs[0])
|
||||
self.assertEqual(len(set(encode_idxs)), len(encode_idxs))
|
||||
assert len(set(encode_idxs)) == len(encode_idxs)
|
||||
|
||||
self.assertEqual(1, len(set(first_frames)))
|
||||
assert 1 == len(set(first_frames))
|
||||
|
||||
if TICI:
|
||||
expected_frames = fps * SEGMENT_LENGTH
|
||||
self.assertEqual(min(counts), expected_frames)
|
||||
assert min(counts) == expected_frames
|
||||
shutil.rmtree(f"{route_prefix_path}--{i}")
|
||||
|
||||
try:
|
||||
@@ -150,7 +148,3 @@ class TestEncoder(unittest.TestCase):
|
||||
managed_processes['encoderd'].stop()
|
||||
managed_processes['camerad'].stop()
|
||||
managed_processes['sensord'].stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Executable → Regular
+2
-3
@@ -1,4 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
import numpy as np
|
||||
import os
|
||||
import re
|
||||
@@ -19,11 +18,11 @@ from openpilot.common.timeout import Timeout
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.system.loggerd.xattr_cache import getxattr
|
||||
from openpilot.system.loggerd.deleter import PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.system.version import get_version
|
||||
from openpilot.tools.lib.helpers import RE
|
||||
from openpilot.tools.lib.logreader import LogReader
|
||||
from cereal.visionipc import VisionIpcServer, VisionStreamType
|
||||
from msgq.visionipc import VisionIpcServer, VisionStreamType
|
||||
from openpilot.common.transformations.camera import DEVICE_CAMERAS
|
||||
|
||||
SentinelType = log.Sentinel.SentinelType
|
||||
|
||||
Executable → Regular
+25
-31
@@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import unittest
|
||||
import logging
|
||||
import json
|
||||
from pathlib import Path
|
||||
@@ -38,8 +36,8 @@ cloudlog.addHandler(log_handler)
|
||||
|
||||
|
||||
class TestUploader(UploaderTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
def setup_method(self):
|
||||
super().setup_method()
|
||||
log_handler.reset()
|
||||
|
||||
def start_thread(self):
|
||||
@@ -80,13 +78,13 @@ class TestUploader(UploaderTestCase):
|
||||
|
||||
exp_order = self.gen_order([self.seg_num], [])
|
||||
|
||||
self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored")
|
||||
self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
|
||||
self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
|
||||
assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
|
||||
assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
|
||||
assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
|
||||
for f_path in exp_order:
|
||||
self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded")
|
||||
assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
|
||||
|
||||
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
|
||||
assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
|
||||
|
||||
def test_upload_with_wrong_xattr(self):
|
||||
self.gen_files(lock=False, xattr=b'0')
|
||||
@@ -98,13 +96,13 @@ class TestUploader(UploaderTestCase):
|
||||
|
||||
exp_order = self.gen_order([self.seg_num], [])
|
||||
|
||||
self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored")
|
||||
self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
|
||||
self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
|
||||
assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
|
||||
assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
|
||||
assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
|
||||
for f_path in exp_order:
|
||||
self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded")
|
||||
assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
|
||||
|
||||
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
|
||||
assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
|
||||
|
||||
def test_upload_ignored(self):
|
||||
self.set_ignore()
|
||||
@@ -117,13 +115,13 @@ class TestUploader(UploaderTestCase):
|
||||
|
||||
exp_order = self.gen_order([self.seg_num], [])
|
||||
|
||||
self.assertTrue(len(log_handler.upload_order) == 0, "Some files were not ignored")
|
||||
self.assertFalse(len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore")
|
||||
self.assertFalse(len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice")
|
||||
assert len(log_handler.upload_order) == 0, "Some files were not ignored"
|
||||
assert not len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore"
|
||||
assert not len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice"
|
||||
for f_path in exp_order:
|
||||
self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not ignored")
|
||||
assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not ignored"
|
||||
|
||||
self.assertTrue(log_handler.upload_ignored == exp_order, "Files ignored in wrong order")
|
||||
assert log_handler.upload_ignored == exp_order, "Files ignored in wrong order"
|
||||
|
||||
def test_upload_files_in_create_order(self):
|
||||
seg1_nums = [0, 1, 2, 10, 20]
|
||||
@@ -142,13 +140,13 @@ class TestUploader(UploaderTestCase):
|
||||
time.sleep(5)
|
||||
self.join_thread()
|
||||
|
||||
self.assertTrue(len(log_handler.upload_ignored) == 0, "Some files were ignored")
|
||||
self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload")
|
||||
self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice")
|
||||
assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
|
||||
assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
|
||||
assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
|
||||
for f_path in exp_order:
|
||||
self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded")
|
||||
assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
|
||||
|
||||
self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order")
|
||||
assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
|
||||
|
||||
def test_no_upload_with_lock_file(self):
|
||||
self.start_thread()
|
||||
@@ -163,7 +161,7 @@ class TestUploader(UploaderTestCase):
|
||||
for f_path in f_paths:
|
||||
fn = f_path.with_suffix(f_path.suffix.replace(".bz2", ""))
|
||||
uploaded = UPLOAD_ATTR_NAME in os.listxattr(fn) and os.getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
|
||||
self.assertFalse(uploaded, "File upload when locked")
|
||||
assert not uploaded, "File upload when locked"
|
||||
|
||||
def test_no_upload_with_xattr(self):
|
||||
self.gen_files(lock=False, xattr=UPLOAD_ATTR_VALUE)
|
||||
@@ -173,7 +171,7 @@ class TestUploader(UploaderTestCase):
|
||||
time.sleep(5)
|
||||
self.join_thread()
|
||||
|
||||
self.assertEqual(len(log_handler.upload_order), 0, "File uploaded again")
|
||||
assert len(log_handler.upload_order) == 0, "File uploaded again"
|
||||
|
||||
def test_clear_locks_on_startup(self):
|
||||
f_paths = self.gen_files(lock=True, boot=False)
|
||||
@@ -183,8 +181,4 @@ class TestUploader(UploaderTestCase):
|
||||
|
||||
for f_path in f_paths:
|
||||
lock_path = f_path.with_suffix(f_path.suffix + ".lock")
|
||||
self.assertFalse(lock_path.is_file(), "File lock not cleared on startup")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
assert not lock_path.is_file(), "File lock not cleared on startup"
|
||||
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
# NOTE: Do NOT import anything here that needs be built (e.g. params)
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.spinner import Spinner
|
||||
from openpilot.common.text_window import TextWindow
|
||||
from openpilot.system.hardware import AGNOS
|
||||
from openpilot.common.swaglog import cloudlog, add_file_handler
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
|
||||
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
|
||||
|
||||
TOTAL_SCONS_NODES = 2820
|
||||
MAX_BUILD_PROGRESS = 100
|
||||
|
||||
def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
|
||||
env = os.environ.copy()
|
||||
env['SCONS_PROGRESS'] = "1"
|
||||
nproc = os.cpu_count()
|
||||
if nproc is None:
|
||||
nproc = 2
|
||||
|
||||
extra_args = ["--minimal"] if minimal else []
|
||||
|
||||
# building with all cores can result in using too
|
||||
# much memory, so retry with less parallelism
|
||||
compile_output: list[bytes] = []
|
||||
for n in (nproc, nproc/2, 1):
|
||||
compile_output.clear()
|
||||
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
|
||||
assert scons.stderr is not None
|
||||
|
||||
# Read progress from stderr and update spinner
|
||||
while scons.poll() is None:
|
||||
try:
|
||||
line = scons.stderr.readline()
|
||||
if line is None:
|
||||
continue
|
||||
line = line.rstrip()
|
||||
|
||||
prefix = b'progress: '
|
||||
if line.startswith(prefix):
|
||||
i = int(line[len(prefix):])
|
||||
spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
|
||||
elif len(line):
|
||||
compile_output.append(line)
|
||||
print(line.decode('utf8', 'replace'))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if scons.returncode == 0:
|
||||
break
|
||||
|
||||
if scons.returncode != 0:
|
||||
# Read remaining output
|
||||
if scons.stderr is not None:
|
||||
compile_output += scons.stderr.read().split(b'\n')
|
||||
|
||||
# Build failed log errors
|
||||
error_s = b"\n".join(compile_output).decode('utf8', 'replace')
|
||||
add_file_handler(cloudlog)
|
||||
cloudlog.error("scons build failed\n" + error_s)
|
||||
|
||||
# Show TextWindow
|
||||
spinner.close()
|
||||
if not os.getenv("CI"):
|
||||
with TextWindow("openpilot failed to build\n \n" + error_s) as t:
|
||||
t.wait_for_exit()
|
||||
exit(1)
|
||||
|
||||
# enforce max cache size
|
||||
cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
|
||||
cache_files.sort(key=lambda f: f.stat().st_mtime)
|
||||
cache_size = sum(f.stat().st_size for f in cache_files)
|
||||
for f in cache_files:
|
||||
if cache_size < MAX_CACHE_SIZE:
|
||||
break
|
||||
cache_size -= f.stat().st_size
|
||||
f.unlink()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
spinner = Spinner()
|
||||
spinner.update_progress(0, 100)
|
||||
build_metadata = get_build_metadata()
|
||||
build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS)
|
||||
@@ -0,0 +1,67 @@
|
||||
import errno
|
||||
import fcntl
|
||||
import os
|
||||
import sys
|
||||
import pathlib
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params
|
||||
|
||||
def unblock_stdout() -> None:
|
||||
# get a non-blocking stdout
|
||||
child_pid, child_pty = os.forkpty()
|
||||
if child_pid != 0: # parent
|
||||
|
||||
# child is in its own process group, manually pass kill signals
|
||||
signal.signal(signal.SIGINT, lambda signum, frame: os.kill(child_pid, signal.SIGINT))
|
||||
signal.signal(signal.SIGTERM, lambda signum, frame: os.kill(child_pid, signal.SIGTERM))
|
||||
|
||||
fcntl.fcntl(sys.stdout, fcntl.F_SETFL, fcntl.fcntl(sys.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
|
||||
|
||||
while True:
|
||||
try:
|
||||
dat = os.read(child_pty, 4096)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EIO:
|
||||
break
|
||||
continue
|
||||
|
||||
if not dat:
|
||||
break
|
||||
|
||||
try:
|
||||
sys.stdout.write(dat.decode('utf8'))
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
|
||||
# os.wait() returns a tuple with the pid and a 16 bit value
|
||||
# whose low byte is the signal number and whose high byte is the exit status
|
||||
exit_status = os.wait()[1] >> 8
|
||||
os._exit(exit_status)
|
||||
|
||||
|
||||
def write_onroad_params(started, params):
|
||||
params.put_bool("IsOnroad", started)
|
||||
params.put_bool("IsOffroad", not started)
|
||||
|
||||
|
||||
def save_bootlog():
|
||||
# copy current params
|
||||
tmp = tempfile.mkdtemp()
|
||||
params_dirname = pathlib.Path(Params().get_param_path()).name
|
||||
params_dir = os.path.join(tmp, params_dirname)
|
||||
shutil.copytree(Params().get_param_path(), params_dir, dirs_exist_ok=True)
|
||||
|
||||
def fn(tmpdir):
|
||||
env = os.environ.copy()
|
||||
env['PARAMS_COPY_PATH'] = tmpdir
|
||||
subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), env=env)
|
||||
shutil.rmtree(tmpdir)
|
||||
t = threading.Thread(target=fn, args=(tmp, ))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
Executable
+239
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
import datetime
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from cereal import log
|
||||
import cereal.messaging as messaging
|
||||
import openpilot.system.sentry as sentry
|
||||
from openpilot.common.params import Params, ParamKeyType
|
||||
from openpilot.common.text_window import TextWindow
|
||||
from openpilot.system.hardware import HARDWARE, PC
|
||||
from openpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog
|
||||
from openpilot.system.manager.process import ensure_running
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
|
||||
from openpilot.common.swaglog import cloudlog, add_file_handler
|
||||
from openpilot.system.version import get_build_metadata, terms_version, training_version
|
||||
|
||||
|
||||
|
||||
def manager_init() -> None:
|
||||
save_bootlog()
|
||||
|
||||
build_metadata = get_build_metadata()
|
||||
|
||||
params = Params()
|
||||
params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
|
||||
params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
|
||||
params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
|
||||
if build_metadata.release_channel:
|
||||
params.clear_all(ParamKeyType.DEVELOPMENT_ONLY)
|
||||
|
||||
default_params: list[tuple[str, str | bytes]] = [
|
||||
("CompletedTrainingVersion", "0"),
|
||||
("DisengageOnAccelerator", "0"),
|
||||
("GsmMetered", "1"),
|
||||
("HasAcceptedTerms", "0"),
|
||||
("LanguageSetting", "main_en"),
|
||||
("OpenpilotEnabledToggle", "1"),
|
||||
("LongitudinalPersonality", str(log.LongitudinalPersonality.standard)),
|
||||
|
||||
# dp
|
||||
("dp_device_display_off_mode", "0"),
|
||||
("dp_ui_rainbow", "0"),
|
||||
("dp_ui_flight_panel", "0"),
|
||||
("dp_long_de2e", "0"),
|
||||
("dp_long_personality_btn", "0"),
|
||||
("dp_nav_full_screen", "0"),
|
||||
]
|
||||
if not PC:
|
||||
default_params.append(("LastUpdateTime", datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat().encode('utf8')))
|
||||
|
||||
if params.get_bool("RecordFrontLock"):
|
||||
params.put_bool("RecordFront", True)
|
||||
|
||||
# set unset params
|
||||
for k, v in default_params:
|
||||
if params.get(k) is None:
|
||||
params.put(k, v)
|
||||
|
||||
# Create folders needed for msgq
|
||||
try:
|
||||
os.mkdir("/dev/shm")
|
||||
except FileExistsError:
|
||||
pass
|
||||
except PermissionError:
|
||||
print("WARNING: failed to make /dev/shm")
|
||||
|
||||
# set version params
|
||||
params.put("Version", build_metadata.openpilot.version)
|
||||
params.put("TermsVersion", terms_version)
|
||||
params.put("TrainingVersion", training_version)
|
||||
params.put("GitCommit", build_metadata.openpilot.git_commit)
|
||||
params.put("GitCommitDate", build_metadata.openpilot.git_commit_date)
|
||||
params.put("GitBranch", build_metadata.channel)
|
||||
params.put("GitRemote", build_metadata.openpilot.git_origin)
|
||||
params.put_bool("IsTestedBranch", build_metadata.tested_channel)
|
||||
params.put_bool("IsReleaseBranch", build_metadata.release_channel)
|
||||
|
||||
# set dongle id
|
||||
reg_res = register(show_spinner=True)
|
||||
if reg_res:
|
||||
dongle_id = reg_res
|
||||
else:
|
||||
serial = params.get("HardwareSerial")
|
||||
raise Exception(f"Registration failed for device {serial}")
|
||||
os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog
|
||||
os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
|
||||
os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
|
||||
os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog
|
||||
|
||||
if not build_metadata.openpilot.is_dirty:
|
||||
os.environ['CLEAN'] = '1'
|
||||
|
||||
# init logging
|
||||
sentry.init(sentry.SentryProject.SELFDRIVE)
|
||||
cloudlog.bind_global(dongle_id=dongle_id,
|
||||
version=build_metadata.openpilot.version,
|
||||
origin=build_metadata.openpilot.git_normalized_origin,
|
||||
branch=build_metadata.channel,
|
||||
commit=build_metadata.openpilot.git_commit,
|
||||
dirty=build_metadata.openpilot.is_dirty,
|
||||
device=HARDWARE.get_device_type())
|
||||
|
||||
# preimport all processes
|
||||
for p in managed_processes.values():
|
||||
p.prepare()
|
||||
|
||||
|
||||
def manager_cleanup() -> None:
|
||||
# send signals to kill all procs
|
||||
for p in managed_processes.values():
|
||||
p.stop(block=False)
|
||||
|
||||
# ensure all are killed
|
||||
for p in managed_processes.values():
|
||||
p.stop(block=True)
|
||||
|
||||
cloudlog.info("everything is dead")
|
||||
|
||||
|
||||
def manager_thread() -> None:
|
||||
cloudlog.bind(daemon="manager")
|
||||
cloudlog.info("manager start")
|
||||
cloudlog.info({"environ": os.environ})
|
||||
|
||||
params = Params()
|
||||
|
||||
ignore: list[str] = []
|
||||
if params.get("DongleId", encoding='utf8') in (None, UNREGISTERED_DONGLE_ID):
|
||||
ignore += ["manage_athenad", "uploader"]
|
||||
if os.getenv("NOBOARD") is not None:
|
||||
ignore.append("pandad")
|
||||
ignore += [x for x in os.getenv("BLOCK", "").split(",") if len(x) > 0]
|
||||
|
||||
sm = messaging.SubMaster(['deviceState', 'carParams'], poll='deviceState')
|
||||
pm = messaging.PubMaster(['managerState'])
|
||||
|
||||
write_onroad_params(False, params)
|
||||
ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore)
|
||||
|
||||
started_prev = False
|
||||
|
||||
while True:
|
||||
sm.update(1000)
|
||||
|
||||
started = sm['deviceState'].started
|
||||
|
||||
if started and not started_prev:
|
||||
params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
|
||||
elif not started and started_prev:
|
||||
params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
|
||||
|
||||
# update onroad params, which drives pandad's safety setter thread
|
||||
if started != started_prev:
|
||||
write_onroad_params(started, params)
|
||||
|
||||
started_prev = started
|
||||
|
||||
ensure_running(managed_processes.values(), started, params=params, CP=sm['carParams'], not_run=ignore)
|
||||
|
||||
running = ' '.join("{}{}\u001b[0m".format("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
|
||||
for p in managed_processes.values() if p.proc)
|
||||
print(running)
|
||||
cloudlog.debug(running)
|
||||
|
||||
# send managerState
|
||||
msg = messaging.new_message('managerState', valid=True)
|
||||
msg.managerState.processes = [p.get_process_state_msg() for p in managed_processes.values()]
|
||||
pm.send('managerState', msg)
|
||||
|
||||
# Exit main loop when uninstall/shutdown/reboot is needed
|
||||
shutdown = False
|
||||
for param in ("DoUninstall", "DoShutdown", "DoReboot"):
|
||||
if params.get_bool(param):
|
||||
shutdown = True
|
||||
params.put("LastManagerExitReason", f"{param} {datetime.datetime.now()}")
|
||||
cloudlog.warning(f"Shutting down manager - {param} set")
|
||||
|
||||
if shutdown:
|
||||
break
|
||||
|
||||
|
||||
def main() -> None:
|
||||
manager_init()
|
||||
if os.getenv("PREPAREONLY") is not None:
|
||||
return
|
||||
|
||||
# SystemExit on sigterm
|
||||
signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))
|
||||
|
||||
try:
|
||||
manager_thread()
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
sentry.capture_exception()
|
||||
finally:
|
||||
manager_cleanup()
|
||||
|
||||
params = Params()
|
||||
if params.get_bool("DoUninstall"):
|
||||
cloudlog.warning("uninstalling")
|
||||
HARDWARE.uninstall()
|
||||
elif params.get_bool("DoReboot"):
|
||||
cloudlog.warning("reboot")
|
||||
HARDWARE.reboot()
|
||||
elif params.get_bool("DoShutdown"):
|
||||
cloudlog.warning("shutdown")
|
||||
HARDWARE.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unblock_stdout()
|
||||
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("got CTRL-C, exiting")
|
||||
except Exception:
|
||||
add_file_handler(cloudlog)
|
||||
cloudlog.exception("Manager failed to start")
|
||||
|
||||
try:
|
||||
managed_processes['ui'].stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Show last 3 lines of traceback
|
||||
error = traceback.format_exc(-3)
|
||||
error = "Manager failed to start\n\n" + error
|
||||
with TextWindow(error) as t:
|
||||
t.wait_for_exit()
|
||||
|
||||
raise
|
||||
|
||||
# manual exit because we are forked
|
||||
sys.exit(0)
|
||||
@@ -0,0 +1,293 @@
|
||||
import importlib
|
||||
import os
|
||||
import signal
|
||||
import struct
|
||||
import time
|
||||
import subprocess
|
||||
from collections.abc import Callable, ValuesView
|
||||
from abc import ABC, abstractmethod
|
||||
from multiprocessing import Process
|
||||
|
||||
from openpilot.common.threadname import setthreadname
|
||||
|
||||
from cereal import car, log
|
||||
import cereal.messaging as messaging
|
||||
import openpilot.system.sentry as sentry
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
|
||||
WATCHDOG_FN = "/dev/shm/wd_"
|
||||
ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None
|
||||
|
||||
|
||||
def launcher(proc: str, name: str) -> None:
|
||||
try:
|
||||
# import the process
|
||||
mod = importlib.import_module(proc)
|
||||
|
||||
# rename the process
|
||||
setthreadname(proc)
|
||||
|
||||
# create new context since we forked
|
||||
messaging.context = messaging.Context()
|
||||
|
||||
# add daemon name tag to logs
|
||||
cloudlog.bind(daemon=name)
|
||||
sentry.set_tag("daemon", name)
|
||||
|
||||
# exec the process
|
||||
mod.main()
|
||||
except KeyboardInterrupt:
|
||||
cloudlog.warning(f"child {proc} got SIGINT")
|
||||
except Exception:
|
||||
# can't install the crash handler because sys.excepthook doesn't play nice
|
||||
# with threads, so catch it here.
|
||||
sentry.capture_exception()
|
||||
raise
|
||||
|
||||
|
||||
def nativelauncher(pargs: list[str], cwd: str, name: str) -> None:
|
||||
os.environ['MANAGER_DAEMON'] = name
|
||||
|
||||
# exec the process
|
||||
os.chdir(cwd)
|
||||
os.execvp(pargs[0], pargs)
|
||||
|
||||
|
||||
def join_process(process: Process, timeout: float) -> None:
|
||||
# Process().join(timeout) will hang due to a python 3 bug: https://bugs.python.org/issue28382
|
||||
# We have to poll the exitcode instead
|
||||
t = time.monotonic()
|
||||
while time.monotonic() - t < timeout and process.exitcode is None:
|
||||
time.sleep(0.001)
|
||||
|
||||
|
||||
class ManagerProcess(ABC):
|
||||
daemon = False
|
||||
sigkill = False
|
||||
should_run: Callable[[bool, Params, car.CarParams], bool]
|
||||
proc: Process | None = None
|
||||
enabled = True
|
||||
name = ""
|
||||
|
||||
last_watchdog_time = 0
|
||||
watchdog_max_dt: int | None = None
|
||||
watchdog_seen = False
|
||||
shutting_down = False
|
||||
|
||||
@abstractmethod
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def start(self) -> None:
|
||||
pass
|
||||
|
||||
def restart(self) -> None:
|
||||
self.stop(sig=signal.SIGKILL)
|
||||
self.start()
|
||||
|
||||
def check_watchdog(self, started: bool) -> None:
|
||||
if self.watchdog_max_dt is None or self.proc is None:
|
||||
return
|
||||
|
||||
try:
|
||||
fn = WATCHDOG_FN + str(self.proc.pid)
|
||||
with open(fn, "rb") as f:
|
||||
# TODO: why can't pylint find struct.unpack?
|
||||
self.last_watchdog_time = struct.unpack('Q', f.read())[0]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
dt = time.monotonic() - self.last_watchdog_time / 1e9
|
||||
|
||||
if dt > self.watchdog_max_dt:
|
||||
if self.watchdog_seen and ENABLE_WATCHDOG:
|
||||
cloudlog.error(f"Watchdog timeout for {self.name} (exitcode {self.proc.exitcode}) restarting ({started=})")
|
||||
self.restart()
|
||||
else:
|
||||
self.watchdog_seen = True
|
||||
|
||||
def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals = None) -> int | None:
|
||||
if self.proc is None:
|
||||
return None
|
||||
|
||||
if self.proc.exitcode is None:
|
||||
if not self.shutting_down:
|
||||
cloudlog.info(f"killing {self.name}")
|
||||
if sig is None:
|
||||
sig = signal.SIGKILL if self.sigkill else signal.SIGINT
|
||||
self.signal(sig)
|
||||
self.shutting_down = True
|
||||
|
||||
if not block:
|
||||
return None
|
||||
|
||||
join_process(self.proc, 5)
|
||||
|
||||
# If process failed to die send SIGKILL
|
||||
if self.proc.exitcode is None and retry:
|
||||
cloudlog.info(f"killing {self.name} with SIGKILL")
|
||||
self.signal(signal.SIGKILL)
|
||||
self.proc.join()
|
||||
|
||||
ret = self.proc.exitcode
|
||||
cloudlog.info(f"{self.name} is dead with {ret}")
|
||||
|
||||
if self.proc.exitcode is not None:
|
||||
self.shutting_down = False
|
||||
self.proc = None
|
||||
|
||||
return ret
|
||||
|
||||
def signal(self, sig: int) -> None:
|
||||
if self.proc is None:
|
||||
return
|
||||
|
||||
# Don't signal if already exited
|
||||
if self.proc.exitcode is not None and self.proc.pid is not None:
|
||||
return
|
||||
|
||||
# Can't signal if we don't have a pid
|
||||
if self.proc.pid is None:
|
||||
return
|
||||
|
||||
cloudlog.info(f"sending signal {sig} to {self.name}")
|
||||
os.kill(self.proc.pid, sig)
|
||||
|
||||
def get_process_state_msg(self):
|
||||
state = log.ManagerState.ProcessState.new_message()
|
||||
state.name = self.name
|
||||
if self.proc:
|
||||
state.running = self.proc.is_alive()
|
||||
state.shouldBeRunning = self.proc is not None and not self.shutting_down
|
||||
state.pid = self.proc.pid or 0
|
||||
state.exitCode = self.proc.exitcode or 0
|
||||
return state
|
||||
|
||||
|
||||
class NativeProcess(ManagerProcess):
|
||||
def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
|
||||
self.name = name
|
||||
self.cwd = cwd
|
||||
self.cmdline = cmdline
|
||||
self.should_run = should_run
|
||||
self.enabled = enabled
|
||||
self.sigkill = sigkill
|
||||
self.watchdog_max_dt = watchdog_max_dt
|
||||
self.launcher = nativelauncher
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def start(self) -> None:
|
||||
# In case we only tried a non blocking stop we need to stop it before restarting
|
||||
if self.shutting_down:
|
||||
self.stop()
|
||||
|
||||
if self.proc is not None:
|
||||
return
|
||||
|
||||
cwd = os.path.join(BASEDIR, self.cwd)
|
||||
cloudlog.info(f"starting process {self.name}")
|
||||
self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name))
|
||||
self.proc.start()
|
||||
self.watchdog_seen = False
|
||||
self.shutting_down = False
|
||||
|
||||
|
||||
class PythonProcess(ManagerProcess):
|
||||
def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None):
|
||||
self.name = name
|
||||
self.module = module
|
||||
self.should_run = should_run
|
||||
self.enabled = enabled
|
||||
self.sigkill = sigkill
|
||||
self.watchdog_max_dt = watchdog_max_dt
|
||||
self.launcher = launcher
|
||||
|
||||
def prepare(self) -> None:
|
||||
if self.enabled:
|
||||
cloudlog.info(f"preimporting {self.module}")
|
||||
importlib.import_module(self.module)
|
||||
|
||||
def start(self) -> None:
|
||||
# In case we only tried a non blocking stop we need to stop it before restarting
|
||||
if self.shutting_down:
|
||||
self.stop()
|
||||
|
||||
if self.proc is not None:
|
||||
return
|
||||
|
||||
cloudlog.info(f"starting python {self.module}")
|
||||
self.proc = Process(name=self.name, target=self.launcher, args=(self.module, self.name))
|
||||
self.proc.start()
|
||||
self.watchdog_seen = False
|
||||
self.shutting_down = False
|
||||
|
||||
|
||||
class DaemonProcess(ManagerProcess):
|
||||
"""Python process that has to stay running across manager restart.
|
||||
This is used for athena so you don't lose SSH access when restarting manager."""
|
||||
def __init__(self, name, module, param_name, enabled=True):
|
||||
self.name = name
|
||||
self.module = module
|
||||
self.param_name = param_name
|
||||
self.enabled = enabled
|
||||
self.params = None
|
||||
|
||||
@staticmethod
|
||||
def should_run(started, params, CP):
|
||||
return True
|
||||
|
||||
def prepare(self) -> None:
|
||||
pass
|
||||
|
||||
def start(self) -> None:
|
||||
if self.params is None:
|
||||
self.params = Params()
|
||||
|
||||
pid = self.params.get(self.param_name, encoding='utf-8')
|
||||
if pid is not None:
|
||||
try:
|
||||
os.kill(int(pid), 0)
|
||||
with open(f'/proc/{pid}/cmdline') as f:
|
||||
if self.module in f.read():
|
||||
# daemon is running
|
||||
return
|
||||
except (OSError, FileNotFoundError):
|
||||
# process is dead
|
||||
pass
|
||||
|
||||
cloudlog.info(f"starting daemon {self.name}")
|
||||
proc = subprocess.Popen(['python', '-m', self.module],
|
||||
stdin=open('/dev/null'),
|
||||
stdout=open('/dev/null', 'w'),
|
||||
stderr=open('/dev/null', 'w'),
|
||||
preexec_fn=os.setpgrp)
|
||||
|
||||
self.params.put(self.param_name, str(proc.pid))
|
||||
|
||||
def stop(self, retry=True, block=True, sig=None) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None, CP: car.CarParams=None,
|
||||
not_run: list[str] | None=None) -> list[ManagerProcess]:
|
||||
if not_run is None:
|
||||
not_run = []
|
||||
|
||||
running = []
|
||||
for p in procs:
|
||||
if p.enabled and p.name not in not_run and p.should_run(started, params, CP):
|
||||
running.append(p)
|
||||
else:
|
||||
p.stop(block=False)
|
||||
|
||||
p.check_watchdog(started)
|
||||
|
||||
for p in running:
|
||||
p.start()
|
||||
|
||||
return running
|
||||
@@ -0,0 +1,91 @@
|
||||
import os
|
||||
|
||||
from cereal import car
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.hardware import PC, TICI
|
||||
from openpilot.system.manager.process import PythonProcess, NativeProcess, DaemonProcess
|
||||
|
||||
WEBCAM = os.getenv("USE_WEBCAM") is not None
|
||||
|
||||
def driverview(started: bool, params: Params, CP: car.CarParams) -> bool:
|
||||
return started or params.get_bool("IsDriverViewEnabled")
|
||||
|
||||
def notcar(started: bool, params: Params, CP: car.CarParams) -> bool:
|
||||
return started and CP.notCar
|
||||
|
||||
def iscar(started: bool, params: Params, CP: car.CarParams) -> bool:
|
||||
return started and not CP.notCar
|
||||
|
||||
def logging(started, params, CP: car.CarParams) -> bool:
|
||||
run = (not CP.notCar) or not params.get_bool("DisableLogging")
|
||||
return started and run
|
||||
|
||||
def ublox_available() -> bool:
|
||||
return os.path.exists('/dev/ttyHS0') and not os.path.exists('/persist/comma/use-quectel-gps')
|
||||
|
||||
def ublox(started, params, CP: car.CarParams) -> bool:
|
||||
use_ublox = ublox_available()
|
||||
if use_ublox != params.get_bool("UbloxAvailable"):
|
||||
params.put_bool("UbloxAvailable", use_ublox)
|
||||
return started and use_ublox
|
||||
|
||||
def qcomgps(started, params, CP: car.CarParams) -> bool:
|
||||
return started and not ublox_available()
|
||||
|
||||
def always_run(started, params, CP: car.CarParams) -> bool:
|
||||
return True
|
||||
|
||||
def only_onroad(started: bool, params, CP: car.CarParams) -> bool:
|
||||
return started
|
||||
|
||||
def only_offroad(started, params, CP: car.CarParams) -> bool:
|
||||
return not started
|
||||
|
||||
procs = [
|
||||
DaemonProcess("manage_athenad", "system.athena.manage_athenad", "AthenadPid"),
|
||||
|
||||
NativeProcess("camerad", "system/camerad", ["./camerad"], driverview),
|
||||
NativeProcess("logcatd", "system/logcatd", ["./logcatd"], only_onroad),
|
||||
NativeProcess("proclogd", "system/proclogd", ["./proclogd"], only_onroad),
|
||||
PythonProcess("logmessaged", "system.logmessaged", always_run),
|
||||
PythonProcess("micd", "system.micd", iscar),
|
||||
PythonProcess("timed", "system.timed", always_run, enabled=not PC),
|
||||
|
||||
PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)),
|
||||
NativeProcess("encoderd", "system/loggerd", ["./encoderd"], only_onroad),
|
||||
NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], notcar),
|
||||
NativeProcess("loggerd", "system/loggerd", ["./loggerd"], logging),
|
||||
NativeProcess("modeld", "selfdrive/modeld", ["./modeld"], only_onroad),
|
||||
NativeProcess("sensord", "system/sensord", ["./sensord"], only_onroad, enabled=not PC),
|
||||
NativeProcess("ui", "selfdrive/ui", ["./ui"], always_run, watchdog_max_dt=(5 if not PC else None)),
|
||||
PythonProcess("soundd", "selfdrive.ui.soundd", only_onroad),
|
||||
NativeProcess("locationd", "selfdrive/locationd", ["./locationd"], only_onroad),
|
||||
NativeProcess("pandad", "selfdrive/pandad", ["./pandad"], always_run, enabled=False),
|
||||
PythonProcess("calibrationd", "selfdrive.locationd.calibrationd", only_onroad),
|
||||
PythonProcess("torqued", "selfdrive.locationd.torqued", only_onroad),
|
||||
PythonProcess("controlsd", "selfdrive.controls.controlsd", only_onroad),
|
||||
PythonProcess("card", "selfdrive.car.card", only_onroad),
|
||||
PythonProcess("deleter", "system.loggerd.deleter", always_run),
|
||||
PythonProcess("dmonitoringd", "selfdrive.monitoring.dmonitoringd", driverview, enabled=(not PC or WEBCAM)),
|
||||
PythonProcess("qcomgpsd", "system.qcomgpsd.qcomgpsd", qcomgps, enabled=TICI),
|
||||
#PythonProcess("ugpsd", "system.ugpsd", only_onroad, enabled=TICI),
|
||||
PythonProcess("navd", "selfdrive.navd.navd", only_onroad),
|
||||
PythonProcess("pandad", "selfdrive.pandad.pandad", always_run),
|
||||
PythonProcess("paramsd", "selfdrive.locationd.paramsd", only_onroad),
|
||||
NativeProcess("ubloxd", "system/ubloxd", ["./ubloxd"], ublox, enabled=TICI),
|
||||
PythonProcess("pigeond", "system.ubloxd.pigeond", ublox, enabled=TICI),
|
||||
PythonProcess("plannerd", "selfdrive.controls.plannerd", only_onroad),
|
||||
PythonProcess("radard", "selfdrive.controls.radard", only_onroad),
|
||||
PythonProcess("hardwared", "system.hardware.hardwared", always_run),
|
||||
PythonProcess("tombstoned", "system.tombstoned", always_run, enabled=not PC),
|
||||
PythonProcess("updated", "system.updated.updated", only_offroad, enabled=not PC),
|
||||
PythonProcess("uploader", "system.loggerd.uploader", always_run),
|
||||
PythonProcess("statsd", "system.statsd", always_run),
|
||||
|
||||
# debug procs
|
||||
NativeProcess("bridge", "cereal/messaging", ["./bridge"], notcar),
|
||||
PythonProcess("webrtcd", "system.webrtc.webrtcd", notcar),
|
||||
PythonProcess("webjoystick", "tools.bodyteleop.web", notcar),
|
||||
]
|
||||
|
||||
managed_processes = {p.name: p for p in procs}
|
||||
@@ -0,0 +1,76 @@
|
||||
import os
|
||||
import pytest
|
||||
import signal
|
||||
import time
|
||||
|
||||
from parameterized import parameterized
|
||||
|
||||
from cereal import car
|
||||
from openpilot.common.params import Params
|
||||
import openpilot.system.manager.manager as manager
|
||||
from openpilot.system.manager.process import ensure_running
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.system.hardware import HARDWARE
|
||||
|
||||
os.environ['FAKEUPLOAD'] = "1"
|
||||
|
||||
MAX_STARTUP_TIME = 3
|
||||
BLACKLIST_PROCS = ['manage_athenad', 'pandad', 'pigeond']
|
||||
|
||||
|
||||
@pytest.mark.tici
|
||||
class TestManager:
|
||||
def setup_method(self):
|
||||
HARDWARE.set_power_save(False)
|
||||
|
||||
# ensure clean CarParams
|
||||
params = Params()
|
||||
params.clear_all()
|
||||
|
||||
def teardown_method(self):
|
||||
manager.manager_cleanup()
|
||||
|
||||
def test_manager_prepare(self):
|
||||
os.environ['PREPAREONLY'] = '1'
|
||||
manager.main()
|
||||
|
||||
def test_blacklisted_procs(self):
|
||||
# TODO: ensure there are blacklisted procs until we have a dedicated test
|
||||
assert len(BLACKLIST_PROCS), "No blacklisted procs to test not_run"
|
||||
|
||||
@parameterized.expand([(i,) for i in range(10)])
|
||||
def test_startup_time(self, index):
|
||||
start = time.monotonic()
|
||||
os.environ['PREPAREONLY'] = '1'
|
||||
manager.main()
|
||||
t = time.monotonic() - start
|
||||
assert t < MAX_STARTUP_TIME, f"startup took {t}s, expected <{MAX_STARTUP_TIME}s"
|
||||
|
||||
@pytest.mark.skip("this test is flaky the way it's currently written, should be moved to test_onroad")
|
||||
def test_clean_exit(self, subtests):
|
||||
"""
|
||||
Ensure all processes exit cleanly when stopped.
|
||||
"""
|
||||
HARDWARE.set_power_save(False)
|
||||
manager.manager_init()
|
||||
|
||||
CP = car.CarParams.new_message()
|
||||
procs = ensure_running(managed_processes.values(), True, Params(), CP, not_run=BLACKLIST_PROCS)
|
||||
|
||||
time.sleep(10)
|
||||
|
||||
for p in procs:
|
||||
with subtests.test(proc=p.name):
|
||||
state = p.get_process_state_msg()
|
||||
assert state.running, f"{p.name} not running"
|
||||
exit_code = p.stop(retry=False)
|
||||
|
||||
assert p.name not in BLACKLIST_PROCS, f"{p.name} was started"
|
||||
|
||||
assert exit_code is not None, f"{p.name} failed to exit"
|
||||
|
||||
# TODO: interrupted blocking read exits with 1 in cereal. use a more unique return code
|
||||
exit_codes = [0, 1]
|
||||
if p.sigkill:
|
||||
exit_codes = [-signal.SIGKILL]
|
||||
assert exit_code in exit_codes, f"{p.name} died with {exit_code}"
|
||||
@@ -1,5 +1,5 @@
|
||||
Import('env', 'cereal', 'messaging', 'common')
|
||||
libs = [cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj', 'common', 'zmq', 'json11']
|
||||
Import('env', 'messaging', 'common')
|
||||
libs = [messaging, 'pthread', 'common', 'zmq', 'json11']
|
||||
env.Program('proclogd', ['main.cc', 'proclog.cc'], LIBS=libs)
|
||||
|
||||
if GetOption('extras'):
|
||||
|
||||
@@ -127,7 +127,7 @@ def main() -> NoReturn:
|
||||
print("qcomgpsd is running, please kill openpilot before running this script! (aborted)")
|
||||
sys.exit(1)
|
||||
except CalledProcessError as e:
|
||||
if e.returncode != 1: # 1 == no process found (boardd not running)
|
||||
if e.returncode != 1: # 1 == no process found (pandad not running)
|
||||
raise e
|
||||
|
||||
print("power up antenna ...")
|
||||
|
||||
@@ -173,7 +173,7 @@ def setup_quectel(diag: ModemDiag) -> bool:
|
||||
os.remove(ASSIST_DATA_FILE)
|
||||
#at_cmd("AT+QGPSXTRADATA?")
|
||||
if system_time_valid():
|
||||
time_str = datetime.datetime.utcnow().strftime("%Y/%m/%d,%H:%M:%S")
|
||||
time_str = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).strftime("%Y/%m/%d,%H:%M:%S")
|
||||
at_cmd(f"AT+QGPSXTRATIME=0,\"{time_str}\",1,1,1000")
|
||||
|
||||
at_cmd("AT+QGPSCFG=\"outport\",\"usbnmea\"")
|
||||
|
||||
Executable → Regular
+11
-16
@@ -1,38 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import pytest
|
||||
import json
|
||||
import time
|
||||
import datetime
|
||||
import unittest
|
||||
import subprocess
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from openpilot.system.qcomgpsd.qcomgpsd import at_cmd, wait_for_modem
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
|
||||
GOOD_SIGNAL = bool(int(os.getenv("GOOD_SIGNAL", '0')))
|
||||
|
||||
|
||||
@pytest.mark.tici
|
||||
class TestRawgpsd(unittest.TestCase):
|
||||
class TestRawgpsd:
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
def setup_class(cls):
|
||||
os.system("sudo systemctl start systemd-resolved")
|
||||
os.system("sudo systemctl restart ModemManager lte")
|
||||
wait_for_modem()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
def teardown_class(cls):
|
||||
managed_processes['qcomgpsd'].stop()
|
||||
os.system("sudo systemctl restart systemd-resolved")
|
||||
os.system("sudo systemctl restart ModemManager lte")
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
at_cmd("AT+QGPSDEL=0")
|
||||
self.sm = messaging.SubMaster(['qcomGnss', 'gpsLocation', 'gnssMeasurements'])
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
managed_processes['qcomgpsd'].stop()
|
||||
os.system("sudo systemctl restart systemd-resolved")
|
||||
|
||||
@@ -57,18 +55,18 @@ class TestRawgpsd(unittest.TestCase):
|
||||
os.system("sudo systemctl restart ModemManager")
|
||||
assert self._wait_for_output(30)
|
||||
|
||||
def test_startup_time(self):
|
||||
def test_startup_time(self, subtests):
|
||||
for internet in (True, False):
|
||||
if not internet:
|
||||
os.system("sudo systemctl stop systemd-resolved")
|
||||
with self.subTest(internet=internet):
|
||||
with subtests.test(internet=internet):
|
||||
managed_processes['qcomgpsd'].start()
|
||||
assert self._wait_for_output(7)
|
||||
managed_processes['qcomgpsd'].stop()
|
||||
|
||||
def test_turns_off_gnss(self):
|
||||
def test_turns_off_gnss(self, subtests):
|
||||
for s in (0.1, 1, 5):
|
||||
with self.subTest(runtime=s):
|
||||
with subtests.test(runtime=s):
|
||||
managed_processes['qcomgpsd'].start()
|
||||
time.sleep(s)
|
||||
managed_processes['qcomgpsd'].stop()
|
||||
@@ -87,7 +85,7 @@ class TestRawgpsd(unittest.TestCase):
|
||||
if should_be_loaded:
|
||||
assert valid_duration == "10080" # should be max time
|
||||
injected_time = datetime.datetime.strptime(injected_time_str.replace("\"", ""), "%Y/%m/%d,%H:%M:%S")
|
||||
self.assertLess(abs((datetime.datetime.utcnow() - injected_time).total_seconds()), 60*60*12)
|
||||
assert abs((datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - injected_time).total_seconds()) < 60*60*12
|
||||
else:
|
||||
valid_duration, injected_time_str = out.split(",", 1)
|
||||
injected_time_str = injected_time_str.replace('\"', '').replace('\'', '')
|
||||
@@ -119,6 +117,3 @@ class TestRawgpsd(unittest.TestCase):
|
||||
time.sleep(15)
|
||||
managed_processes['qcomgpsd'].stop()
|
||||
self.check_assistance(True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(failfast=True)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
Import('env', 'arch', 'common', 'cereal', 'messaging')
|
||||
Import('env', 'arch', 'common', 'messaging')
|
||||
|
||||
sensors = [
|
||||
'sensors/i2c_sensor.cc',
|
||||
@@ -11,7 +11,7 @@ sensors = [
|
||||
'sensors/lsm6ds3_temp.cc',
|
||||
'sensors/mmc5603nj_magn.cc',
|
||||
]
|
||||
libs = [common, cereal, messaging, 'capnp', 'zmq', 'kj', 'pthread']
|
||||
libs = [common, messaging, 'pthread']
|
||||
if arch == "larch64":
|
||||
libs.append('i2c')
|
||||
env.Program('sensord', ['sensors_qcom2.cc'] + sensors, LIBS=libs)
|
||||
|
||||
Executable → Regular
+10
-15
@@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import pytest
|
||||
import time
|
||||
import unittest
|
||||
import numpy as np
|
||||
from collections import namedtuple, defaultdict
|
||||
|
||||
@@ -11,7 +9,7 @@ from cereal import log
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.gpio import get_irqs_for_action
|
||||
from openpilot.common.timeout import Timeout
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
|
||||
BMX = {
|
||||
('bmx055', 'acceleration'),
|
||||
@@ -99,9 +97,9 @@ def read_sensor_events(duration_sec):
|
||||
return {k: v for k, v in events.items() if len(v) > 0}
|
||||
|
||||
@pytest.mark.tici
|
||||
class TestSensord(unittest.TestCase):
|
||||
class TestSensord:
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
def setup_class(cls):
|
||||
# enable LSM self test
|
||||
os.environ["LSM_SELF_TEST"] = "1"
|
||||
|
||||
@@ -119,10 +117,10 @@ class TestSensord(unittest.TestCase):
|
||||
managed_processes["sensord"].stop()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
def teardown_class(cls):
|
||||
managed_processes["sensord"].stop()
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
managed_processes["sensord"].stop()
|
||||
|
||||
def test_sensors_present(self):
|
||||
@@ -133,9 +131,9 @@ class TestSensord(unittest.TestCase):
|
||||
m = getattr(measurement, measurement.which())
|
||||
seen.add((str(m.source), m.which()))
|
||||
|
||||
self.assertIn(seen, SENSOR_CONFIGURATIONS)
|
||||
assert seen in SENSOR_CONFIGURATIONS
|
||||
|
||||
def test_lsm6ds3_timing(self):
|
||||
def test_lsm6ds3_timing(self, subtests):
|
||||
# verify measurements are sampled and published at 104Hz
|
||||
|
||||
sensor_t = {
|
||||
@@ -152,7 +150,7 @@ class TestSensord(unittest.TestCase):
|
||||
sensor_t[m.sensor].append(m.timestamp)
|
||||
|
||||
for s, vals in sensor_t.items():
|
||||
with self.subTest(sensor=s):
|
||||
with subtests.test(sensor=s):
|
||||
assert len(vals) > 0
|
||||
tdiffs = np.diff(vals) / 1e6 # millis
|
||||
|
||||
@@ -166,9 +164,9 @@ class TestSensord(unittest.TestCase):
|
||||
stddev = np.std(tdiffs)
|
||||
assert stddev < 2.0, f"Standard-dev to big {stddev}"
|
||||
|
||||
def test_sensor_frequency(self):
|
||||
def test_sensor_frequency(self, subtests):
|
||||
for s, msgs in self.events.items():
|
||||
with self.subTest(sensor=s):
|
||||
with subtests.test(sensor=s):
|
||||
freq = len(msgs) / self.sample_secs
|
||||
ef = SERVICE_LIST[s].frequency
|
||||
assert ef*0.85 <= freq <= ef*1.15
|
||||
@@ -246,6 +244,3 @@ class TestSensord(unittest.TestCase):
|
||||
state_two = get_irq_count(self.sensord_irq)
|
||||
assert state_one == state_two, "Interrupts received after sensord stop!"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -4,7 +4,7 @@ import time
|
||||
import atexit
|
||||
|
||||
from cereal import messaging
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
|
||||
TIMEOUT = 10*60
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Install exception handler for process crash."""
|
||||
import sentry_sdk
|
||||
from enum import Enum
|
||||
from sentry_sdk.integrations.threading import ThreadingIntegration
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.athena.registration import is_registered_device
|
||||
from openpilot.system.hardware import HARDWARE, PC
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.version import get_build_metadata, get_version
|
||||
|
||||
|
||||
class SentryProject(Enum):
|
||||
# python project
|
||||
SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
|
||||
# native project
|
||||
SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
|
||||
|
||||
|
||||
def report_tombstone(fn: str, message: str, contents: str) -> None:
|
||||
cloudlog.error({'tombstone': message})
|
||||
|
||||
with sentry_sdk.configure_scope() as scope:
|
||||
scope.set_extra("tombstone_fn", fn)
|
||||
scope.set_extra("tombstone", contents)
|
||||
sentry_sdk.capture_message(message=message)
|
||||
sentry_sdk.flush()
|
||||
|
||||
|
||||
def capture_exception(*args, **kwargs) -> None:
|
||||
cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
|
||||
|
||||
try:
|
||||
sentry_sdk.capture_exception(*args, **kwargs)
|
||||
sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291
|
||||
except Exception:
|
||||
cloudlog.exception("sentry exception")
|
||||
|
||||
|
||||
def set_tag(key: str, value: str) -> None:
|
||||
sentry_sdk.set_tag(key, value)
|
||||
|
||||
|
||||
def init(project: SentryProject) -> bool:
|
||||
build_metadata = get_build_metadata()
|
||||
# forks like to mess with this, so double check
|
||||
comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin
|
||||
if not comma_remote or not is_registered_device() or PC:
|
||||
return False
|
||||
|
||||
env = "release" if build_metadata.tested_channel else "master"
|
||||
dongle_id = Params().get("DongleId", encoding='utf-8')
|
||||
|
||||
integrations = []
|
||||
if project == SentryProject.SELFDRIVE:
|
||||
integrations.append(ThreadingIntegration(propagate_hub=True))
|
||||
|
||||
sentry_sdk.init(project.value,
|
||||
default_integrations=False,
|
||||
release=get_version(),
|
||||
integrations=integrations,
|
||||
traces_sample_rate=1.0,
|
||||
max_value_length=8192,
|
||||
environment=env)
|
||||
|
||||
build_metadata = get_build_metadata()
|
||||
|
||||
sentry_sdk.set_user({"id": dongle_id})
|
||||
sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty)
|
||||
sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin)
|
||||
sentry_sdk.set_tag("branch", build_metadata.channel)
|
||||
sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit)
|
||||
sentry_sdk.set_tag("device", HARDWARE.get_device_type())
|
||||
|
||||
if project == SentryProject.SELFDRIVE:
|
||||
sentry_sdk.Hub.current.start_session()
|
||||
|
||||
return True
|
||||
Executable
+181
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import zmq
|
||||
import time
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, UTC
|
||||
from typing import NoReturn
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from cereal.messaging import SubMaster
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.hardware import HARDWARE
|
||||
from openpilot.common.file_helpers import atomic_write_in_dir
|
||||
from openpilot.system.version import get_build_metadata
|
||||
from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
|
||||
|
||||
|
||||
class METRIC_TYPE:
|
||||
GAUGE = 'g'
|
||||
SAMPLE = 'sa'
|
||||
|
||||
class StatLog:
|
||||
def __init__(self):
|
||||
self.pid = None
|
||||
self.zctx = None
|
||||
self.sock = None
|
||||
|
||||
def connect(self) -> None:
|
||||
self.zctx = zmq.Context()
|
||||
self.sock = self.zctx.socket(zmq.PUSH)
|
||||
self.sock.setsockopt(zmq.LINGER, 10)
|
||||
self.sock.connect(STATS_SOCKET)
|
||||
self.pid = os.getpid()
|
||||
|
||||
def __del__(self):
|
||||
if self.sock is not None:
|
||||
self.sock.close()
|
||||
if self.zctx is not None:
|
||||
self.zctx.term()
|
||||
|
||||
def _send(self, metric: str) -> None:
|
||||
if os.getpid() != self.pid:
|
||||
self.connect()
|
||||
|
||||
try:
|
||||
self.sock.send_string(metric, zmq.NOBLOCK)
|
||||
except zmq.error.Again:
|
||||
# drop :/
|
||||
pass
|
||||
|
||||
def gauge(self, name: str, value: float) -> None:
|
||||
self._send(f"{name}:{value}|{METRIC_TYPE.GAUGE}")
|
||||
|
||||
# Samples will be recorded in a buffer and at aggregation time,
|
||||
# statistical properties will be logged (mean, count, percentiles, ...)
|
||||
def sample(self, name: str, value: float):
|
||||
self._send(f"{name}:{value}|{METRIC_TYPE.SAMPLE}")
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
dongle_id = Params().get("DongleId", encoding='utf-8')
|
||||
def get_influxdb_line(measurement: str, value: float | dict[str, float], timestamp: datetime, tags: dict) -> str:
|
||||
res = f"{measurement}"
|
||||
for k, v in tags.items():
|
||||
res += f",{k}={str(v)}"
|
||||
res += " "
|
||||
|
||||
if isinstance(value, float):
|
||||
value = {'value': value}
|
||||
|
||||
for k, v in value.items():
|
||||
res += f"{k}={v},"
|
||||
|
||||
res += f"dongle_id=\"{dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n"
|
||||
return res
|
||||
|
||||
# open statistics socket
|
||||
ctx = zmq.Context.instance()
|
||||
sock = ctx.socket(zmq.PULL)
|
||||
sock.bind(STATS_SOCKET)
|
||||
|
||||
STATS_DIR = Paths.stats_root()
|
||||
|
||||
# initialize stats directory
|
||||
Path(STATS_DIR).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
build_metadata = get_build_metadata()
|
||||
|
||||
# initialize tags
|
||||
tags = {
|
||||
'started': False,
|
||||
'version': build_metadata.openpilot.version,
|
||||
'branch': build_metadata.channel,
|
||||
'dirty': build_metadata.openpilot.is_dirty,
|
||||
'origin': build_metadata.openpilot.git_normalized_origin,
|
||||
'deviceType': HARDWARE.get_device_type(),
|
||||
}
|
||||
|
||||
# subscribe to deviceState for started state
|
||||
sm = SubMaster(['deviceState'])
|
||||
|
||||
idx = 0
|
||||
last_flush_time = time.monotonic()
|
||||
gauges = {}
|
||||
samples: dict[str, list[float]] = defaultdict(list)
|
||||
try:
|
||||
while True:
|
||||
started_prev = sm['deviceState'].started
|
||||
sm.update()
|
||||
|
||||
# Update metrics
|
||||
while True:
|
||||
try:
|
||||
metric = sock.recv_string(zmq.NOBLOCK)
|
||||
try:
|
||||
metric_type = metric.split('|')[1]
|
||||
metric_name = metric.split(':')[0]
|
||||
metric_value = float(metric.split('|')[0].split(':')[1])
|
||||
|
||||
if metric_type == METRIC_TYPE.GAUGE:
|
||||
gauges[metric_name] = metric_value
|
||||
elif metric_type == METRIC_TYPE.SAMPLE:
|
||||
samples[metric_name].append(metric_value)
|
||||
else:
|
||||
cloudlog.event("unknown metric type", metric_type=metric_type)
|
||||
except Exception:
|
||||
cloudlog.event("malformed metric", metric=metric)
|
||||
except zmq.error.Again:
|
||||
break
|
||||
|
||||
# flush when started state changes or after FLUSH_TIME_S
|
||||
if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
|
||||
result = ""
|
||||
current_time = datetime.now(UTC)
|
||||
tags['started'] = sm['deviceState'].started
|
||||
|
||||
for key, value in gauges.items():
|
||||
result += get_influxdb_line(f"gauge.{key}", value, current_time, tags)
|
||||
|
||||
for key, values in samples.items():
|
||||
values.sort()
|
||||
sample_count = len(values)
|
||||
sample_sum = sum(values)
|
||||
|
||||
stats = {
|
||||
'count': sample_count,
|
||||
'min': values[0],
|
||||
'max': values[-1],
|
||||
'mean': sample_sum / sample_count,
|
||||
}
|
||||
for percentile in [0.05, 0.5, 0.95]:
|
||||
value = values[int(round(percentile * (sample_count - 1)))]
|
||||
stats[f"p{int(percentile * 100)}"] = value
|
||||
|
||||
result += get_influxdb_line(f"sample.{key}", stats, current_time, tags)
|
||||
|
||||
# clear intermediate data
|
||||
gauges.clear()
|
||||
samples.clear()
|
||||
last_flush_time = time.monotonic()
|
||||
|
||||
# check that we aren't filling up the drive
|
||||
if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT:
|
||||
if len(result) > 0:
|
||||
stats_path = os.path.join(STATS_DIR, f"{current_time.timestamp():.0f}_{idx}")
|
||||
with atomic_write_in_dir(stats_path) as f:
|
||||
f.write(result)
|
||||
idx += 1
|
||||
else:
|
||||
cloudlog.error("stats dir full")
|
||||
finally:
|
||||
sock.close()
|
||||
ctx.term()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
else:
|
||||
statlog = StatLog()
|
||||
Executable → Regular
+4
-9
@@ -1,17 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
import glob
|
||||
import os
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.common.swaglog import cloudlog, ipchandler
|
||||
|
||||
|
||||
class TestLogmessaged(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestLogmessaged:
|
||||
def setup_method(self):
|
||||
# clear the IPC buffer in case some other tests used cloudlog and filled it
|
||||
ipchandler.close()
|
||||
ipchandler.connect()
|
||||
@@ -25,7 +23,7 @@ class TestLogmessaged(unittest.TestCase):
|
||||
messaging.drain_sock(self.sock)
|
||||
messaging.drain_sock(self.error_sock)
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
del self.sock
|
||||
del self.error_sock
|
||||
managed_processes['logmessaged'].stop(block=True)
|
||||
@@ -55,6 +53,3 @@ class TestLogmessaged(unittest.TestCase):
|
||||
logsize = sum([os.path.getsize(f) for f in self._get_log_files()])
|
||||
assert (n*len(msg)) < logsize < (n*(len(msg)+1024))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Executable
+176
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import glob
|
||||
from typing import NoReturn
|
||||
|
||||
import openpilot.system.sentry as sentry
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
MAX_SIZE = 1_000_000 * 100 # allow up to 100M
|
||||
MAX_TOMBSTONE_FN_LEN = 62 # 85 - 23 ("<dongle id>/crash/")
|
||||
|
||||
TOMBSTONE_DIR = "/data/tombstones/"
|
||||
APPORT_DIR = "/var/crash/"
|
||||
|
||||
|
||||
def safe_fn(s):
|
||||
extra = ['_']
|
||||
return "".join(c for c in s if c.isalnum() or c in extra).rstrip()
|
||||
|
||||
|
||||
def clear_apport_folder():
|
||||
for f in glob.glob(APPORT_DIR + '*'):
|
||||
try:
|
||||
os.remove(f)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_apport_stacktrace(fn):
|
||||
try:
|
||||
cmd = f'apport-retrace -s <(cat <(echo "Package: openpilot") "{fn}")'
|
||||
return subprocess.check_output(cmd, shell=True, encoding='utf8', timeout=30, executable='/bin/bash')
|
||||
except subprocess.CalledProcessError:
|
||||
return "Error getting stacktrace"
|
||||
except subprocess.TimeoutExpired:
|
||||
return "Timeout getting stacktrace"
|
||||
|
||||
|
||||
def get_tombstones():
|
||||
"""Returns list of (filename, ctime) for all crashlogs"""
|
||||
files = []
|
||||
if os.path.exists(APPORT_DIR):
|
||||
with os.scandir(APPORT_DIR) as d:
|
||||
# Loop over first 1000 directory entries
|
||||
for _, f in zip(range(1000), d, strict=False):
|
||||
if f.name.startswith("tombstone"):
|
||||
files.append((f.path, int(f.stat().st_ctime)))
|
||||
elif f.name.endswith(".crash") and f.stat().st_mode == 0o100640:
|
||||
files.append((f.path, int(f.stat().st_ctime)))
|
||||
return files
|
||||
|
||||
|
||||
def report_tombstone_apport(fn):
|
||||
f_size = os.path.getsize(fn)
|
||||
if f_size > MAX_SIZE:
|
||||
cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...")
|
||||
return
|
||||
|
||||
message = "" # One line description of the crash
|
||||
contents = "" # Full file contents without coredump
|
||||
path = "" # File path relative to openpilot directory
|
||||
|
||||
proc_maps = False
|
||||
|
||||
with open(fn) as f:
|
||||
for line in f:
|
||||
if "CoreDump" in line:
|
||||
break
|
||||
elif "ProcMaps" in line:
|
||||
proc_maps = True
|
||||
elif "ProcStatus" in line:
|
||||
proc_maps = False
|
||||
|
||||
if not proc_maps:
|
||||
contents += line
|
||||
|
||||
if "ExecutablePath" in line:
|
||||
path = line.strip().split(': ')[-1]
|
||||
path = path.replace('/data/openpilot/', '')
|
||||
message += path
|
||||
elif "Signal" in line:
|
||||
message += " - " + line.strip()
|
||||
|
||||
try:
|
||||
sig_num = int(line.strip().split(': ')[-1])
|
||||
message += " (" + signal.Signals(sig_num).name + ")"
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
stacktrace = get_apport_stacktrace(fn)
|
||||
stacktrace_s = stacktrace.split('\n')
|
||||
crash_function = "No stacktrace"
|
||||
|
||||
if len(stacktrace_s) > 2:
|
||||
found = False
|
||||
|
||||
# Try to find first entry in openpilot, fall back to first line
|
||||
for line in stacktrace_s:
|
||||
if "at selfdrive/" in line:
|
||||
crash_function = line
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
crash_function = stacktrace_s[1]
|
||||
|
||||
# Remove arguments that can contain pointers to make sentry one-liner unique
|
||||
crash_function = " ".join(x for x in crash_function.split(' ')[1:] if not x.startswith('0x'))
|
||||
crash_function = re.sub(r'\(.*?\)', '', crash_function)
|
||||
|
||||
contents = stacktrace + "\n\n" + contents
|
||||
message = message + " - " + crash_function
|
||||
sentry.report_tombstone(fn, message, contents)
|
||||
|
||||
# Copy crashlog to upload folder
|
||||
clean_path = path.replace('/', '_')
|
||||
date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S")
|
||||
|
||||
build_metadata = get_build_metadata()
|
||||
|
||||
new_fn = f"{date}_{(build_metadata.openpilot.git_commit or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN]
|
||||
|
||||
crashlog_dir = os.path.join(Paths.log_root(), "crash")
|
||||
os.makedirs(crashlog_dir, exist_ok=True)
|
||||
|
||||
# Files could be on different filesystems, copy, then delete
|
||||
shutil.copy(fn, os.path.join(crashlog_dir, new_fn))
|
||||
|
||||
try:
|
||||
os.remove(fn)
|
||||
except PermissionError:
|
||||
pass
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
should_report = sentry.init(sentry.SentryProject.SELFDRIVE_NATIVE)
|
||||
|
||||
# Clear apport folder on start, otherwise duplicate crashes won't register
|
||||
clear_apport_folder()
|
||||
initial_tombstones = set(get_tombstones())
|
||||
|
||||
while True:
|
||||
now_tombstones = set(get_tombstones())
|
||||
|
||||
for fn, _ in (now_tombstones - initial_tombstones):
|
||||
# clear logs if we're not interested in them
|
||||
if not should_report:
|
||||
try:
|
||||
os.remove(fn)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
try:
|
||||
cloudlog.info(f"reporting new tombstone {fn}")
|
||||
if fn.endswith(".crash"):
|
||||
report_tombstone_apport(fn)
|
||||
else:
|
||||
cloudlog.error(f"unknown crash type: {fn}")
|
||||
except Exception:
|
||||
cloudlog.exception(f"Error reporting tombstone {fn}")
|
||||
|
||||
initial_tombstones = now_tombstones
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,6 @@
|
||||
Import('env', 'common', 'cereal', 'messaging')
|
||||
Import('env', 'common', 'messaging')
|
||||
|
||||
loc_libs = [cereal, messaging, 'zmq', common, 'capnp', 'kj', 'kaitai', 'pthread']
|
||||
loc_libs = [messaging, common, 'kaitai', 'pthread']
|
||||
|
||||
if GetOption('kaitai'):
|
||||
generated = Dir('generated').srcnode().abspath
|
||||
|
||||
@@ -6,7 +6,7 @@ import serial
|
||||
import struct
|
||||
import requests
|
||||
import urllib.parse
|
||||
from datetime import datetime
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from cereal import messaging
|
||||
from openpilot.common.params import Params
|
||||
@@ -61,7 +61,7 @@ def get_assistnow_messages(token: bytes) -> list[bytes]:
|
||||
return msgs
|
||||
|
||||
|
||||
class TTYPigeon():
|
||||
class TTYPigeon:
|
||||
def __init__(self):
|
||||
self.tty = serial.VTIMESerial(UBLOX_TTY, baudrate=9600, timeout=0)
|
||||
|
||||
@@ -196,7 +196,7 @@ def initialize_pigeon(pigeon: TTYPigeon) -> bool:
|
||||
cloudlog.error(f"failed to restore almanac backup, status: {restore_status}")
|
||||
|
||||
# sending time to ublox
|
||||
t_now = datetime.utcnow()
|
||||
t_now = datetime.now(UTC).replace(tzinfo=None)
|
||||
if t_now >= datetime(2021, 6, 1):
|
||||
cloudlog.warning("Sending current time to ublox")
|
||||
|
||||
|
||||
Executable → Regular
+3
-9
@@ -1,21 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
import pytest
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from cereal.services import SERVICE_LIST
|
||||
from openpilot.common.gpio import gpio_read
|
||||
from openpilot.selfdrive.test.helpers import with_processes
|
||||
from openpilot.selfdrive.manager.process_config import managed_processes
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
from openpilot.system.hardware.tici.pins import GPIO
|
||||
|
||||
|
||||
# TODO: test TTFF when we have good A-GNSS
|
||||
@pytest.mark.tici
|
||||
class TestPigeond(unittest.TestCase):
|
||||
class TestPigeond:
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
managed_processes['pigeond'].stop()
|
||||
|
||||
@with_processes(['pigeond'])
|
||||
@@ -54,7 +52,3 @@ class TestPigeond(unittest.TestCase):
|
||||
|
||||
assert gpio_read(GPIO.UBLOX_RST_N) == 0
|
||||
assert gpio_read(GPIO.GNSS_PWR_EN) == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import pathlib
|
||||
import tarfile
|
||||
from typing import IO, Callable
|
||||
from typing import IO
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
def include_default(_) -> bool:
|
||||
|
||||
Executable → Regular
+43
-46
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import pytest
|
||||
import os
|
||||
import pathlib
|
||||
import unittest
|
||||
import tempfile
|
||||
import subprocess
|
||||
|
||||
@@ -14,9 +13,10 @@ from openpilot.system.updated.casync import tar
|
||||
LOOPBACK = os.environ.get('LOOPBACK', None)
|
||||
|
||||
|
||||
class TestCasync(unittest.TestCase):
|
||||
@pytest.mark.skip("not used yet")
|
||||
class TestCasync:
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
def setup_class(cls):
|
||||
cls.tmpdir = tempfile.TemporaryDirectory()
|
||||
|
||||
# Build example contents
|
||||
@@ -43,7 +43,7 @@ class TestCasync(unittest.TestCase):
|
||||
# Ensure we have chunk reuse
|
||||
assert len(hashes) > len(set(hashes))
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
# Clear target_lo
|
||||
if LOOPBACK is not None:
|
||||
self.target_lo = LOOPBACK
|
||||
@@ -53,7 +53,7 @@ class TestCasync(unittest.TestCase):
|
||||
self.target_fn = os.path.join(self.tmpdir.name, next(tempfile._get_candidate_names()))
|
||||
self.seed_fn = os.path.join(self.tmpdir.name, next(tempfile._get_candidate_names()))
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
for fn in [self.target_fn, self.seed_fn]:
|
||||
try:
|
||||
os.unlink(fn)
|
||||
@@ -67,9 +67,9 @@ class TestCasync(unittest.TestCase):
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as target_f:
|
||||
self.assertEqual(target_f.read(), self.contents)
|
||||
assert target_f.read() == self.contents
|
||||
|
||||
self.assertEqual(stats['remote'], len(self.contents))
|
||||
assert stats['remote'] == len(self.contents)
|
||||
|
||||
def test_seed(self):
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
@@ -83,10 +83,10 @@ class TestCasync(unittest.TestCase):
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as target_f:
|
||||
self.assertEqual(target_f.read(), self.contents)
|
||||
assert target_f.read() == self.contents
|
||||
|
||||
self.assertGreater(stats['seed'], 0)
|
||||
self.assertLess(stats['remote'], len(self.contents))
|
||||
assert stats['seed'] > 0
|
||||
assert stats['remote'] < len(self.contents)
|
||||
|
||||
def test_already_done(self):
|
||||
"""Test that an already flashed target doesn't download any chunks"""
|
||||
@@ -101,9 +101,9 @@ class TestCasync(unittest.TestCase):
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.contents)
|
||||
assert f.read() == self.contents
|
||||
|
||||
self.assertEqual(stats['target'], len(self.contents))
|
||||
assert stats['target'] == len(self.contents)
|
||||
|
||||
def test_chunk_reuse(self):
|
||||
"""Test that chunks that are reused are only downloaded once"""
|
||||
@@ -119,11 +119,11 @@ class TestCasync(unittest.TestCase):
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as f:
|
||||
self.assertEqual(f.read(), self.contents)
|
||||
assert f.read() == self.contents
|
||||
|
||||
self.assertLess(stats['remote'], len(self.contents))
|
||||
assert stats['remote'] < len(self.contents)
|
||||
|
||||
@unittest.skipUnless(LOOPBACK, "requires loopback device")
|
||||
@pytest.mark.skipif(not LOOPBACK, reason="requires loopback device")
|
||||
def test_lo_simple_extract(self):
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
sources = [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
@@ -131,11 +131,11 @@ class TestCasync(unittest.TestCase):
|
||||
stats = casync.extract(target, sources, self.target_lo)
|
||||
|
||||
with open(self.target_lo, 'rb') as target_f:
|
||||
self.assertEqual(target_f.read(len(self.contents)), self.contents)
|
||||
assert target_f.read(len(self.contents)) == self.contents
|
||||
|
||||
self.assertEqual(stats['remote'], len(self.contents))
|
||||
assert stats['remote'] == len(self.contents)
|
||||
|
||||
@unittest.skipUnless(LOOPBACK, "requires loopback device")
|
||||
@pytest.mark.skipif(not LOOPBACK, reason="requires loopback device")
|
||||
def test_lo_chunk_reuse(self):
|
||||
"""Test that chunks that are reused are only downloaded once"""
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
@@ -146,12 +146,13 @@ class TestCasync(unittest.TestCase):
|
||||
stats = casync.extract(target, sources, self.target_lo)
|
||||
|
||||
with open(self.target_lo, 'rb') as f:
|
||||
self.assertEqual(f.read(len(self.contents)), self.contents)
|
||||
assert f.read(len(self.contents)) == self.contents
|
||||
|
||||
self.assertLess(stats['remote'], len(self.contents))
|
||||
assert stats['remote'] < len(self.contents)
|
||||
|
||||
|
||||
class TestCasyncDirectory(unittest.TestCase):
|
||||
@pytest.mark.skip("not used yet")
|
||||
class TestCasyncDirectory:
|
||||
"""Tests extracting a directory stored as a casync tar archive"""
|
||||
|
||||
NUM_FILES = 16
|
||||
@@ -174,7 +175,7 @@ class TestCasyncDirectory(unittest.TestCase):
|
||||
os.symlink(f"file_{i}.txt", os.path.join(directory, f"link_{i}.txt"))
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
def setup_class(cls):
|
||||
cls.tmpdir = tempfile.TemporaryDirectory()
|
||||
|
||||
# Create casync files
|
||||
@@ -190,16 +191,16 @@ class TestCasyncDirectory(unittest.TestCase):
|
||||
subprocess.check_output(["casync", "make", "--compression=xz", "--store", cls.store_fn, cls.manifest_fn, cls.orig_fn])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
def teardown_class(cls):
|
||||
cls.tmpdir.cleanup()
|
||||
cls.directory_to_extract.cleanup()
|
||||
|
||||
def setUp(self):
|
||||
def setup_method(self):
|
||||
self.cache_dir = tempfile.TemporaryDirectory()
|
||||
self.working_dir = tempfile.TemporaryDirectory()
|
||||
self.out_dir = tempfile.TemporaryDirectory()
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
self.cache_dir.cleanup()
|
||||
self.working_dir.cleanup()
|
||||
self.out_dir.cleanup()
|
||||
@@ -216,32 +217,32 @@ class TestCasyncDirectory(unittest.TestCase):
|
||||
stats = casync.extract_directory(target, sources, pathlib.Path(self.out_dir.name), tmp_filename)
|
||||
|
||||
with open(os.path.join(self.out_dir.name, "file_0.txt"), "rb") as f:
|
||||
self.assertEqual(f.read(), self.contents)
|
||||
assert f.read() == self.contents
|
||||
|
||||
with open(os.path.join(self.out_dir.name, "link_0.txt"), "rb") as f:
|
||||
self.assertEqual(f.read(), self.contents)
|
||||
self.assertEqual(os.readlink(os.path.join(self.out_dir.name, "link_0.txt")), "file_0.txt")
|
||||
assert f.read() == self.contents
|
||||
assert os.readlink(os.path.join(self.out_dir.name, "link_0.txt")) == "file_0.txt"
|
||||
|
||||
return stats
|
||||
|
||||
def test_no_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, [])
|
||||
stats = self.run_test()
|
||||
self.assertGreater(stats['remote'], 0)
|
||||
self.assertEqual(stats['cache'], 0)
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] == 0
|
||||
|
||||
def test_full_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(self.NUM_FILES))
|
||||
stats = self.run_test()
|
||||
self.assertEqual(stats['remote'], 0)
|
||||
self.assertGreater(stats['cache'], 0)
|
||||
assert stats['remote'] == 0
|
||||
assert stats['cache'] > 0
|
||||
|
||||
def test_one_file_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(1))
|
||||
stats = self.run_test()
|
||||
self.assertGreater(stats['remote'], 0)
|
||||
self.assertGreater(stats['cache'], 0)
|
||||
self.assertLess(stats['cache'], stats['remote'])
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] > 0
|
||||
assert stats['cache'] < stats['remote']
|
||||
|
||||
def test_one_file_incorrect_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(self.NUM_FILES))
|
||||
@@ -249,19 +250,15 @@ class TestCasyncDirectory(unittest.TestCase):
|
||||
f.write(b"1234")
|
||||
|
||||
stats = self.run_test()
|
||||
self.assertGreater(stats['remote'], 0)
|
||||
self.assertGreater(stats['cache'], 0)
|
||||
self.assertGreater(stats['cache'], stats['remote'])
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] > 0
|
||||
assert stats['cache'] > stats['remote']
|
||||
|
||||
def test_one_file_missing_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(self.NUM_FILES))
|
||||
os.unlink(os.path.join(self.cache_dir.name, "file_12.txt"))
|
||||
|
||||
stats = self.run_test()
|
||||
self.assertGreater(stats['remote'], 0)
|
||||
self.assertGreater(stats['cache'], 0)
|
||||
self.assertGreater(stats['cache'], stats['remote'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] > 0
|
||||
assert stats['cache'] > stats['remote']
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.manager.process import ManagerProcess
|
||||
from openpilot.selfdrive.test.helpers import processes_context
|
||||
|
||||
|
||||
def get_consistent_flag(path: str) -> bool:
|
||||
consistent_file = pathlib.Path(os.path.join(path, ".overlay_consistent"))
|
||||
return consistent_file.is_file()
|
||||
|
||||
|
||||
def run(args, **kwargs):
|
||||
return subprocess.check_output(args, **kwargs)
|
||||
|
||||
|
||||
def update_release(directory, name, version, agnos_version, release_notes):
|
||||
with open(directory / "RELEASES.md", "w") as f:
|
||||
f.write(release_notes)
|
||||
|
||||
(directory / "common").mkdir(exist_ok=True)
|
||||
|
||||
with open(directory / "common" / "version.h", "w") as f:
|
||||
f.write(f'#define COMMA_VERSION "{version}"')
|
||||
|
||||
launch_env = directory / "launch_env.sh"
|
||||
with open(launch_env, "w") as f:
|
||||
f.write(f'export AGNOS_VERSION="{agnos_version}"')
|
||||
|
||||
st = os.stat(launch_env)
|
||||
os.chmod(launch_env, st.st_mode | stat.S_IEXEC)
|
||||
|
||||
test_symlink = directory / "test_symlink"
|
||||
if not os.path.exists(str(test_symlink)):
|
||||
os.symlink("common/version.h", test_symlink)
|
||||
|
||||
|
||||
def get_version(path: str) -> str:
|
||||
with open(os.path.join(path, "common", "version.h")) as f:
|
||||
return f.read().split('"')[1]
|
||||
|
||||
|
||||
@pytest.mark.slow # TODO: can we test overlayfs in GHA?
|
||||
class TestBaseUpdate:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
if "Base" in cls.__name__:
|
||||
pytest.skip()
|
||||
|
||||
def setup_method(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
run(["sudo", "mount", "-t", "tmpfs", "tmpfs", self.tmpdir]) # overlayfs doesn't work inside of docker unless this is a tmpfs
|
||||
|
||||
self.mock_update_path = pathlib.Path(self.tmpdir)
|
||||
|
||||
self.params = Params()
|
||||
|
||||
self.basedir = self.mock_update_path / "openpilot"
|
||||
self.basedir.mkdir()
|
||||
|
||||
self.staging_root = self.mock_update_path / "safe_staging"
|
||||
self.staging_root.mkdir()
|
||||
|
||||
self.remote_dir = self.mock_update_path / "remote"
|
||||
self.remote_dir.mkdir()
|
||||
|
||||
os.environ["UPDATER_STAGING_ROOT"] = str(self.staging_root)
|
||||
os.environ["UPDATER_LOCK_FILE"] = str(self.mock_update_path / "safe_staging_overlay.lock")
|
||||
|
||||
self.MOCK_RELEASES = {
|
||||
"release3": ("0.1.2", "1.2", "0.1.2 release notes"),
|
||||
"master": ("0.1.3", "1.2", "0.1.3 release notes"),
|
||||
}
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_basedir(self, mocker):
|
||||
mocker.patch("openpilot.common.basedir.BASEDIR", self.basedir)
|
||||
|
||||
def set_target_branch(self, branch):
|
||||
self.params.put("UpdaterTargetBranch", branch)
|
||||
|
||||
def setup_basedir_release(self, release):
|
||||
self.params = Params()
|
||||
self.set_target_branch(release)
|
||||
|
||||
def update_remote_release(self, release):
|
||||
raise NotImplementedError("")
|
||||
|
||||
def setup_remote_release(self, release):
|
||||
raise NotImplementedError("")
|
||||
|
||||
def additional_context(self):
|
||||
raise NotImplementedError("")
|
||||
|
||||
def teardown_method(self):
|
||||
try:
|
||||
run(["sudo", "umount", "-l", str(self.staging_root / "merged")])
|
||||
run(["sudo", "umount", "-l", self.tmpdir])
|
||||
shutil.rmtree(self.tmpdir)
|
||||
except Exception:
|
||||
print("cleanup failed...")
|
||||
|
||||
def wait_for_condition(self, condition, timeout=12):
|
||||
start = time.monotonic()
|
||||
while True:
|
||||
waited = time.monotonic() - start
|
||||
if condition():
|
||||
print(f"waited {waited}s for condition ")
|
||||
return waited
|
||||
|
||||
if waited > timeout:
|
||||
raise TimeoutError("timed out waiting for condition")
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
def _test_finalized_update(self, branch, version, agnos_version, release_notes):
|
||||
assert get_version(str(self.staging_root / "finalized")) == version
|
||||
assert get_consistent_flag(str(self.staging_root / "finalized"))
|
||||
assert os.access(str(self.staging_root / "finalized" / "launch_env.sh"), os.X_OK)
|
||||
|
||||
with open(self.staging_root / "finalized" / "test_symlink") as f:
|
||||
assert version in f.read()
|
||||
|
||||
class ParamsBaseUpdateTest(TestBaseUpdate):
|
||||
def _test_finalized_update(self, branch, version, agnos_version, release_notes):
|
||||
assert self.params.get("UpdaterNewDescription", encoding="utf-8").startswith(f"{version} / {branch}")
|
||||
assert self.params.get("UpdaterNewReleaseNotes", encoding="utf-8") == f"<p>{release_notes}</p>\n"
|
||||
super()._test_finalized_update(branch, version, agnos_version, release_notes)
|
||||
|
||||
def send_check_for_updates_signal(self, updated: ManagerProcess):
|
||||
updated.signal(signal.SIGUSR1.value)
|
||||
|
||||
def send_download_signal(self, updated: ManagerProcess):
|
||||
updated.signal(signal.SIGHUP.value)
|
||||
|
||||
def _test_params(self, branch, fetch_available, update_available):
|
||||
assert self.params.get("UpdaterTargetBranch", encoding="utf-8") == branch
|
||||
assert self.params.get_bool("UpdaterFetchAvailable") == fetch_available
|
||||
assert self.params.get_bool("UpdateAvailable") == update_available
|
||||
|
||||
def wait_for_idle(self):
|
||||
self.wait_for_condition(lambda: self.params.get("UpdaterState", encoding="utf-8") == "idle")
|
||||
|
||||
def wait_for_failed(self):
|
||||
self.wait_for_condition(lambda: self.params.get("UpdateFailedCount", encoding="utf-8") is not None and \
|
||||
int(self.params.get("UpdateFailedCount", encoding="utf-8")) > 0)
|
||||
|
||||
def wait_for_fetch_available(self):
|
||||
self.wait_for_condition(lambda: self.params.get_bool("UpdaterFetchAvailable"))
|
||||
|
||||
def wait_for_update_available(self):
|
||||
self.wait_for_condition(lambda: self.params.get_bool("UpdateAvailable"))
|
||||
|
||||
def test_no_update(self):
|
||||
# Start on release3, ensure we don't fetch any updates
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_idle()
|
||||
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
def test_new_release(self):
|
||||
# Start on release3, simulate a release3 commit, ensure we fetch that update properly
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.MOCK_RELEASES["release3"] = ("0.1.3", "1.2", "0.1.3 release notes")
|
||||
self.update_remote_release("release3")
|
||||
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_fetch_available()
|
||||
|
||||
self._test_params("release3", True, False)
|
||||
|
||||
self.send_download_signal(updated)
|
||||
|
||||
self.wait_for_update_available()
|
||||
|
||||
self._test_params("release3", False, True)
|
||||
self._test_finalized_update("release3", *self.MOCK_RELEASES["release3"])
|
||||
|
||||
def test_switch_branches(self):
|
||||
# Start on release3, request to switch to master manually, ensure we switched
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_remote_release("master")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.set_target_branch("master")
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_fetch_available()
|
||||
|
||||
self._test_params("master", True, False)
|
||||
|
||||
self.send_download_signal(updated)
|
||||
|
||||
self.wait_for_update_available()
|
||||
|
||||
self._test_params("master", False, True)
|
||||
self._test_finalized_update("master", *self.MOCK_RELEASES["master"])
|
||||
|
||||
def test_agnos_update(self, mocker):
|
||||
# Start on release3, push an update with an agnos change
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
mocker.patch("openpilot.system.hardware.AGNOS", "True")
|
||||
mocker.patch("openpilot.system.hardware.tici.hardware.Tici.get_os_version", "1.2")
|
||||
mocker.patch("openpilot.system.hardware.tici.agnos.get_target_slot_number")
|
||||
mocker.patch("openpilot.system.hardware.tici.agnos.flash_agnos_update")
|
||||
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.MOCK_RELEASES["release3"] = ("0.1.3", "1.3", "0.1.3 release notes")
|
||||
self.update_remote_release("release3")
|
||||
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_fetch_available()
|
||||
|
||||
self._test_params("release3", True, False)
|
||||
|
||||
self.send_download_signal(updated)
|
||||
|
||||
self.wait_for_update_available()
|
||||
|
||||
self._test_params("release3", False, True)
|
||||
self._test_finalized_update("release3", *self.MOCK_RELEASES["release3"])
|
||||
@@ -0,0 +1,22 @@
|
||||
import contextlib
|
||||
from openpilot.system.updated.tests.test_base import ParamsBaseUpdateTest, run, update_release
|
||||
|
||||
|
||||
class TestUpdateDGitStrategy(ParamsBaseUpdateTest):
|
||||
def update_remote_release(self, release):
|
||||
update_release(self.remote_dir, release, *self.MOCK_RELEASES[release])
|
||||
run(["git", "add", "."], cwd=self.remote_dir)
|
||||
run(["git", "commit", "-m", f"openpilot release {release}"], cwd=self.remote_dir)
|
||||
|
||||
def setup_remote_release(self, release):
|
||||
run(["git", "init"], cwd=self.remote_dir)
|
||||
run(["git", "checkout", "-b", release], cwd=self.remote_dir)
|
||||
self.update_remote_release(release)
|
||||
|
||||
def setup_basedir_release(self, release):
|
||||
super().setup_basedir_release(release)
|
||||
run(["git", "clone", "-b", release, self.remote_dir, self.basedir])
|
||||
|
||||
@contextlib.contextmanager
|
||||
def additional_context(self):
|
||||
yield
|
||||
Executable
+509
@@ -0,0 +1,509 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import re
|
||||
import datetime
|
||||
import subprocess
|
||||
import psutil
|
||||
import shutil
|
||||
import signal
|
||||
import fcntl
|
||||
import time
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.time import system_time_valid
|
||||
from openpilot.common.markdown import parse_markdown
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
|
||||
from openpilot.system.hardware import AGNOS, HARDWARE
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
|
||||
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
|
||||
|
||||
OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper")
|
||||
OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata")
|
||||
OVERLAY_MERGED = os.path.join(STAGING_ROOT, "merged")
|
||||
FINALIZED = os.path.join(STAGING_ROOT, "finalized")
|
||||
|
||||
OVERLAY_INIT = Path(os.path.join(BASEDIR, ".overlay_init"))
|
||||
|
||||
DAYS_NO_CONNECTIVITY_MAX = 14 # do not allow to engage after this many days
|
||||
DAYS_NO_CONNECTIVITY_PROMPT = 10 # send an offroad prompt after this many days
|
||||
|
||||
class UserRequest:
|
||||
NONE = 0
|
||||
CHECK = 1
|
||||
FETCH = 2
|
||||
|
||||
class WaitTimeHelper:
|
||||
def __init__(self):
|
||||
self.ready_event = threading.Event()
|
||||
self.user_request = UserRequest.NONE
|
||||
signal.signal(signal.SIGHUP, self.update_now)
|
||||
signal.signal(signal.SIGUSR1, self.check_now)
|
||||
|
||||
def update_now(self, signum: int, frame) -> None:
|
||||
cloudlog.info("caught SIGHUP, attempting to downloading update")
|
||||
self.user_request = UserRequest.FETCH
|
||||
self.ready_event.set()
|
||||
|
||||
def check_now(self, signum: int, frame) -> None:
|
||||
cloudlog.info("caught SIGUSR1, checking for updates")
|
||||
self.user_request = UserRequest.CHECK
|
||||
self.ready_event.set()
|
||||
|
||||
def sleep(self, t: float) -> None:
|
||||
self.ready_event.wait(timeout=t)
|
||||
|
||||
def write_time_to_param(params, param) -> None:
|
||||
t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
params.put(param, t.isoformat().encode('utf8'))
|
||||
|
||||
def read_time_from_param(params, param) -> datetime.datetime | None:
|
||||
t = params.get(param, encoding='utf8')
|
||||
try:
|
||||
return datetime.datetime.fromisoformat(t)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return None
|
||||
|
||||
def run(cmd: list[str], cwd: str = None) -> str:
|
||||
return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')
|
||||
|
||||
|
||||
def set_consistent_flag(consistent: bool) -> None:
|
||||
os.sync()
|
||||
consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
|
||||
if consistent:
|
||||
consistent_file.touch()
|
||||
elif not consistent:
|
||||
consistent_file.unlink(missing_ok=True)
|
||||
os.sync()
|
||||
|
||||
def parse_release_notes(basedir: str) -> bytes:
|
||||
try:
|
||||
with open(os.path.join(basedir, "RELEASES.md"), "rb") as f:
|
||||
r = f.read().split(b'\n\n', 1)[0] # Slice latest release notes
|
||||
try:
|
||||
return bytes(parse_markdown(r.decode("utf-8")), encoding="utf-8")
|
||||
except Exception:
|
||||
return r + b"\n"
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception:
|
||||
cloudlog.exception("failed to parse release notes")
|
||||
return b""
|
||||
|
||||
def setup_git_options(cwd: str) -> None:
|
||||
# We sync FS object atimes (which NEOS doesn't use) and mtimes, but ctimes
|
||||
# are outside user control. Make sure Git is set up to ignore system ctimes,
|
||||
# because they change when we make hard links during finalize. Otherwise,
|
||||
# there is a lot of unnecessary churn. This appears to be a common need on
|
||||
# OSX as well: https://www.git-tower.com/blog/make-git-rebase-safe-on-osx/
|
||||
|
||||
# We are using copytree to copy the directory, which also changes
|
||||
# inode numbers. Ignore those changes too.
|
||||
|
||||
# Set protocol to the new version (default after git 2.26) to reduce data
|
||||
# usage on git fetch --dry-run from about 400KB to 18KB.
|
||||
git_cfg = [
|
||||
("core.trustctime", "false"),
|
||||
("core.checkStat", "minimal"),
|
||||
("protocol.version", "2"),
|
||||
("gc.auto", "0"),
|
||||
("gc.autoDetach", "false"),
|
||||
]
|
||||
for option, value in git_cfg:
|
||||
run(["git", "config", option, value], cwd)
|
||||
|
||||
|
||||
def dismount_overlay() -> None:
|
||||
if os.path.ismount(OVERLAY_MERGED):
|
||||
cloudlog.info("unmounting existing overlay")
|
||||
run(["sudo", "umount", "-l", OVERLAY_MERGED])
|
||||
|
||||
|
||||
def init_overlay() -> None:
|
||||
|
||||
# Re-create the overlay if BASEDIR/.git has changed since we created the overlay
|
||||
if OVERLAY_INIT.is_file() and os.path.ismount(OVERLAY_MERGED):
|
||||
git_dir_path = os.path.join(BASEDIR, ".git")
|
||||
new_files = run(["find", git_dir_path, "-newer", str(OVERLAY_INIT)])
|
||||
if not len(new_files.splitlines()):
|
||||
# A valid overlay already exists
|
||||
return
|
||||
else:
|
||||
cloudlog.info(".git directory changed, recreating overlay")
|
||||
|
||||
cloudlog.info("preparing new safe staging area")
|
||||
|
||||
params = Params()
|
||||
params.put_bool("UpdateAvailable", False)
|
||||
set_consistent_flag(False)
|
||||
dismount_overlay()
|
||||
run(["sudo", "rm", "-rf", STAGING_ROOT])
|
||||
if os.path.isdir(STAGING_ROOT):
|
||||
shutil.rmtree(STAGING_ROOT)
|
||||
|
||||
for dirname in [STAGING_ROOT, OVERLAY_UPPER, OVERLAY_METADATA, OVERLAY_MERGED]:
|
||||
os.mkdir(dirname, 0o755)
|
||||
|
||||
if os.lstat(BASEDIR).st_dev != os.lstat(OVERLAY_MERGED).st_dev:
|
||||
raise RuntimeError("base and overlay merge directories are on different filesystems; not valid for overlay FS!")
|
||||
|
||||
# Leave a timestamped canary in BASEDIR to check at startup. The device clock
|
||||
# should be correct by the time we get here. If the init file disappears, or
|
||||
# critical mtimes in BASEDIR are newer than .overlay_init, continue.sh can
|
||||
# assume that BASEDIR has used for local development or otherwise modified,
|
||||
# and skips the update activation attempt.
|
||||
consistent_file = Path(os.path.join(BASEDIR, ".overlay_consistent"))
|
||||
if consistent_file.is_file():
|
||||
consistent_file.unlink()
|
||||
OVERLAY_INIT.touch()
|
||||
|
||||
os.sync()
|
||||
overlay_opts = f"lowerdir={BASEDIR},upperdir={OVERLAY_UPPER},workdir={OVERLAY_METADATA}"
|
||||
|
||||
mount_cmd = ["mount", "-t", "overlay", "-o", overlay_opts, "none", OVERLAY_MERGED]
|
||||
run(["sudo"] + mount_cmd)
|
||||
run(["sudo", "chmod", "755", os.path.join(OVERLAY_METADATA, "work")])
|
||||
|
||||
git_diff = run(["git", "diff"], OVERLAY_MERGED)
|
||||
params.put("GitDiff", git_diff)
|
||||
cloudlog.info(f"git diff output:\n{git_diff}")
|
||||
|
||||
|
||||
def finalize_update() -> None:
|
||||
"""Take the current OverlayFS merged view and finalize a copy outside of
|
||||
OverlayFS, ready to be swapped-in at BASEDIR. Copy using shutil.copytree"""
|
||||
|
||||
# Remove the update ready flag and any old updates
|
||||
cloudlog.info("creating finalized version of the overlay")
|
||||
set_consistent_flag(False)
|
||||
|
||||
# Copy the merged overlay view and set the update ready flag
|
||||
if os.path.exists(FINALIZED):
|
||||
shutil.rmtree(FINALIZED)
|
||||
shutil.copytree(OVERLAY_MERGED, FINALIZED, symlinks=True)
|
||||
|
||||
run(["git", "reset", "--hard"], FINALIZED)
|
||||
run(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"], FINALIZED)
|
||||
|
||||
cloudlog.info("Starting git cleanup in finalized update")
|
||||
t = time.monotonic()
|
||||
try:
|
||||
run(["git", "gc"], FINALIZED)
|
||||
run(["git", "lfs", "prune"], FINALIZED)
|
||||
cloudlog.event("Done git cleanup", duration=time.monotonic() - t)
|
||||
except subprocess.CalledProcessError:
|
||||
cloudlog.exception(f"Failed git cleanup, took {time.monotonic() - t:.3f} s")
|
||||
|
||||
set_consistent_flag(True)
|
||||
cloudlog.info("done finalizing overlay")
|
||||
|
||||
|
||||
def handle_agnos_update() -> None:
|
||||
from openpilot.system.hardware.tici.agnos import flash_agnos_update, get_target_slot_number
|
||||
|
||||
cur_version = HARDWARE.get_os_version()
|
||||
updated_version = run(["bash", "-c", r"unset AGNOS_VERSION && source launch_env.sh && \
|
||||
echo -n $AGNOS_VERSION"], OVERLAY_MERGED).strip()
|
||||
|
||||
cloudlog.info(f"AGNOS version check: {cur_version} vs {updated_version}")
|
||||
if cur_version == updated_version:
|
||||
return
|
||||
|
||||
# prevent an openpilot getting swapped in with a mismatched or partially downloaded agnos
|
||||
set_consistent_flag(False)
|
||||
|
||||
cloudlog.info(f"Beginning background installation for AGNOS {updated_version}")
|
||||
set_offroad_alert("Offroad_NeosUpdate", True)
|
||||
|
||||
manifest_path = os.path.join(OVERLAY_MERGED, "system/hardware/tici/agnos.json")
|
||||
target_slot_number = get_target_slot_number()
|
||||
flash_agnos_update(manifest_path, target_slot_number, cloudlog)
|
||||
set_offroad_alert("Offroad_NeosUpdate", False)
|
||||
|
||||
|
||||
|
||||
class Updater:
|
||||
def __init__(self):
|
||||
self.params = Params()
|
||||
self.branches = defaultdict(str)
|
||||
self._has_internet: bool = False
|
||||
|
||||
@property
|
||||
def has_internet(self) -> bool:
|
||||
return self._has_internet
|
||||
|
||||
@property
|
||||
def target_branch(self) -> str:
|
||||
b: str | None = self.params.get("UpdaterTargetBranch", encoding='utf-8')
|
||||
if b is None:
|
||||
b = self.get_branch(BASEDIR)
|
||||
return b
|
||||
|
||||
@property
|
||||
def update_ready(self) -> bool:
|
||||
consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
|
||||
if consistent_file.is_file():
|
||||
hash_mismatch = self.get_commit_hash(BASEDIR) != self.branches[self.target_branch]
|
||||
branch_mismatch = self.get_branch(BASEDIR) != self.target_branch
|
||||
on_target_branch = self.get_branch(FINALIZED) == self.target_branch
|
||||
return ((hash_mismatch or branch_mismatch) and on_target_branch)
|
||||
return False
|
||||
|
||||
@property
|
||||
def update_available(self) -> bool:
|
||||
if os.path.isdir(OVERLAY_MERGED) and len(self.branches) > 0:
|
||||
hash_mismatch = self.get_commit_hash(OVERLAY_MERGED) != self.branches[self.target_branch]
|
||||
branch_mismatch = self.get_branch(OVERLAY_MERGED) != self.target_branch
|
||||
return hash_mismatch or branch_mismatch
|
||||
return False
|
||||
|
||||
def get_branch(self, path: str) -> str:
|
||||
return run(["git", "rev-parse", "--abbrev-ref", "HEAD"], path).rstrip()
|
||||
|
||||
def get_commit_hash(self, path: str = OVERLAY_MERGED) -> str:
|
||||
return run(["git", "rev-parse", "HEAD"], path).rstrip()
|
||||
|
||||
def set_params(self, update_success: bool, failed_count: int, exception: str | None) -> None:
|
||||
self.params.put("UpdateFailedCount", str(failed_count))
|
||||
self.params.put("UpdaterTargetBranch", self.target_branch)
|
||||
|
||||
self.params.put_bool("UpdaterFetchAvailable", self.update_available)
|
||||
if len(self.branches):
|
||||
self.params.put("UpdaterAvailableBranches", ','.join(self.branches.keys()))
|
||||
|
||||
last_update = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
if update_success:
|
||||
write_time_to_param(self.params, "LastUpdateTime")
|
||||
else:
|
||||
t = read_time_from_param(self.params, "LastUpdateTime")
|
||||
if t is not None:
|
||||
last_update = t
|
||||
|
||||
if exception is None:
|
||||
self.params.remove("LastUpdateException")
|
||||
else:
|
||||
self.params.put("LastUpdateException", exception)
|
||||
|
||||
# Write out current and new version info
|
||||
def get_description(basedir: str) -> str:
|
||||
if not os.path.exists(basedir):
|
||||
return ""
|
||||
|
||||
version = ""
|
||||
branch = ""
|
||||
commit = ""
|
||||
commit_date = ""
|
||||
try:
|
||||
branch = self.get_branch(basedir)
|
||||
commit = self.get_commit_hash(basedir)[:7]
|
||||
with open(os.path.join(basedir, "common", "version.h")) as f:
|
||||
version = f.read().split('"')[1]
|
||||
|
||||
commit_unix_ts = run(["git", "show", "-s", "--format=%ct", "HEAD"], basedir).rstrip()
|
||||
dt = datetime.datetime.fromtimestamp(int(commit_unix_ts))
|
||||
commit_date = dt.strftime("%b %d")
|
||||
except Exception:
|
||||
cloudlog.exception("updater.get_description")
|
||||
return f"{version} / {branch} / {commit} / {commit_date}"
|
||||
self.params.put("UpdaterCurrentDescription", get_description(BASEDIR))
|
||||
self.params.put("UpdaterCurrentReleaseNotes", parse_release_notes(BASEDIR))
|
||||
self.params.put("UpdaterNewDescription", get_description(FINALIZED))
|
||||
self.params.put("UpdaterNewReleaseNotes", parse_release_notes(FINALIZED))
|
||||
self.params.put_bool("UpdateAvailable", self.update_ready)
|
||||
|
||||
# Handle user prompt
|
||||
for alert in ("Offroad_UpdateFailed", "Offroad_ConnectivityNeeded", "Offroad_ConnectivityNeededPrompt"):
|
||||
set_offroad_alert(alert, False)
|
||||
|
||||
now = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
dt = now - last_update
|
||||
build_metadata = get_build_metadata()
|
||||
if failed_count > 15 and exception is not None and self.has_internet:
|
||||
if build_metadata.tested_channel:
|
||||
extra_text = "Ensure the software is correctly installed. Uninstall and re-install if this error persists."
|
||||
else:
|
||||
extra_text = exception
|
||||
set_offroad_alert("Offroad_UpdateFailed", True, extra_text=extra_text)
|
||||
elif failed_count > 0:
|
||||
if dt.days > DAYS_NO_CONNECTIVITY_MAX:
|
||||
set_offroad_alert("Offroad_ConnectivityNeeded", True)
|
||||
elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT:
|
||||
remaining = max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 1)
|
||||
set_offroad_alert("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining} day{'' if remaining == 1 else 's'}.")
|
||||
|
||||
def check_for_update(self) -> None:
|
||||
cloudlog.info("checking for updates")
|
||||
|
||||
excluded_branches = ('release2', 'release2-staging')
|
||||
|
||||
try:
|
||||
run(["git", "ls-remote", "origin", "HEAD"], OVERLAY_MERGED)
|
||||
self._has_internet = True
|
||||
except subprocess.CalledProcessError:
|
||||
self._has_internet = False
|
||||
|
||||
setup_git_options(OVERLAY_MERGED)
|
||||
output = run(["git", "ls-remote", "--heads"], OVERLAY_MERGED)
|
||||
|
||||
self.branches = defaultdict(lambda: None)
|
||||
for line in output.split('\n'):
|
||||
ls_remotes_re = r'(?P<commit_sha>\b[0-9a-f]{5,40}\b)(\s+)(refs\/heads\/)(?P<branch_name>.*$)'
|
||||
x = re.fullmatch(ls_remotes_re, line.strip())
|
||||
if x is not None and x.group('branch_name') not in excluded_branches:
|
||||
self.branches[x.group('branch_name')] = x.group('commit_sha')
|
||||
|
||||
cur_branch = self.get_branch(OVERLAY_MERGED)
|
||||
cur_commit = self.get_commit_hash(OVERLAY_MERGED)
|
||||
new_branch = self.target_branch
|
||||
new_commit = self.branches[new_branch]
|
||||
if (cur_branch, cur_commit) != (new_branch, new_commit):
|
||||
cloudlog.info(f"update available, {cur_branch} ({str(cur_commit)[:7]}) -> {new_branch} ({str(new_commit)[:7]})")
|
||||
else:
|
||||
cloudlog.info(f"up to date on {cur_branch} ({str(cur_commit)[:7]})")
|
||||
|
||||
def fetch_update(self) -> None:
|
||||
cloudlog.info("attempting git fetch inside staging overlay")
|
||||
|
||||
self.params.put("UpdaterState", "downloading...")
|
||||
|
||||
# TODO: cleanly interrupt this and invalidate old update
|
||||
set_consistent_flag(False)
|
||||
self.params.put_bool("UpdateAvailable", False)
|
||||
|
||||
setup_git_options(OVERLAY_MERGED)
|
||||
|
||||
branch = self.target_branch
|
||||
git_fetch_output = run(["git", "fetch", "origin", branch], OVERLAY_MERGED)
|
||||
cloudlog.info("git fetch success: %s", git_fetch_output)
|
||||
|
||||
cloudlog.info("git reset in progress")
|
||||
cmds = [
|
||||
["git", "checkout", "--force", "--no-recurse-submodules", "-B", branch, "FETCH_HEAD"],
|
||||
["git", "reset", "--hard"],
|
||||
["git", "clean", "-xdff"],
|
||||
["git", "submodule", "sync"],
|
||||
["git", "submodule", "update", "--init", "--recursive"],
|
||||
["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"],
|
||||
]
|
||||
r = [run(cmd, OVERLAY_MERGED) for cmd in cmds]
|
||||
cloudlog.info("git reset success: %s", '\n'.join(r))
|
||||
|
||||
# TODO: show agnos download progress
|
||||
if AGNOS:
|
||||
handle_agnos_update()
|
||||
|
||||
# Create the finalized, ready-to-swap update
|
||||
self.params.put("UpdaterState", "finalizing update...")
|
||||
finalize_update()
|
||||
cloudlog.info("finalize success!")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
params = Params()
|
||||
|
||||
if params.get_bool("DisableUpdates"):
|
||||
cloudlog.warning("updates are disabled by the DisableUpdates param")
|
||||
exit(0)
|
||||
|
||||
with open(LOCK_FILE, 'w') as ov_lock_fd:
|
||||
try:
|
||||
fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as e:
|
||||
raise RuntimeError("couldn't get overlay lock; is another instance running?") from e
|
||||
|
||||
# Set low io priority
|
||||
proc = psutil.Process()
|
||||
if psutil.LINUX:
|
||||
proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)
|
||||
|
||||
# Check if we just performed an update
|
||||
if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
|
||||
cloudlog.event("update installed")
|
||||
|
||||
if not params.get("InstallDate"):
|
||||
t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat()
|
||||
params.put("InstallDate", t.encode('utf8'))
|
||||
|
||||
updater = Updater()
|
||||
update_failed_count = 0 # TODO: Load from param?
|
||||
wait_helper = WaitTimeHelper()
|
||||
|
||||
# invalidate old finalized update
|
||||
set_consistent_flag(False)
|
||||
|
||||
# set initial state
|
||||
params.put("UpdaterState", "idle")
|
||||
|
||||
# Run the update loop
|
||||
first_run = True
|
||||
while True:
|
||||
wait_helper.ready_event.clear()
|
||||
|
||||
# Attempt an update
|
||||
exception = None
|
||||
try:
|
||||
# TODO: reuse overlay from previous updated instance if it looks clean
|
||||
init_overlay()
|
||||
|
||||
# ensure we have some params written soon after startup
|
||||
updater.set_params(False, update_failed_count, exception)
|
||||
|
||||
if not system_time_valid() or first_run:
|
||||
first_run = False
|
||||
wait_helper.sleep(60)
|
||||
continue
|
||||
|
||||
update_failed_count += 1
|
||||
|
||||
# check for update
|
||||
params.put("UpdaterState", "checking...")
|
||||
updater.check_for_update()
|
||||
|
||||
# download update
|
||||
last_fetch = read_time_from_param(params, "UpdaterLastFetchTime")
|
||||
timed_out = last_fetch is None or (datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_fetch > datetime.timedelta(days=3))
|
||||
user_requested_fetch = wait_helper.user_request == UserRequest.FETCH
|
||||
if params.get_bool("NetworkMetered") and not timed_out and not user_requested_fetch:
|
||||
cloudlog.info("skipping fetch, connection metered")
|
||||
elif wait_helper.user_request == UserRequest.CHECK:
|
||||
cloudlog.info("skipping fetch, only checking")
|
||||
else:
|
||||
updater.fetch_update()
|
||||
write_time_to_param(params, "UpdaterLastFetchTime")
|
||||
update_failed_count = 0
|
||||
except subprocess.CalledProcessError as e:
|
||||
cloudlog.event(
|
||||
"update process failed",
|
||||
cmd=e.cmd,
|
||||
output=e.output,
|
||||
returncode=e.returncode
|
||||
)
|
||||
exception = f"command failed: {e.cmd}\n{e.output}"
|
||||
OVERLAY_INIT.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
cloudlog.exception("uncaught updated exception, shouldn't happen")
|
||||
exception = str(e)
|
||||
OVERLAY_INIT.unlink(missing_ok=True)
|
||||
|
||||
try:
|
||||
params.put("UpdaterState", "idle")
|
||||
update_successful = (update_failed_count == 0)
|
||||
updater.set_params(update_successful, update_failed_count, exception)
|
||||
except Exception:
|
||||
cloudlog.exception("uncaught updated exception while setting params, shouldn't happen")
|
||||
|
||||
# infrequent attempts if we successfully updated recently
|
||||
wait_helper.user_request = UserRequest.NONE
|
||||
wait_helper.sleep(5*60 if update_failed_count > 0 else 1.5*60*60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+2
-3
@@ -1,14 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.common.utils import cache
|
||||
from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_commit_date
|
||||
|
||||
RELEASE_BRANCHES = ['release3-staging', 'release3', 'nightly']
|
||||
@@ -27,7 +26,7 @@ def get_version(path: str = BASEDIR) -> str:
|
||||
|
||||
|
||||
def get_release_notes(path: str = BASEDIR) -> str:
|
||||
with open(os.path.join(path, "RELEASES.md"), "r") as f:
|
||||
with open(os.path.join(path, "RELEASES.md")) as f:
|
||||
return f.read().split('\n\n', 1)[0]
|
||||
|
||||
|
||||
|
||||
Executable → Regular
+35
-42
@@ -1,17 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
import unittest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
import json
|
||||
# for aiortc and its dependencies
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
|
||||
|
||||
from aiortc import RTCDataChannel
|
||||
from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
|
||||
import capnp
|
||||
import pyaudio
|
||||
|
||||
from cereal import messaging, log
|
||||
|
||||
from openpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
|
||||
@@ -20,15 +17,15 @@ from openpilot.system.webrtc.device.audio import AudioInputStreamTrack
|
||||
from openpilot.common.realtime import DT_DMON
|
||||
|
||||
|
||||
class TestStreamSession(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class TestStreamSession:
|
||||
def setup_method(self):
|
||||
self.loop = asyncio.new_event_loop()
|
||||
|
||||
def tearDown(self):
|
||||
def teardown_method(self):
|
||||
self.loop.stop()
|
||||
self.loop.close()
|
||||
|
||||
def test_outgoing_proxy(self):
|
||||
def test_outgoing_proxy(self, mocker):
|
||||
test_msg = log.Event.new_message()
|
||||
test_msg.logMonoTime = 123
|
||||
test_msg.valid = True
|
||||
@@ -36,27 +33,27 @@ class TestStreamSession(unittest.TestCase):
|
||||
expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
|
||||
expected_json = json.dumps(expected_dict).encode()
|
||||
|
||||
channel = Mock(spec=RTCDataChannel)
|
||||
channel = mocker.Mock(spec=RTCDataChannel)
|
||||
mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
|
||||
def mocked_update(t):
|
||||
mocked_submaster.update_msgs(0, [test_msg])
|
||||
|
||||
with patch.object(messaging.SubMaster, "update", side_effect=mocked_update):
|
||||
proxy = CerealOutgoingMessageProxy(mocked_submaster)
|
||||
proxy.add_channel(channel)
|
||||
mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
|
||||
proxy = CerealOutgoingMessageProxy(mocked_submaster)
|
||||
proxy.add_channel(channel)
|
||||
|
||||
proxy.update()
|
||||
proxy.update()
|
||||
|
||||
channel.send.assert_called_once_with(expected_json)
|
||||
channel.send.assert_called_once_with(expected_json)
|
||||
|
||||
def test_incoming_proxy(self):
|
||||
def test_incoming_proxy(self, mocker):
|
||||
tested_msgs = [
|
||||
{"type": "customReservedRawData0", "data": "test"}, # primitive
|
||||
{"type": "can", "data": [{"address": 0, "busTime": 0, "dat": "", "src": 0}]}, # list
|
||||
{"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
|
||||
]
|
||||
|
||||
mocked_pubmaster = MagicMock(spec=messaging.PubMaster)
|
||||
mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
|
||||
|
||||
proxy = CerealIncomingMessageProxy(mocked_pubmaster)
|
||||
|
||||
@@ -65,44 +62,40 @@ class TestStreamSession(unittest.TestCase):
|
||||
|
||||
mocked_pubmaster.send.assert_called_once()
|
||||
mt, md = mocked_pubmaster.send.call_args.args
|
||||
self.assertEqual(mt, msg["type"])
|
||||
self.assertIsInstance(md, capnp._DynamicStructBuilder)
|
||||
self.assertTrue(hasattr(md, msg["type"]))
|
||||
assert mt == msg["type"]
|
||||
assert isinstance(md, capnp._DynamicStructBuilder)
|
||||
assert hasattr(md, msg["type"])
|
||||
|
||||
mocked_pubmaster.reset_mock()
|
||||
|
||||
def test_livestream_track(self):
|
||||
def test_livestream_track(self, mocker):
|
||||
fake_msg = messaging.new_message("livestreamDriverEncodeData")
|
||||
|
||||
config = {"receive.return_value": fake_msg.to_bytes()}
|
||||
with patch("cereal.messaging.SubSocket", spec=True, **config):
|
||||
track = LiveStreamVideoStreamTrack("driver")
|
||||
mocker.patch("msgq.SubSocket", spec=True, **config)
|
||||
track = LiveStreamVideoStreamTrack("driver")
|
||||
|
||||
self.assertTrue(track.id.startswith("driver"))
|
||||
self.assertEqual(track.codec_preference(), "H264")
|
||||
assert track.id.startswith("driver")
|
||||
assert track.codec_preference() == "H264"
|
||||
|
||||
for i in range(5):
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
self.assertEqual(packet.time_base, VIDEO_TIME_BASE)
|
||||
self.assertEqual(packet.pts, int(i * DT_DMON * VIDEO_CLOCK_RATE))
|
||||
self.assertEqual(packet.size, 0)
|
||||
for i in range(5):
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
assert packet.time_base == VIDEO_TIME_BASE
|
||||
assert packet.pts == int(i * DT_DMON * VIDEO_CLOCK_RATE)
|
||||
assert packet.size == 0
|
||||
|
||||
def test_input_audio_track(self):
|
||||
def test_input_audio_track(self, mocker):
|
||||
packet_time, rate = 0.02, 16000
|
||||
sample_count = int(packet_time * rate)
|
||||
mocked_stream = MagicMock(spec=pyaudio.Stream)
|
||||
mocked_stream = mocker.MagicMock(spec=pyaudio.Stream)
|
||||
mocked_stream.read.return_value = b"\x00" * 2 * sample_count
|
||||
|
||||
config = {"open.side_effect": lambda *args, **kwargs: mocked_stream}
|
||||
with patch("pyaudio.PyAudio", spec=True, **config):
|
||||
track = AudioInputStreamTrack(audio_format=pyaudio.paInt16, packet_time=packet_time, rate=rate)
|
||||
mocker.patch("pyaudio.PyAudio", spec=True, **config)
|
||||
track = AudioInputStreamTrack(audio_format=pyaudio.paInt16, packet_time=packet_time, rate=rate)
|
||||
|
||||
for i in range(5):
|
||||
frame = self.loop.run_until_complete(track.recv())
|
||||
self.assertEqual(frame.rate, rate)
|
||||
self.assertEqual(frame.samples, sample_count)
|
||||
self.assertEqual(frame.pts, i * sample_count)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
for i in range(5):
|
||||
frame = self.loop.run_until_complete(track.recv())
|
||||
assert frame.rate == rate
|
||||
assert frame.samples == sample_count
|
||||
assert frame.pts == i * sample_count
|
||||
|
||||
Executable → Regular
+12
-15
@@ -1,11 +1,10 @@
|
||||
#!/usr/bin/env python
|
||||
import pytest
|
||||
import asyncio
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, AsyncMock
|
||||
# for aiortc and its dependencies
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
|
||||
|
||||
from openpilot.system.webrtc.webrtcd import get_stream
|
||||
|
||||
@@ -20,19 +19,20 @@ from parameterized import parameterized_class
|
||||
(["testJoystick"], []),
|
||||
([], []),
|
||||
])
|
||||
class TestWebrtcdProc(unittest.IsolatedAsyncioTestCase):
|
||||
@pytest.mark.asyncio
|
||||
class TestWebrtcdProc:
|
||||
async def assertCompletesWithTimeout(self, awaitable, timeout=1):
|
||||
try:
|
||||
async with asyncio.timeout(timeout):
|
||||
await awaitable
|
||||
except TimeoutError:
|
||||
self.fail("Timeout while waiting for awaitable to complete")
|
||||
pytest.fail("Timeout while waiting for awaitable to complete")
|
||||
|
||||
async def test_webrtcd(self):
|
||||
mock_request = MagicMock()
|
||||
async def test_webrtcd(self, mocker):
|
||||
mock_request = mocker.MagicMock()
|
||||
async def connect(offer):
|
||||
body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
|
||||
mock_request.json.side_effect = AsyncMock(return_value=body)
|
||||
mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
|
||||
response = await get_stream(mock_request)
|
||||
response_json = json.loads(response.text)
|
||||
return aiortc.RTCSessionDescription(**response_json)
|
||||
@@ -48,9 +48,9 @@ class TestWebrtcdProc(unittest.IsolatedAsyncioTestCase):
|
||||
await self.assertCompletesWithTimeout(stream.start())
|
||||
await self.assertCompletesWithTimeout(stream.wait_for_connection())
|
||||
|
||||
self.assertTrue(stream.has_incoming_video_track("road"))
|
||||
self.assertTrue(stream.has_incoming_audio_track())
|
||||
self.assertEqual(stream.has_messaging_channel(), len(self.in_services) > 0 or len(self.out_services) > 0)
|
||||
assert stream.has_incoming_video_track("road")
|
||||
assert stream.has_incoming_audio_track()
|
||||
assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
|
||||
|
||||
video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
|
||||
await self.assertCompletesWithTimeout(video_track.recv())
|
||||
@@ -59,10 +59,7 @@ class TestWebrtcdProc(unittest.IsolatedAsyncioTestCase):
|
||||
await self.assertCompletesWithTimeout(stream.stop())
|
||||
|
||||
# cleanup, very implementation specific, test may break if it changes
|
||||
self.assertTrue(mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test")
|
||||
assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
|
||||
_, session = mock_request.app["streams"].__setitem__.call_args.args
|
||||
await self.assertCompletesWithTimeout(session.post_run_cleanup())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Any, TYPE_CHECKING
|
||||
# aiortc and its dependencies have lots of internal warnings :(
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
|
||||
|
||||
import capnp
|
||||
from aiohttp import web
|
||||
@@ -24,7 +25,7 @@ from cereal import messaging, log
|
||||
class CerealOutgoingMessageProxy:
|
||||
def __init__(self, sm: messaging.SubMaster):
|
||||
self.sm = sm
|
||||
self.channels: list['RTCDataChannel'] = []
|
||||
self.channels: list[RTCDataChannel] = []
|
||||
|
||||
def add_channel(self, channel: 'RTCDataChannel'):
|
||||
self.channels.append(channel)
|
||||
@@ -97,8 +98,8 @@ class CerealProxyRunner:
|
||||
except InvalidStateError:
|
||||
self.logger.warning("Cereal outgoing proxy invalid state (connection closed)")
|
||||
break
|
||||
except Exception as ex:
|
||||
self.logger.error("Cereal outgoing proxy failure: %s", ex)
|
||||
except Exception:
|
||||
self.logger.exception("Cereal outgoing proxy failure")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
@@ -175,8 +176,8 @@ class StreamSession:
|
||||
assert self.incoming_bridge is not None
|
||||
try:
|
||||
self.incoming_bridge.send(message)
|
||||
except Exception as ex:
|
||||
self.logger.error("Cereal incoming proxy failure: %s", ex)
|
||||
except Exception:
|
||||
self.logger.exception("Cereal incoming proxy failure")
|
||||
|
||||
async def run(self):
|
||||
try:
|
||||
@@ -200,8 +201,8 @@ class StreamSession:
|
||||
await self.post_run_cleanup()
|
||||
|
||||
self.logger.info("Stream session (%s) ended", self.identifier)
|
||||
except Exception as ex:
|
||||
self.logger.error("Stream session failure: %s", ex)
|
||||
except Exception:
|
||||
self.logger.exception("Stream session failure")
|
||||
|
||||
async def post_run_cleanup(self):
|
||||
await self.stream.stop()
|
||||
|
||||
Reference in New Issue
Block a user