openpilot v0.7 release

This commit is contained in:
Vehicle Researcher
2019-12-13 13:03:08 -08:00
parent 37134796c0
commit 36282d9752
840 changed files with 247180 additions and 7303 deletions
+1
View File
@@ -0,0 +1 @@
*.cpp
+6
View File
@@ -0,0 +1,6 @@
Import('env')
# parser
env.Command(['common_pyx.so'],
['common_pyx_setup.py', 'clock.pyx'],
"cd common && python3 common_pyx_setup.py build_ext --inplace")
+73
View File
@@ -0,0 +1,73 @@
import binascii
import itertools
import re
import struct
import subprocess
def getprop(key):
return subprocess.check_output(["getprop", key], encoding='utf8').strip()
def get_imei():
ret = getprop("oem.device.imeicache")
if ret == "":
ret = "000000000000000"
return ret
def get_serial():
return getprop("ro.serialno")
def get_subscriber_info():
ret = parse_service_call_string(["iphonesubinfo", "7"])
if ret is None or len(ret) < 8:
return ""
return ret
def reboot(reason=None):
if reason is None:
reason_args = ["null"]
else:
reason_args = ["s16", reason]
subprocess.check_output([
"service", "call", "power", "16", # IPowerManager.reboot
"i32", "0", # no confirmation,
*reason_args,
"i32", "1" # wait
])
def parse_service_call_unpack(call, fmt):
r = parse_service_call_bytes(call)
try:
return struct.unpack(fmt, r)[0]
except Exception:
return None
def parse_service_call_string(call):
r = parse_service_call_bytes(call)
try:
r = r[8:] # Cut off length field
r = r.decode('utf_16_be')
# All pairs of two characters seem to be swapped. Not sure why
result = ""
for a, b, in itertools.zip_longest(r[::2], r[1::2], fillvalue='\x00'):
result += b + a
result = result.replace('\x00', '')
return result
except Exception:
return None
def parse_service_call_bytes(call):
ret = subprocess.check_output(["service", "call", *call], encoding='utf8').strip()
if 'Parcel' not in ret:
return None
try:
r = b""
for hex_part in re.findall(r'[ (]([0-9a-f]{8})', ret):
r += binascii.unhexlify(hex_part)
return r
except Exception:
return None
+98
View File
@@ -0,0 +1,98 @@
import os
import subprocess
import glob
import hashlib
import shutil
from common.basedir import BASEDIR
from selfdrive.swaglog import cloudlog
android_packages = ("ai.comma.plus.offroad", "ai.comma.plus.frame")
def get_installed_apks():
dat = subprocess.check_output(["pm", "list", "packages", "-f"], encoding='utf8').strip().split("\n")
ret = {}
for x in dat:
if x.startswith("package:"):
v,k = x.split("package:")[1].split("=")
ret[k] = v
return ret
def install_apk(path):
# can only install from world readable path
install_path = "/sdcard/%s" % os.path.basename(path)
shutil.copyfile(path, install_path)
ret = subprocess.call(["pm", "install", "-r", install_path])
os.remove(install_path)
return ret == 0
def start_frame():
set_package_permissions()
system("am start -n ai.comma.plus.frame/.MainActivity")
def set_package_permissions():
pm_grant("ai.comma.plus.offroad", "android.permission.ACCESS_FINE_LOCATION")
appops_set("ai.comma.plus.offroad", "SU", "allow")
appops_set("ai.comma.plus.offroad", "WIFI_SCAN", "allow")
appops_set("ai.comma.plus.offroad", "READ_EXTERNAL_STORAGE", "allow")
appops_set("ai.comma.plus.offroad", "WRITE_EXTERNAL_STORAGE", "allow")
def appops_set(package, op, mode):
system(f"LD_LIBRARY_PATH= appops set {package} {op} {mode}")
def pm_grant(package, permission):
system(f"pm grant {package} {permission}")
def system(cmd):
try:
cloudlog.info("running %s" % cmd)
subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
cloudlog.event("running failed",
cmd=e.cmd,
output=e.output[-1024:],
returncode=e.returncode)
# *** external functions ***
def update_apks():
# install apks
installed = get_installed_apks()
install_apks = glob.glob(os.path.join(BASEDIR, "apk/*.apk"))
for apk in install_apks:
app = os.path.basename(apk)[:-4]
if app not in installed:
installed[app] = None
cloudlog.info("installed apks %s" % (str(installed), ))
for app in installed.keys():
apk_path = os.path.join(BASEDIR, "apk/"+app+".apk")
if not os.path.exists(apk_path):
continue
h1 = hashlib.sha1(open(apk_path, 'rb').read()).hexdigest()
h2 = None
if installed[app] is not None:
h2 = hashlib.sha1(open(installed[app], 'rb').read()).hexdigest()
cloudlog.info("comparing version of %s %s vs %s" % (app, h1, h2))
if h2 is None or h1 != h2:
cloudlog.info("installing %s" % app)
success = install_apk(apk_path)
if not success:
cloudlog.info("needing to uninstall %s" % app)
system("pm uninstall %s" % app)
success = install_apk(apk_path)
assert success
def pm_apply_packages(cmd):
for p in android_packages:
system("pm %s %s" % (cmd, p))
if __name__ == "__main__":
update_apks()
+20
View File
@@ -0,0 +1,20 @@
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
from Cython.Build import cythonize
from common.cython_hacks import BuildExtWithoutPlatformSuffix
sourcefiles = ['clock.pyx']
extra_compile_args = ["-std=c++11"]
setup(name='Common',
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
ext_modules=cythonize(
Extension(
"common_pyx",
language="c++",
sources=sourcefiles,
extra_compile_args=extra_compile_args,
)
),
nthreads=4,
)
-276
View File
@@ -1,276 +0,0 @@
import re
import os
import struct
import sys
import numbers
from collections import namedtuple, defaultdict
def int_or_float(s):
# return number, trying to maintain int format
if s.isdigit():
return int(s, 10)
else:
return float(s)
DBCSignal = namedtuple(
"DBCSignal", ["name", "start_bit", "size", "is_little_endian", "is_signed",
"factor", "offset", "tmin", "tmax", "units"])
class dbc():
def __init__(self, fn):
self.name, _ = os.path.splitext(os.path.basename(fn))
with open(fn, encoding="ascii") as f:
self.txt = f.readlines()
self._warned_addresses = set()
# regexps from https://github.com/ebroecker/canmatrix/blob/master/canmatrix/importdbc.py
bo_regexp = re.compile(r"^BO\_ (\w+) (\w+) *: (\w+) (\w+)")
sg_regexp = re.compile(r"^SG\_ (\w+) : (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] \"(.*)\" (.*)")
sgm_regexp = re.compile(r"^SG\_ (\w+) (\w+) *: (\d+)\|(\d+)@(\d+)([\+|\-]) \(([0-9.+\-eE]+),([0-9.+\-eE]+)\) \[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] \"(.*)\" (.*)")
val_regexp = re.compile(r"VAL\_ (\w+) (\w+) (\s*[-+]?[0-9]+\s+\".+?\"[^;]*)")
# A dictionary which maps message ids to tuples ((name, size), signals).
# name is the ASCII name of the message.
# size is the size of the message in bytes.
# signals is a list signals contained in the message.
# signals is a list of DBCSignal in order of increasing start_bit.
self.msgs = {}
# A dictionary which maps message ids to a list of tuples (signal name, definition value pairs)
self.def_vals = defaultdict(list)
# lookup to bit reverse each byte
self.bits_index = [(i & ~0b111) + ((-i-1) & 0b111) for i in range(64)]
for l in self.txt:
l = l.strip()
if l.startswith("BO_ "):
# new group
dat = bo_regexp.match(l)
if dat is None:
print("bad BO {0}".format(l))
name = dat.group(2)
size = int(dat.group(3))
ids = int(dat.group(1), 0) # could be hex
if ids in self.msgs:
sys.exit("Duplicate address detected %d %s" % (ids, self.name))
self.msgs[ids] = ((name, size), [])
if l.startswith("SG_ "):
# new signal
dat = sg_regexp.match(l)
go = 0
if dat is None:
dat = sgm_regexp.match(l)
go = 1
if dat is None:
print("bad SG {0}".format(l))
sgname = dat.group(1)
start_bit = int(dat.group(go+2))
signal_size = int(dat.group(go+3))
is_little_endian = int(dat.group(go+4))==1
is_signed = dat.group(go+5)=='-'
factor = int_or_float(dat.group(go+6))
offset = int_or_float(dat.group(go+7))
tmin = int_or_float(dat.group(go+8))
tmax = int_or_float(dat.group(go+9))
units = dat.group(go+10)
self.msgs[ids][1].append(
DBCSignal(sgname, start_bit, signal_size, is_little_endian,
is_signed, factor, offset, tmin, tmax, units))
if l.startswith("VAL_ "):
# new signal value/definition
dat = val_regexp.match(l)
if dat is None:
print("bad VAL {0}".format(l))
ids = int(dat.group(1), 0) # could be hex
sgname = dat.group(2)
defvals = dat.group(3)
defvals = defvals.replace("?",r"\?") #escape sequence in C++
defvals = defvals.split('"')[:-1]
# convert strings to UPPER_CASE_WITH_UNDERSCORES
defvals[1::2] = [d.strip().upper().replace(" ","_") for d in defvals[1::2]]
defvals = '"'+"".join(str(i) for i in defvals)+'"'
self.def_vals[ids].append((sgname, defvals))
for msg in self.msgs.values():
msg[1].sort(key=lambda x: x.start_bit)
self.msg_name_to_address = {}
for address, m in self.msgs.items():
name = m[0][0]
self.msg_name_to_address[name] = address
def lookup_msg_id(self, msg_id):
if not isinstance(msg_id, numbers.Number):
msg_id = self.msg_name_to_address[msg_id]
return msg_id
def reverse_bytes(self, x):
return ((x & 0xff00000000000000) >> 56) | \
((x & 0x00ff000000000000) >> 40) | \
((x & 0x0000ff0000000000) >> 24) | \
((x & 0x000000ff00000000) >> 8) | \
((x & 0x00000000ff000000) << 8) | \
((x & 0x0000000000ff0000) << 24) | \
((x & 0x000000000000ff00) << 40) | \
((x & 0x00000000000000ff) << 56)
def encode(self, msg_id, dd):
"""Encode a CAN message using the dbc.
Inputs:
msg_id: The message ID.
dd: A dictionary mapping signal name to signal data.
"""
msg_id = self.lookup_msg_id(msg_id)
msg_def = self.msgs[msg_id]
size = msg_def[0][1]
result = 0
for s in msg_def[1]:
ival = dd.get(s.name)
if ival is not None:
ival = (ival / s.factor) - s.offset
ival = int(round(ival))
if s.is_signed and ival < 0:
ival = (1 << s.size) + ival
if s.is_little_endian:
shift = s.start_bit
else:
b1 = (s.start_bit // 8) * 8 + (-s.start_bit - 1) % 8
shift = 64 - (b1 + s.size)
mask = ((1 << s.size) - 1) << shift
dat = (ival & ((1 << s.size) - 1)) << shift
if s.is_little_endian:
mask = self.reverse_bytes(mask)
dat = self.reverse_bytes(dat)
result &= ~mask
result |= dat
result = struct.pack('>Q', result)
return result[:size]
def decode(self, x, arr=None, debug=False):
"""Decode a CAN message using the dbc.
Inputs:
x: A collection with elements (address, time, data), where address is
the CAN address, time is the bus time, and data is the CAN data as a
hex string.
arr: Optional list of signals which should be decoded and returned.
debug: True to print debugging statements.
Returns:
A tuple (name, data), where name is the name of the CAN message and data
is the decoded result. If arr is None, data is a dict of properties.
Otherwise data is a list of the same length as arr.
Returns (None, None) if the message could not be decoded.
"""
if arr is None:
out = {}
else:
out = [None]*len(arr)
msg = self.msgs.get(x[0])
if msg is None:
if x[0] not in self._warned_addresses:
#print("WARNING: Unknown message address {}".format(x[0]))
self._warned_addresses.add(x[0])
return None, None
name = msg[0][0]
if debug:
print(name)
st = x[2].ljust(8, b'\x00')
le, be = None, None
for s in msg[1]:
if arr is not None and s[0] not in arr:
continue
start_bit = s[1]
signal_size = s[2]
little_endian = s[3]
signed = s[4]
factor = s[5]
offset = s[6]
if little_endian:
if le is None:
le = struct.unpack("<Q", st)[0]
tmp = le
shift_amount = start_bit
else:
if be is None:
be = struct.unpack(">Q", st)[0]
tmp = be
b1 = (start_bit // 8) * 8 + (-start_bit - 1) % 8
shift_amount = 64 - (b1 + signal_size)
if shift_amount < 0:
continue
tmp = (tmp >> shift_amount) & ((1 << signal_size) - 1)
if signed and (tmp >> (signal_size - 1)):
tmp -= (1 << signal_size)
tmp = tmp * factor + offset
# if debug:
# print("%40s %2d %2d %7.2f %s" % (s[0], s[1], s[2], tmp, s[-1]))
if arr is None:
out[s[0]] = tmp
else:
out[arr.index(s[0])] = tmp
return name, out
def get_signals(self, msg):
msg = self.lookup_msg_id(msg)
return [sgs.name for sgs in self.msgs[msg][1]]
if __name__ == "__main__":
from opendbc import DBC_PATH
import numpy as np
dbc_test = dbc(os.path.join(DBC_PATH, 'toyota_prius_2017_pt_generated.dbc'))
msg = ('STEER_ANGLE_SENSOR', {'STEER_ANGLE': -6.0, 'STEER_RATE': 4, 'STEER_FRACTION': -0.2})
encoded = dbc_test.encode(*msg)
decoded = dbc_test.decode((0x25, 0, encoded))
assert decoded == msg
dbc_test = dbc(os.path.join(DBC_PATH, 'hyundai_santa_fe_2019_ccan.dbc'))
decoded = dbc_test.decode((0x2b0, 0, "\xfa\xfe\x00\x07\x12"))
assert np.isclose(decoded[1]['SAS_Angle'], -26.2)
msg = ('SAS11', {'SAS_Stat': 7.0, 'MsgCount': 0.0, 'SAS_Angle': -26.200000000000003, 'SAS_Speed': 0.0, 'CheckSum': 0.0})
encoded = dbc_test.encode(*msg)
decoded = dbc_test.decode((0x2b0, 0, encoded))
assert decoded == msg
-10
View File
@@ -1,10 +0,0 @@
all: simple_kalman_impl.so
simple_kalman_impl.so: simple_kalman_impl.pyx simple_kalman_impl.pxd simple_kalman_setup.py
python3 simple_kalman_setup.py build_ext --inplace
rm -rf build
rm simple_kalman_impl.c
.PHONY: clean
clean:
rm -f simple_kalman_impl.so
+6
View File
@@ -0,0 +1,6 @@
Import('env')
env.Command(['simple_kalman_impl.so'],
['simple_kalman_impl.pyx', 'simple_kalman_impl.pxd', 'simple_kalman_setup.py'],
"cd common/kalman && python3 simple_kalman_setup.py build_ext --inplace")
+1 -7
View File
@@ -1,9 +1,3 @@
# pylint: skip-file
import os
import subprocess
kalman_dir = os.path.dirname(os.path.abspath(__file__))
subprocess.check_call(["make", "simple_kalman_impl.so"], cwd=kalman_dir)
from .simple_kalman_impl import KF1D as KF1D
from common.kalman.simple_kalman_impl import KF1D as KF1D
assert KF1D
+1 -1
View File
@@ -1,4 +1,4 @@
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
from distutils.core import Extension, setup
from Cython.Build import cythonize
+4 -1
View File
@@ -55,6 +55,7 @@ keys = {
"CalibrationParams": [TxType.PERSISTENT],
"CarParams": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
"CarVin": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
"CommunityFeaturesToggle": [TxType.PERSISTENT],
"CompletedTrainingVersion": [TxType.PERSISTENT],
"ControlsParams": [TxType.PERSISTENT],
"DoUninstall": [TxType.CLEAR_ON_MANAGER_START],
@@ -65,12 +66,13 @@ keys = {
"GithubSshKeys": [TxType.PERSISTENT],
"HasAcceptedTerms": [TxType.PERSISTENT],
"HasCompletedSetup": [TxType.PERSISTENT],
"IsLdwEnabled": [TxType.PERSISTENT],
"IsGeofenceEnabled": [TxType.PERSISTENT],
"IsMetric": [TxType.PERSISTENT],
"IsRHD": [TxType.PERSISTENT],
"IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
"IsUpdateAvailable": [TxType.PERSISTENT],
"IsUploadRawEnabled": [TxType.PERSISTENT],
"IsUploadVideoOverCellularEnabled": [TxType.PERSISTENT],
"LastUpdateTime": [TxType.PERSISTENT],
"LimitSetSpeed": [TxType.PERSISTENT],
"LimitSetSpeedNeural": [TxType.PERSISTENT],
@@ -95,6 +97,7 @@ keys = {
"Offroad_TemperatureTooHigh": [TxType.CLEAR_ON_MANAGER_START],
"Offroad_PandaFirmwareMismatch": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
"Offroad_InvalidTime": [TxType.CLEAR_ON_MANAGER_START],
"Offroad_IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
}
+1 -9
View File
@@ -6,20 +6,12 @@ import subprocess
import multiprocessing
from cffi import FFI
# Build and load cython module
import pyximport
installer = pyximport.install(inplace=True, build_dir='/tmp')
from common.clock import monotonic_time, sec_since_boot # pylint: disable=no-name-in-module, import-error
pyximport.uninstall(*installer)
assert monotonic_time
assert sec_since_boot
from common.common_pyx import sec_since_boot # pylint: disable=no-name-in-module, import-error
# time step for each process
DT_CTRL = 0.01 # controlsd
DT_PLAN = 0.05 # mpc
DT_MDL = 0.05 # model
DT_RDR = 0.05 # radar
DT_DMON = 0.1 # driver monitoring
DT_TRML = 0.5 # thermald and manager
+30 -5
View File
@@ -2,22 +2,46 @@ import os
import subprocess
from common.basedir import BASEDIR
class Spinner():
def __enter__(self):
def __init__(self):
self.spinner_proc = subprocess.Popen(["./spinner"],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "selfdrive", "ui", "spinner"),
close_fds=True)
def __enter__(self):
return self
def update(self, spinner_text):
self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
self.spinner_proc.stdin.flush()
def __exit__(self, type, value, traceback):
self.spinner_proc.stdin.close()
self.spinner_proc.terminate()
def close(self):
if self.spinner_proc is not None:
self.spinner_proc.stdin.close()
self.spinner_proc.terminate()
self.spinner_proc = None
def __del__(self):
self.close()
def __exit__(self, type, value, traceback):
self.close()
class FakeSpinner():
def __init__(self):
pass
def __enter__(self):
return self
def update(self, _):
pass
def __exit__(self, type, value, traceback):
pass
if __name__ == "__main__":
@@ -25,4 +49,5 @@ if __name__ == "__main__":
with Spinner() as s:
s.update("Spinner text")
time.sleep(5.0)
print("gone")
time.sleep(5.0)
+3 -3
View File
@@ -125,7 +125,7 @@ def img_from_device(pt_device):
#TODO please use generic img transform below
def rotate_img(img, eulers, crop=None, intrinsics=eon_intrinsics):
import cv2 # pylint: disable=no-name-in-module, import-error
import cv2 # pylint: disable=import-error
size = img.shape[:2]
rot = orient.rot_from_euler(eulers)
@@ -183,7 +183,7 @@ def transform_img(base_img,
alpha=1.0,
beta=0,
blur=0):
import cv2 # pylint: disable=no-name-in-module, import-error
import cv2 # pylint: disable=import-error
cv2.setNumThreads(1)
if yuv:
@@ -241,7 +241,7 @@ def transform_img(base_img,
def yuv_crop(frame, output_size, center=None):
# output_size in camera coordinates so u,v
# center in array coordinates so row, column
import cv2 # pylint: disable=no-name-in-module, import-error
import cv2 # pylint: disable=import-error
rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
if not center:
center = (rgb.shape[0]/2, rgb.shape[1]/2)