manager runs on Mac, and other openpilot for PC fixes (#1037)

* use the openpilot/persist directory on PC

* manager runs on mac

* sim runs w/o carla

* fix params location in test

* that rmtree can fail and it's okay

* refactor params clear functionality

* set PARAMS_PATH

selfdrive/athena/athenad.py

@@ -19,6 +19,7 @@ from selfdrive.loggerd.config import ROOT
 
 import cereal.messaging as messaging
 from common import android
+from common.basedir import PERSIST
 from common.api import Api
 from common.params import Params
 from cereal.services import service_list
@@ -188,10 +189,10 @@ def startLocalProxy(global_end_event, remote_ws_uri, local_port):
 
 @dispatcher.add_method
 def getPublicKey():
-  if not os.path.isfile('/persist/comma/id_rsa.pub'):
+  if not os.path.isfile(PERSIST+'/comma/id_rsa.pub'):
     return None
 
-  with open('/persist/comma/id_rsa.pub', 'r') as f:
+  with open(PERSIST+'/comma/id_rsa.pub', 'r') as f:
     return f.read()
 
 @dispatcher.add_method

selfdrive/controls/lib/cluster/fastcluster_py.py

@@ -2,9 +2,10 @@ import os
 import numpy as np
 
 from cffi import FFI
+from common.ffi_wrapper import suffix
 
 cluster_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
-cluster_fn = os.path.join(cluster_dir, "libfastcluster.so")
+cluster_fn = os.path.join(cluster_dir, "libfastcluster"+suffix())
 
 ffi = FFI()
 ffi.cdef("""

selfdrive/controls/lib/lateral_mpc/libmpc_py.py

@@ -1,9 +1,10 @@
 import os
 
 from cffi import FFI
+from common.ffi_wrapper import suffix
 
 mpc_dir = os.path.dirname(os.path.abspath(__file__))
-libmpc_fn = os.path.join(mpc_dir, "libmpc.so")
+libmpc_fn = os.path.join(mpc_dir, "libmpc"+suffix())
 
 ffi = FFI()
 ffi.cdef("""

selfdrive/controls/lib/longitudinal_mpc/libmpc_py.py

@@ -1,11 +1,12 @@
 import os
 
 from cffi import FFI
+from common.ffi_wrapper import suffix
 
 mpc_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
 
 def _get_libmpc(mpc_id):
-    libmpc_fn = os.path.join(mpc_dir, "libmpc%d.so" % mpc_id)
+    libmpc_fn = os.path.join(mpc_dir, "libmpc%d%s" % (mpc_id, suffix()))
 
     ffi = FFI()
     ffi.cdef("""

selfdrive/manager.py

@@ -9,7 +9,7 @@ import shutil
 import subprocess
 import datetime
 
-from common.basedir import BASEDIR
+from common.basedir import BASEDIR, PARAMS
 from common.android import ANDROID
 sys.path.append(os.path.join(BASEDIR, "pyextra"))
 os.environ['BASEDIR'] = BASEDIR
@@ -22,6 +22,8 @@ try:
   os.mkdir("/dev/shm")
 except FileExistsError:
   pass
+except PermissionError:
+  print("WARNING: failed to make /dev/shm")
 
 if ANDROID:
   os.chmod("/dev/shm", 0o777)
@@ -365,9 +367,10 @@ def manager_init(should_register=True):
     pass
 
   # ensure shared libraries are readable by apks
-  os.chmod(BASEDIR, 0o755)
-  os.chmod(os.path.join(BASEDIR, "cereal"), 0o755)
-  os.chmod(os.path.join(BASEDIR, "cereal", "libmessaging_shared.so"), 0o755)
+  if ANDROID:
+    os.chmod(BASEDIR, 0o755)
+    os.chmod(os.path.join(BASEDIR, "cereal"), 0o755)
+    os.chmod(os.path.join(BASEDIR, "cereal", "libmessaging_shared.so"), 0o755)
 
 def manager_thread():
   # now loop
@@ -454,6 +457,8 @@ def uninstall():
   android.reboot(reason="recovery")
 
 def main():
+  os.environ['PARAMS_PATH'] = PARAMS
+
   # the flippening!
   os.system('LD_LIBRARY_PATH="" content insert --uri content://settings/system --bind name:s:user_rotation --bind value:i:1')
 

selfdrive/registration.py

@@ -8,6 +8,7 @@ from common.android import get_imei, get_serial, get_subscriber_info
 from common.api import api_get
 from common.params import Params
 from common.file_helpers import mkdirs_exists_ok
+from common.basedir import PERSIST
 
 def register():
   params = Params()
@@ -22,24 +23,24 @@ def register():
   # create a key for auth
   # your private key is kept on your device persist partition and never sent to our servers
   # do not erase your persist partition
-  if not os.path.isfile("/persist/comma/id_rsa.pub"):
+  if not os.path.isfile(PERSIST+"/comma/id_rsa.pub"):
     cloudlog.warning("generating your personal RSA key")
-    mkdirs_exists_ok("/persist/comma")
-    assert os.system("openssl genrsa -out /persist/comma/id_rsa.tmp 2048") == 0
-    assert os.system("openssl rsa -in /persist/comma/id_rsa.tmp -pubout -out /persist/comma/id_rsa.tmp.pub") == 0
-    os.rename("/persist/comma/id_rsa.tmp", "/persist/comma/id_rsa")
-    os.rename("/persist/comma/id_rsa.tmp.pub", "/persist/comma/id_rsa.pub")
+    mkdirs_exists_ok(PERSIST+"/comma")
+    assert os.system("openssl genrsa -out "+PERSIST+"/comma/id_rsa.tmp 2048") == 0
+    assert os.system("openssl rsa -in "+PERSIST+"/comma/id_rsa.tmp -pubout -out "+PERSIST+"/comma/id_rsa.tmp.pub") == 0
+    os.rename(PERSIST+"/comma/id_rsa.tmp", PERSIST+"/comma/id_rsa")
+    os.rename(PERSIST+"/comma/id_rsa.tmp.pub", PERSIST+"/comma/id_rsa.pub")
 
   # make key readable by app users (ai.comma.plus.offroad)
-  os.chmod('/persist/comma/', 0o755)
-  os.chmod('/persist/comma/id_rsa', 0o744)
+  os.chmod(PERSIST+'/comma/', 0o755)
+  os.chmod(PERSIST+'/comma/id_rsa', 0o744)
 
   dongle_id, access_token = params.get("DongleId", encoding='utf8'), params.get("AccessToken", encoding='utf8')
-  public_key = open("/persist/comma/id_rsa.pub").read()
+  public_key = open(PERSIST+"/comma/id_rsa.pub").read()
 
   # create registration token
   # in the future, this key will make JWTs directly
-  private_key = open("/persist/comma/id_rsa").read()
+  private_key = open(PERSIST+"/comma/id_rsa").read()
 
   # late import
   import jwt

selfdrive/test/longitudinal_maneuvers/test_longitudinal.py

@@ -4,7 +4,6 @@ os.environ['OLD_CAN'] = '1'
 os.environ['NOCRASH'] = '1'
 
 import unittest
-import shutil
 import matplotlib
 matplotlib.use('svg')
 
@@ -328,8 +327,8 @@ class LongitudinalControl(unittest.TestCase):
 
     setup_output()
 
-    shutil.rmtree('/data/params', ignore_errors=True)
     params = Params()
+    params.clear_all()
     params.put("Passive", "1" if os.getenv("PASSIVE") else "0")
     params.put("OpenpilotEnabledToggle", "1")
     params.put("CommunityFeaturesToggle", "1")

selfdrive/test/process_replay/process_replay.py

@@ -3,7 +3,6 @@ import os
 import sys
 import threading
 import importlib
-import shutil
 
 if "CI" in os.environ:
   tqdm = lambda x: x
@@ -260,8 +259,8 @@ def replay_process(cfg, lr):
   all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
   pub_msgs = [msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())]
 
-  shutil.rmtree('/data/params', ignore_errors=True)
   params = Params()
+  params.clear_all()
   params.manager_start()
   params.put("OpenpilotEnabledToggle", "1")
   params.put("Passive", "0")

selfdrive/test/test_car_models.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import shutil
 import time
 import os
 import sys
@@ -487,8 +486,8 @@ if __name__ == "__main__":
     elif "UNLOGGER_PATH" not in os.environ:
       continue
 
-    shutil.rmtree('/data/params')
     params = Params()
+    params.clear_all()
     params.manager_start()
     params.put("OpenpilotEnabledToggle", "1")
     params.put("CommunityFeaturesToggle", "1")