manager cleanup (#2634)

* start cleanup * add uninstall to hw abstraction layer * cleanup * litte more * don't build at import time * fix tests * don't build by default * sync old-commit-hash: 220853730d1d3163f5534ed5fd31ec5c30ad129f
2026-07-14 05:42:13 +08:00 · 2020-12-17 11:45:41 -08:00
parent 34e98ac4d7
commit c4bff3f93b
6 changed files with 82 additions and 86 deletions
@@ -124,7 +124,7 @@ pipeline {
                stage('Replay Tests') {
                  steps {
                    phone_steps("eon2", [
-                      ["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"],
+                      ["camerad/modeld replay", "QCOM_REPLAY=1 scons -j4 && cd selfdrive/test/process_replay && ./camera_replay.py"],
                    ])
                  }
                }
@@ -132,7 +132,7 @@ pipeline {
                stage('HW + Unit Tests') {
                  steps {
                    phone_steps("eon", [
-                      ["build cereal", "SCONS_CACHE=1 scons -j4 cereal/"],
+                      ["build", "SCONS_CACHE=1 scons -j4"],
                      ["test sounds", "nosetests -s selfdrive/test/test_sounds.py"],
                      ["test boardd loopback", "nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"],
                      ["test loggerd", "CI=1 python selfdrive/loggerd/tests/test_loggerd.py"],
@@ -16,6 +16,14 @@ class HardwareBase:
    except Exception:
      return default

+  @abstractmethod
+  def reboot(self, reason=None):
+    pass
+
+  @abstractmethod
+  def uninstall(self):
+    pass
+
  @abstractmethod
  def get_sound_card_online(self):
    pass
@@ -32,10 +40,6 @@ class HardwareBase:
  def get_subscriber_info(self):
    pass

-  @abstractmethod
-  def reboot(self, reason=None):
-    pass
-
  @abstractmethod
  def get_network_type(self):
    pass
@@ -98,6 +98,12 @@ class Android(HardwareBase):
      "i32", "1"  # wait
    ])

+  def uninstall(self):
+    with open('/cache/recovery/command', 'w') as f:
+      f.write('--wipe_data\n')
+    # IPowerManager.reboot(confirm=false, reason="recovery", wait=true)
+    self.reboot(reason="recovery")
+
  def get_sim_info(self):
    # Used for athena
    # TODO: build using methods from this class
@@ -11,6 +11,12 @@ class Pc(HardwareBase):
  def get_sound_card_online(self):
    return True

+  def reboot(self, reason=None):
+    print("REBOOT!")
+
+  def uninstall(self):
+    print("uninstall")
+
  def get_imei(self, slot):
    return "%015d" % random.randint(0, 1 << 32)

@@ -20,9 +26,6 @@ class Pc(HardwareBase):
  def get_subscriber_info(self):
    return ""

-  def reboot(self, reason=None):
-    print("REBOOT!")
-
  def get_network_type(self):
    return NetworkType.wifi

@@ -35,12 +35,16 @@ class Tici(HardwareBase):
  def get_sound_card_online(self):
    return True

-  def get_serial(self):
-    return self.get_cmdline()['androidboot.serialno']
-
  def reboot(self, reason=None):
    subprocess.check_output(["sudo", "reboot"])

+  def uninstall(self):
+    # TODO: implement uninstall. reboot to factory reset?
+    pass
+
+  def get_serial(self):
+    return self.get_cmdline()['androidboot.serialno']
+
  def get_network_type(self):
    primary_connection = self.nm.Get(NM, 'PrimaryConnection', dbus_interface=DBUS_PROPS)
    primary_connection = self.bus.get_object(NM, primary_connection)
@@ -1,37 +1,33 @@
 #!/usr/bin/env python3
+import datetime
+import importlib
 import os
-import time
 import sys
 import fcntl
 import errno
 import signal
 import shutil
 import subprocess
-import datetime
 import textwrap
-from typing import Dict, List
-from selfdrive.swaglog import cloudlog, add_logentries_handler
+import time
+import traceback

+from multiprocessing import Process
+from typing import Dict, List

 from common.basedir import BASEDIR
+from common.spinner import Spinner
+from common.text_window import TextWindow
 from selfdrive.hardware import HARDWARE, EON, PC
-WEBCAM = os.getenv("WEBCAM") is not None
-sys.path.append(os.path.join(BASEDIR, "pyextra"))
+from selfdrive.swaglog import cloudlog, add_logentries_handler
+
 os.environ['BASEDIR'] = BASEDIR
+sys.path.append(os.path.join(BASEDIR, "pyextra"))

 TOTAL_SCONS_NODES = 1040
-prebuilt = os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
+WEBCAM = os.getenv("WEBCAM") is not None
+PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt'))

-# Create folders needed for msgq
-try:
-  os.mkdir("/dev/shm")
-except FileExistsError:
-  pass
-except PermissionError:
-  print("WARNING: failed to make /dev/shm")
-
-if EON:
-  os.chmod("/dev/shm", 0o777)

 def unblock_stdout():
  # get a non-blocking stdout
@@ -42,8 +38,7 @@ def unblock_stdout():
    signal.signal(signal.SIGINT, lambda signum, frame: os.kill(child_pid, signal.SIGINT))
    signal.signal(signal.SIGTERM, lambda signum, frame: os.kill(child_pid, signal.SIGTERM))

-    fcntl.fcntl(sys.stdout, fcntl.F_SETFL,
-       fcntl.fcntl(sys.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+    fcntl.fcntl(sys.stdout, fcntl.F_SETFL, fcntl.fcntl(sys.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)

    while True:
      try:
@@ -70,12 +65,6 @@ def unblock_stdout():
 if __name__ == "__main__":
  unblock_stdout()

-from common.spinner import Spinner
-from common.text_window import TextWindow
-
-import importlib
-import traceback
-from multiprocessing import Process

 # Run scons
 spinner = Spinner()
@@ -83,7 +72,7 @@ spinner.update("0")
 if __name__ != "__main__":
  spinner.close()

-if not prebuilt:
+def build():
  for retry in [True, False]:
    # run scons
    env = os.environ.copy()
@@ -91,7 +80,7 @@ if not prebuilt:
    env['SCONS_CACHE'] = "1"

    nproc = os.cpu_count()
-    j_flag = "" if nproc is None else "-j%d" % (nproc - 1)
+    j_flag = "" if nproc is None else f"-j{nproc - 1}"
    scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)

    compile_output = []
@@ -148,7 +137,9 @@ if not prebuilt:
    else:
      break

-import cereal
+if __name__ == "__main__" and not PREBUILT:
+  build()
+
 import cereal.messaging as messaging

 from common.params import Params
@@ -159,7 +150,6 @@ from selfdrive.loggerd.config import ROOT
 from selfdrive.launcher import launcher
 from selfdrive.hardware.eon.apk import update_apks, pm_apply_packages, start_offroad

-ThermalStatus = cereal.log.ThermalData.ThermalStatus

 # comment out anything you don't want to run
 managed_processes = {
@@ -245,13 +235,7 @@ driver_view_processes = [
  'dmonitoringmodeld'
 ]

-if WEBCAM:
-  car_started_processes += [
-    'dmonitoringd',
-    'dmonitoringmodeld',
-  ]
-
-if not PC:
+if not PC or WEBCAM:
  car_started_processes += [
    'ubloxd',
    'dmonitoringd',
@@ -267,7 +251,6 @@ if EON:

 def register_managed_process(name, desc, car_started=False):
  global managed_processes, car_started_processes, persistent_processes
-  print("registering %s" % name)
  managed_processes[name] = desc
  if car_started:
    car_started_processes.append(name)
@@ -323,22 +306,22 @@ def start_daemon_process(name):

  params.put(pid_param, str(proc.pid))

-def prepare_managed_process(p):
+def prepare_managed_process(p, build=False):
  proc = managed_processes[p]
  if isinstance(proc, str):
    # import this python
    cloudlog.info("preimporting %s" % proc)
    importlib.import_module(proc)
-  elif os.path.isfile(os.path.join(BASEDIR, proc[0], "Makefile")):
+  elif os.path.isfile(os.path.join(BASEDIR, proc[0], "SConscript")) and build:
    # build this process
    cloudlog.info("building %s" % (proc,))
    try:
-      subprocess.check_call(["make", "-j4"], cwd=os.path.join(BASEDIR, proc[0]))
+      subprocess.check_call(["scons", "u", "-j4", "."], cwd=os.path.join(BASEDIR, proc[0]))
    except subprocess.CalledProcessError:
-      # make clean if the build failed
-      cloudlog.warning("building %s failed, make clean" % (proc, ))
-      subprocess.check_call(["make", "clean"], cwd=os.path.join(BASEDIR, proc[0]))
-      subprocess.check_call(["make", "-j4"], cwd=os.path.join(BASEDIR, proc[0]))
+      # clean and retry if the build failed
+      cloudlog.warning("building %s failed, cleaning and retrying" % (proc, ))
+      subprocess.check_call(["scons", "-u", "-c", "."], cwd=os.path.join(BASEDIR, proc[0]))
+      subprocess.check_call(["scons", "-u", "-j4", "."], cwd=os.path.join(BASEDIR, proc[0]))


 def join_process(process, timeout):
@@ -404,21 +387,23 @@ def send_managed_process_signal(name, sig):

 # ****************** run loop ******************

-def manager_init(should_register=True):
-  if should_register:
-    reg_res = register()
-    if reg_res:
-      dongle_id = reg_res
-    else:
-      raise Exception("server registration failed")
-  else:
-    dongle_id = "c"*16
+def manager_init():
+  # Create folders needed for msgq
+  try:
+    os.mkdir("/dev/shm")
+  except FileExistsError:
+    pass
+  except PermissionError:
+    print("WARNING: failed to make /dev/shm")

  # set dongle id
-  cloudlog.info("dongle id is " + dongle_id)
+  reg_res = register()
+  if reg_res:
+    dongle_id = reg_res
+  else:
+    raise Exception("server registration failed")
  os.environ['DONGLE_ID'] = dongle_id

-  cloudlog.info("dirty is %d" % dirty)
  if not dirty:
    os.environ['CLEAN'] = '1'

@@ -435,12 +420,11 @@ def manager_init(should_register=True):
  # ensure shared libraries are readable by apks
  if EON:
    os.chmod(BASEDIR, 0o755)
+    os.chmod("/dev/shm", 0o777)
    os.chmod(os.path.join(BASEDIR, "cereal"), 0o755)
    os.chmod(os.path.join(BASEDIR, "cereal", "libmessaging_shared.so"), 0o755)

 def manager_thread():
-  # now loop
-  thermal_sock = messaging.sub_sock('thermal')

  cloudlog.info("manager start")
  cloudlog.info({"environ": os.environ})
@@ -448,8 +432,6 @@ def manager_thread():
  # save boot log
  subprocess.call(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd"))

-  params = Params()
-
  # start daemon processes
  for p in daemon_processes:
    start_daemon_process(p)
@@ -472,6 +454,8 @@ def manager_thread():

  started_prev = False
  logger_dead = False
+  params = Params()
+  thermal_sock = messaging.sub_sock('thermal')

  while 1:
    msg = messaging.recv_sock(thermal_sock, wait=True)
@@ -502,6 +486,7 @@ def manager_thread():

      # trigger an update after going offroad
      if started_prev:
+        os.sync()
        send_managed_process_signal("updated", signal.SIGHUP)

    started_prev = msg.thermal.started
@@ -514,25 +499,18 @@ def manager_thread():
    if params.get("DoUninstall", encoding='utf8') == "1":
      break

-def manager_prepare(spinner=None):
+def manager_prepare():
  # build all processes
  os.chdir(os.path.dirname(os.path.abspath(__file__)))

  # Spinner has to start from 70 here
-  total = 100.0 if prebuilt else 30.0
+  total = 100.0 if PREBUILT else 30.0

  for i, p in enumerate(managed_processes):
-    if spinner is not None:
-      spinner.update("%d" % ((100.0 - total) + total * (i + 1) / len(managed_processes),))
+    perc = (100.0 - total) + total * (i + 1) / len(managed_processes)
+    spinner.update(str(int(perc)))
    prepare_managed_process(p)

-def uninstall():
-  cloudlog.warning("uninstalling")
-  with open('/cache/recovery/command', 'w') as f:
-    f.write('--wipe_data\n')
-  # IPowerManager.reboot(confirm=false, reason="recovery", wait=true)
-  HARDWARE.reboot(reason="recovery")
-
 def main():
  params = Params()
  params.manager_start()
@@ -558,7 +536,7 @@ def main():
    if params.get(k) is None:
      params.put(k, v)

-  # is this chffrplus?
+  # is this dashcam?
  if os.getenv("PASSIVE") is not None:
    params.put("Passive", str(int(os.getenv("PASSIVE"))))

@@ -568,7 +546,7 @@ def main():
  if EON:
    update_apks()
  manager_init()
-  manager_prepare(spinner)
+  manager_prepare()
  spinner.close()

  if os.getenv("PREPAREONLY") is not None:
@@ -586,7 +564,8 @@ def main():
    cleanup_all_processes(None, None)

  if params.get("DoUninstall", encoding='utf8') == "1":
-    uninstall()
+    cloudlog.warning("uninstalling")
+    HARDWARE.uninstall()


 if __name__ == "__main__":
@@ -598,7 +577,7 @@ if __name__ == "__main__":

    # Show last 3 lines of traceback
    error = traceback.format_exc(-3)
-    error = "Manager failed to start\n \n" + error
+    error = "Manager failed to start\n\n" + error
    spinner.close()
    with TextWindow(error) as t:
      t.wait_for_exit()