openpilot v0.7.8 release

selfdrive/updated.py

@@ -26,48 +26,59 @@ import os
 import datetime
 import subprocess
 import psutil
-from stat import S_ISREG, S_ISDIR, S_ISLNK, S_IMODE, ST_MODE, ST_INO, ST_UID, ST_GID, ST_ATIME, ST_MTIME
 import shutil
 import signal
-from pathlib import Path
 import fcntl
-import threading
 import time
+import threading
 from cffi import FFI
+from pathlib import Path
 
 from common.basedir import BASEDIR
 from common.params import Params
 from selfdrive.swaglog import cloudlog
+from selfdrive.controls.lib.alertmanager import set_offroad_alert
 
-STAGING_ROOT = "/data/safe_staging"
+TEST_IP = os.getenv("UPDATER_TEST_IP", "8.8.8.8")
+LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
+STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
+
+NEOS_VERSION = os.getenv("UPDATER_NEOS_VERSION", "/VERSION")
+NEOSUPDATE_DIR = os.getenv("UPDATER_NEOSUPDATE_DIR", "/data/neoupdate")
 
 OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper")
 OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata")
 OVERLAY_MERGED = os.path.join(STAGING_ROOT, "merged")
 FINALIZED = os.path.join(STAGING_ROOT, "finalized")
 
-NICE_LOW_PRIORITY = ["nice", "-n", "19"]
-SHORT = os.getenv("SHORT") is not None
 
-# Workaround for the EON/termux build of Python having os.link removed.
+# Workaround for lack of os.link in the NEOS/termux python
 ffi = FFI()
 ffi.cdef("int link(const char *oldpath, const char *newpath);")
 libc = ffi.dlopen(None)
+def link(src, dest):
+  return libc.link(src.encode(), dest.encode())
 
 
 class WaitTimeHelper:
-  ready_event = threading.Event()
-  shutdown = False
-
-  def __init__(self):
+  def __init__(self, proc):
+    self.proc = proc
+    self.ready_event = threading.Event()
+    self.shutdown = False
     signal.signal(signal.SIGTERM, self.graceful_shutdown)
     signal.signal(signal.SIGINT, self.graceful_shutdown)
     signal.signal(signal.SIGHUP, self.update_now)
 
   def graceful_shutdown(self, signum, frame):
-    # umount -f doesn't appear effective in avoiding "device busy" on EON,
+    # umount -f doesn't appear effective in avoiding "device busy" on NEOS,
     # so don't actually die until the next convenient opportunity in main().
     cloudlog.info("caught SIGINT/SIGTERM, dismounting overlay at next opportunity")
+
+    # forward the signal to all our child processes
+    child_procs = self.proc.children(recursive=True)
+    for p in child_procs:
+      p.send_signal(signum)
+
     self.shutdown = True
     self.ready_event.set()
 
@@ -75,42 +86,27 @@ class WaitTimeHelper:
     cloudlog.info("caught SIGHUP, running update check immediately")
     self.ready_event.set()
 
-
-def wait_between_updates(ready_event):
-  ready_event.clear()
-  if SHORT:
-    ready_event.wait(timeout=10)
-  else:
-    ready_event.wait(timeout=60 * 10)
+  def sleep(self, t):
+    self.ready_event.wait(timeout=t)
 
 
-def link(src, dest):
-  # Workaround for the EON/termux build of Python having os.link removed.
-  return libc.link(src.encode(), dest.encode())
-
-
-def run(cmd, cwd=None):
+def run(cmd, cwd=None, low_priority=False):
+  if low_priority:
+    cmd = ["nice", "-n", "19"] + cmd
   return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')
 
 
-def remove_consistent_flag():
+def set_consistent_flag(consistent):
   os.system("sync")
   consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
-  try:
+  if consistent:
+    consistent_file.touch()
+  elif not consistent and consistent_file.exists():
     consistent_file.unlink()
-  except FileNotFoundError:
-    pass
   os.system("sync")
 
 
-def set_consistent_flag():
-  consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
-  os.system("sync")
-  consistent_file.touch()
-  os.system("sync")
-
-
-def set_update_available_params(new_version=False):
+def set_update_available_params(new_version):
   params = Params()
 
   t = datetime.datetime.utcnow().isoformat()
@@ -134,39 +130,35 @@ def dismount_ovfs():
 
 
 def setup_git_options(cwd):
-  # We sync FS object atimes (which EON doesn't use) and mtimes, but ctimes
+  # We sync FS object atimes (which NEOS doesn't use) and mtimes, but ctimes
   # are outside user control. Make sure Git is set up to ignore system ctimes,
   # because they change when we make hard links during finalize. Otherwise,
   # there is a lot of unnecessary churn. This appears to be a common need on
   # OSX as well: https://www.git-tower.com/blog/make-git-rebase-safe-on-osx/
-  try:
-    trustctime = run(["git", "config", "--get", "core.trustctime"], cwd)
-    trustctime_set = (trustctime.strip() == "false")
-  except subprocess.CalledProcessError:
-    trustctime_set = False
 
-  if not trustctime_set:
-    cloudlog.info("Setting core.trustctime false")
-    run(["git", "config", "core.trustctime", "false"], cwd)
-
-  # We are temporarily using copytree to copy the directory, which also changes
+  # We are using copytree to copy the directory, which also changes
   # inode numbers. Ignore those changes too.
-  try:
-    checkstat = run(["git", "config", "--get", "core.checkStat"], cwd)
-    checkstat_set = (checkstat.strip() == "minimal")
-  except subprocess.CalledProcessError:
-    checkstat_set = False
+  git_cfg = [
+    ("core.trustctime", "false"),
+    ("core.checkStat", "minimal"),
+  ]
+  for option, value in git_cfg:
+    try:
+      ret = run(["git", "config", "--get", option], cwd)
+      config_ok = ret.strip() == value
+    except subprocess.CalledProcessError:
+      config_ok = False
 
-  if not checkstat_set:
-    cloudlog.info("Setting core.checkState minimal")
-    run(["git", "config", "core.checkStat", "minimal"], cwd)
+    if not config_ok:
+      cloudlog.info(f"Setting git '{option}' to '{value}'")
+      run(["git", "config", option, value], cwd)
 
 
 def init_ovfs():
   cloudlog.info("preparing new safe staging area")
   Params().put("UpdateAvailable", "0")
 
-  remove_consistent_flag()
+  set_consistent_flag(False)
 
   dismount_ovfs()
   if os.path.isdir(STAGING_ROOT):
@@ -174,6 +166,7 @@ def init_ovfs():
 
   for dirname in [STAGING_ROOT, OVERLAY_UPPER, OVERLAY_METADATA, OVERLAY_MERGED, FINALIZED]:
     os.mkdir(dirname, 0o755)
+
   if not os.lstat(BASEDIR).st_dev == os.lstat(OVERLAY_MERGED).st_dev:
     raise RuntimeError("base and overlay merge directories are on different filesystems; not valid for overlay FS!")
 
@@ -181,100 +174,39 @@ def init_ovfs():
   if os.path.isfile(os.path.join(BASEDIR, ".overlay_consistent")):
     os.remove(os.path.join(BASEDIR, ".overlay_consistent"))
 
-  # Leave a timestamped canary in BASEDIR to check at startup. The EON clock
+  # Leave a timestamped canary in BASEDIR to check at startup. The device clock
   # should be correct by the time we get here. If the init file disappears, or
   # critical mtimes in BASEDIR are newer than .overlay_init, continue.sh can
   # assume that BASEDIR has used for local development or otherwise modified,
   # and skips the update activation attempt.
   Path(os.path.join(BASEDIR, ".overlay_init")).touch()
 
+  os.system("sync")
   overlay_opts = f"lowerdir={BASEDIR},upperdir={OVERLAY_UPPER},workdir={OVERLAY_METADATA}"
   run(["mount", "-t", "overlay", "-o", overlay_opts, "none", OVERLAY_MERGED])
 
 
-def inodes_in_tree(search_dir):
-  """Given a search root, produce a dictionary mapping of inodes to relative
-  pathnames of regular files (no directories, symlinks, or special files)."""
-  inode_map = {}
-  for root, _, files in os.walk(search_dir, topdown=True):
-    for file_name in files:
-      full_path_name = os.path.join(root, file_name)
-      st = os.lstat(full_path_name)
-      if S_ISREG(st[ST_MODE]):
-        inode_map[st[ST_INO]] = full_path_name
-  return inode_map
-
-
-def dup_ovfs_object(inode_map, source_obj, target_dir):
-  """Given a relative pathname to copy, and a new target root, duplicate the
-  source object in the target root, using hardlinks for regular files."""
-
-  source_full_path = os.path.join(OVERLAY_MERGED, source_obj)
-  st = os.lstat(source_full_path)
-  target_full_path = os.path.join(target_dir, source_obj)
-
-  if S_ISREG(st[ST_MODE]):
-    # Hardlink all regular files; ownership and permissions are shared.
-    link(inode_map[st[ST_INO]], target_full_path)
-  else:
-    # Recreate all directories and symlinks; copy ownership and permissions.
-    if S_ISDIR(st[ST_MODE]):
-      os.mkdir(os.path.join(FINALIZED, source_obj), S_IMODE(st[ST_MODE]))
-    elif S_ISLNK(st[ST_MODE]):
-      os.symlink(os.readlink(source_full_path), target_full_path)
-      os.chmod(target_full_path, S_IMODE(st[ST_MODE]), follow_symlinks=False)
-    else:
-      # Ran into a FIFO, socket, etc. Should not happen in OP install dir.
-      # Ignore without copying for the time being; revisit later if needed.
-      cloudlog.error("can't copy this file type: %s" % source_full_path)
-    os.chown(target_full_path, st[ST_UID], st[ST_GID], follow_symlinks=False)
-
-  # Sync target mtimes to the cached lstat() value from each source object.
-  # Restores shared inode mtimes after linking, fixes symlinks and dirs.
-  os.utime(target_full_path, (st[ST_ATIME], st[ST_MTIME]), follow_symlinks=False)
-
-
-def finalize_from_ovfs_hardlink():
-  """Take the current OverlayFS merged view and finalize a copy outside of
-  OverlayFS, ready to be swapped-in at BASEDIR. Copy using hardlinks"""
-
-  cloudlog.info("creating finalized version of the overlay")
-
-  # The "copy" is done with hardlinks, but since the OverlayFS merge looks
-  # like a different filesystem, and hardlinks can't cross filesystems, we
-  # have to borrow a source pathname from the upper or lower layer.
-  inode_map = inodes_in_tree(BASEDIR)
-  inode_map.update(inodes_in_tree(OVERLAY_UPPER))
-
-  shutil.rmtree(FINALIZED)
-  os.umask(0o077)
-  os.mkdir(FINALIZED)
-  for root, dirs, files in os.walk(OVERLAY_MERGED, topdown=True):
-    for obj_name in dirs:
-      relative_path_name = os.path.relpath(os.path.join(root, obj_name), OVERLAY_MERGED)
-      dup_ovfs_object(inode_map, relative_path_name, FINALIZED)
-    for obj_name in files:
-      relative_path_name = os.path.relpath(os.path.join(root, obj_name), OVERLAY_MERGED)
-      dup_ovfs_object(inode_map, relative_path_name, FINALIZED)
-  cloudlog.info("done finalizing overlay")
-
-
-def finalize_from_ovfs_copy():
+def finalize_from_ovfs():
   """Take the current OverlayFS merged view and finalize a copy outside of
   OverlayFS, ready to be swapped-in at BASEDIR. Copy using shutil.copytree"""
 
+  # Remove the update ready flag and any old updates
   cloudlog.info("creating finalized version of the overlay")
+  set_consistent_flag(False)
   shutil.rmtree(FINALIZED)
+
+  # Copy the merged overlay view and set the update ready flag
   shutil.copytree(OVERLAY_MERGED, FINALIZED, symlinks=True)
+  set_consistent_flag(True)
   cloudlog.info("done finalizing overlay")
 
 
-def attempt_update():
+def attempt_update(wait_helper):
   cloudlog.info("attempting git update inside staging overlay")
 
   setup_git_options(OVERLAY_MERGED)
 
-  git_fetch_output = run(NICE_LOW_PRIORITY + ["git", "fetch"], OVERLAY_MERGED)
+  git_fetch_output = run(["git", "fetch"], OVERLAY_MERGED, low_priority=True)
   cloudlog.info("git fetch success: %s", git_fetch_output)
 
   cur_hash = run(["git", "rev-parse", "HEAD"], OVERLAY_MERGED).rstrip()
@@ -287,106 +219,146 @@ def attempt_update():
   cloudlog.info("comparing %s to %s" % (cur_hash, upstream_hash))
   if new_version or git_fetch_result:
     cloudlog.info("Running update")
+
     if new_version:
       cloudlog.info("git reset in progress")
       r = [
-        run(NICE_LOW_PRIORITY + ["git", "reset", "--hard", "@{u}"], OVERLAY_MERGED),
-        run(NICE_LOW_PRIORITY + ["git", "clean", "-xdf"], OVERLAY_MERGED),
-        run(NICE_LOW_PRIORITY + ["git", "submodule", "init"], OVERLAY_MERGED),
-        run(NICE_LOW_PRIORITY + ["git", "submodule", "update"], OVERLAY_MERGED),
+        run(["git", "reset", "--hard", "@{u}"], OVERLAY_MERGED, low_priority=True),
+        run(["git", "clean", "-xdf"], OVERLAY_MERGED, low_priority=True ),
+        run(["git", "submodule", "init"], OVERLAY_MERGED, low_priority=True),
+        run(["git", "submodule", "update"], OVERLAY_MERGED, low_priority=True),
       ]
       cloudlog.info("git reset success: %s", '\n'.join(r))
 
-    # Un-set the validity flag to prevent the finalized tree from being
-    # activated later if the finalize step is interrupted
-    remove_consistent_flag()
+      # Download the accompanying NEOS version if it doesn't match the current version
+      with open(NEOS_VERSION, "r") as f:
+        cur_neos = f.read().strip()
 
-    finalize_from_ovfs_copy()
+      updated_neos = run(["bash", "-c", r"unset REQUIRED_NEOS_VERSION && source launch_env.sh && \
+                           echo -n $REQUIRED_NEOS_VERSION"], OVERLAY_MERGED).strip()
 
-    # Make sure the validity flag lands on disk LAST, only when the local git
-    # repo and OP install are in a consistent state.
-    set_consistent_flag()
+      cloudlog.info(f"NEOS version check: {cur_neos} vs {updated_neos}")
+      if cur_neos != updated_neos:
+        cloudlog.info(f"Beginning background download for NEOS {updated_neos}")
 
-    cloudlog.info("update successful!")
+        set_offroad_alert("Offroad_NeosUpdate", True)
+        updater_path = os.path.join(OVERLAY_MERGED, "installer/updater/updater")
+        update_manifest = f"file://{OVERLAY_MERGED}/installer/updater/update.json"
+
+        neos_downloaded = False
+        start_time = time.monotonic()
+        # Try to download for one day
+        while (time.monotonic() - start_time < 60*60*24) and not wait_helper.shutdown:
+          wait_helper.ready_event.clear()
+          try:
+            run([updater_path, "bgcache", update_manifest], OVERLAY_MERGED, low_priority=True)
+            neos_downloaded = True
+            break
+          except subprocess.CalledProcessError:
+            cloudlog.info("NEOS background download failed, retrying")
+            wait_helper.sleep(120)
+
+        # If the download failed, we'll show the alert again when we retry
+        set_offroad_alert("Offroad_NeosUpdate", False)
+        if not neos_downloaded:
+          raise Exception("Failed to download NEOS update")
+
+        cloudlog.info(f"NEOS background download successful, took {time.monotonic() - start_time} seconds")
+
+    # Create the finalized, ready-to-swap update
+    finalize_from_ovfs()
+    cloudlog.info("openpilot update successful!")
   else:
     cloudlog.info("nothing new from git at this time")
 
-  set_update_available_params(new_version=new_version)
+  set_update_available_params(new_version)
+  return new_version
 
 
 def main():
-  update_failed_count = 0
-  overlay_init_done = False
   params = Params()
 
   if params.get("DisableUpdates") == b"1":
-    raise RuntimeError("updates are disabled by param")
+    raise RuntimeError("updates are disabled by the DisableUpdates param")
 
-  if not os.geteuid() == 0:
+  if os.geteuid() != 0:
     raise RuntimeError("updated must be launched as root!")
 
   # Set low io priority
-  p = psutil.Process()
+  proc = psutil.Process()
   if psutil.LINUX:
-    p.ionice(psutil.IOPRIO_CLASS_BE, value=7)
+    proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)
 
-  ov_lock_fd = open('/tmp/safe_staging_overlay.lock', 'w')
+  ov_lock_fd = open(LOCK_FILE, 'w')
   try:
     fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
   except IOError:
     raise RuntimeError("couldn't get overlay lock; is another updated running?")
 
-  # Wait a short time before our first update attempt
-  # Avoids race with IsOffroad not being set, reduces manager startup load
-  time.sleep(30)
-  wait_helper = WaitTimeHelper()
+  # Wait for IsOffroad to be set before our first update attempt
+  wait_helper = WaitTimeHelper(proc)
+  wait_helper.sleep(30)
 
-  while True:
-    update_failed_count += 1
+  update_failed_count = 0
+  update_available = False
+  overlay_initialized = False
+  while not wait_helper.shutdown:
+    wait_helper.ready_event.clear()
+
+    # Check for internet every 30s
     time_wrong = datetime.datetime.utcnow().year < 2019
-    ping_failed = subprocess.call(["ping", "-W", "4", "-c", "1", "8.8.8.8"])
+    ping_failed = os.system(f"ping -W 4 -c 1 {TEST_IP}") != 0
+    if ping_failed or time_wrong:
+      wait_helper.sleep(30)
+      continue
 
-    # Wait until we have a valid datetime to initialize the overlay
-    if not (ping_failed or time_wrong):
-      try:
-        # If the git directory has modifcations after we created the overlay
-        # we need to recreate the overlay
-        if overlay_init_done:
-          overlay_init_fn = os.path.join(BASEDIR, ".overlay_init")
-          git_dir_path = os.path.join(BASEDIR, ".git")
-          new_files = run(["find", git_dir_path, "-newer", overlay_init_fn])
+    # Attempt an update
+    exception = None
+    update_failed_count += 1
+    try:
+      # Re-create the overlay if BASEDIR/.git has changed since we created the overlay
+      if overlay_initialized:
+        overlay_init_fn = os.path.join(BASEDIR, ".overlay_init")
+        git_dir_path = os.path.join(BASEDIR, ".git")
+        new_files = run(["find", git_dir_path, "-newer", overlay_init_fn])
 
-          if len(new_files.splitlines()):
-            cloudlog.info(".git directory changed, recreating overlay")
-            overlay_init_done = False
+        if len(new_files.splitlines()):
+          cloudlog.info(".git directory changed, recreating overlay")
+          overlay_initialized = False
 
-        if not overlay_init_done:
-          init_ovfs()
-          overlay_init_done = True
+      if not overlay_initialized:
+        init_ovfs()
+        overlay_initialized = True
 
-        if params.get("IsOffroad") == b"1":
-          attempt_update()
-          update_failed_count = 0
-        else:
-          cloudlog.info("not running updater, openpilot running")
+      if params.get("IsOffroad") == b"1":
+        update_available = attempt_update(wait_helper) or update_available
+        update_failed_count = 0
+        if not update_available and os.path.isdir(NEOSUPDATE_DIR):
+          shutil.rmtree(NEOSUPDATE_DIR)
+      else:
+        cloudlog.info("not running updater, openpilot running")
 
-      except subprocess.CalledProcessError as e:
-        cloudlog.event(
-          "update process failed",
-          cmd=e.cmd,
-          output=e.output,
-          returncode=e.returncode
-        )
-        overlay_init_done = False
-      except Exception:
-        cloudlog.exception("uncaught updated exception, shouldn't happen")
+    except subprocess.CalledProcessError as e:
+      cloudlog.event(
+        "update process failed",
+        cmd=e.cmd,
+        output=e.output,
+        returncode=e.returncode
+      )
+      exception = e
+      overlay_initialized = False
+    except Exception:
+      cloudlog.exception("uncaught updated exception, shouldn't happen")
 
     params.put("UpdateFailedCount", str(update_failed_count))
-    wait_between_updates(wait_helper.ready_event)
-    if wait_helper.shutdown:
-      break
+    if exception is None:
+      params.delete("LastUpdateException")
+    else:
+      params.put("LastUpdateException", f"command failed: {exception.cmd}\n{exception.output}")
+
+    # Wait 10 minutes between update attempts
+    wait_helper.sleep(60*10)
 
-  # We've been signaled to shut down
   dismount_ovfs()
 
 if __name__ == "__main__":