mapd: Preload dependencies (#114)

* exclude third_party/mapd/ from gitignore

* all mapd dependencies

* exclude more

* more dependencies!

* add to files_common

* add script to compress locally

* compressed dependencies

* Revert "more dependencies!"

This reverts commit 0cd08beb3fa81b5e037ceb41b5e0554261c56dd5.

* Revert "all mapd dependencies"

This reverts commit b1a19b82b3911313e543a91d77ece124f5e7ded0.

* decompress preloaded deps

* fix path

* use xz

* fix script

* update deps

* no need

* strip first level

* missed this

* try that

* not needed

* try except extraction

* handle exception

* unused

---------

Co-authored-by: Comma Device <device@comma.ai>

scripts/mapd_dep_compress.sh

@@ -0,0 +1,5 @@
+#! usr/bin/bash
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null && pwd)"
+
+tar -cf - -C "$DIR/third_party" mapd | xz -9zev -T0 > "$DIR/selfdrive/mapd/assets/mapd_deps.tar.xz"

selfdrive/manager/custom_dep.py

@@ -3,6 +3,7 @@ import os
 import sys
 import errno
 import shutil
+import tarfile
 import time
 from common.basedir import BASEDIR
 from urllib.request import urlopen
@@ -21,6 +22,7 @@ MAX_BUILD_PROGRESS = 100
 TMP_DIR = '/data/tmp'
 THIRD_PARTY_DIR = '/data/openpilot/third_party/mapd'
 THIRD_PARTY_DIR_SP = '/data/third_party_community'
+PRELOADED_DEP_FILE = os.path.join(BASEDIR, "selfdrive/mapd/assets/mapd_deps.tar.xz")
 
 
 def wait_for_internet_connection(return_on_failure=False):
@@ -87,12 +89,27 @@ def install_dep(spinner):
 
 if __name__ == "__main__" and (OPSPLINE_SPEC is None or OVERPY_SPEC is None):
   spinner = Spinner()
-  if os.path.exists(THIRD_PARTY_DIR_SP):
-    spinner.update("Loading dependencies")
-    command = f'rm -rf {THIRD_PARTY_DIR}; cp -rf {THIRD_PARTY_DIR_SP} {THIRD_PARTY_DIR}'
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
-    print(f"SP_LOG: Removed directory {THIRD_PARTY_DIR}")
-    print(f"SP_LOG: Copied {THIRD_PARTY_DIR_SP} to {THIRD_PARTY_DIR}")
-  else:
-    spinner.update("Waiting for internet")
-    install_dep(spinner)
+  preload_fault = False
+  if os.path.exists(PRELOADED_DEP_FILE):
+    spinner.update("Loading preloaded dependencies")
+    try:
+      with tarfile.open(PRELOADED_DEP_FILE, "r:xz") as tar:
+        for member in tar.getmembers():
+          split_components = member.name.split('/')
+          if len(split_components) > 1:
+            member.name = '/'.join(split_components[1:])
+          tar.extract(member, path=THIRD_PARTY_DIR)
+      print(f"SP_LOG: Preloaded dependencies extracted to {THIRD_PARTY_DIR}")
+    except Exception as e:
+      preload_fault = True
+      print(f"SP_LOG: An error occurred while extracting preloaded dependencies: {e}")
+  if not os.path.exists(PRELOADED_DEP_FILE) or preload_fault:
+    if os.path.exists(THIRD_PARTY_DIR_SP):
+      spinner.update("Loading dependencies")
+      command = f'rm -rf {THIRD_PARTY_DIR}; cp -rf {THIRD_PARTY_DIR_SP} {THIRD_PARTY_DIR}'
+      process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
+      print(f"SP_LOG: Removed directory {THIRD_PARTY_DIR}")
+      print(f"SP_LOG: Copied {THIRD_PARTY_DIR_SP} to {THIRD_PARTY_DIR}")
+    else:
+      spinner.update("Waiting for internet")
+      install_dep(spinner)