Merge branch 'upstream/master' into sync-20260517-new-new

SConstruct

@@ -10,25 +10,28 @@ import numpy as np
 import SCons.Errors
 from SCons.Defaults import _stripixes
 
+TICI = os.path.isfile('/TICI')
+
 SCons.Warnings.warningAsException(True)
 
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2))))
 
 AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
 AddOption('--verbose', action='store_true', default=False, help='show full build commands')
+release = not os.path.exists(File('#.gitattributes').abspath) # file absent on release branch, see release_files.py
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
+          default=(not TICI and not release),
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
 # Detect platform
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and os.path.isfile('/TICI'):
+elif arch == "aarch64" and TICI:
   arch = "larch64"
 assert arch in [
   "larch64",  # linux tici arm64
@@ -37,8 +40,14 @@ assert arch in [
   "Darwin",   # macOS arm64 (x86 not supported)
 ]
 
-pkg_names = ['bzip2', 'capnproto', 'eigen', 'ffmpeg', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
+pkg_names = ['acados', 'bzip2', 'capnproto', 'catch2', 'eigen', 'ffmpeg', 'json11', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
 pkgs = [importlib.import_module(name) for name in pkg_names]
+acados = pkgs[pkg_names.index('acados')]
+acados_include_dirs = [
+  acados.INCLUDE_DIR,
+  os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
+  os.path.join(acados.INCLUDE_DIR, "hpipm", "include"),
+]
 
 
 # ***** enforce a whitelist of system libraries *****
@@ -82,10 +91,10 @@ def _libflags(target, source, env, for_signature):
 env = Environment(
   ENV={
     "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+    "PYTHONPATH": Dir("#").abspath,
+    "ACADOS_SOURCE_DIR": acados.DIR,
+    "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
+    "TERA_PATH": acados.TERA_PATH
   },
   CCFLAGS=[
     "-g",
@@ -105,22 +114,14 @@ env = Environment(
   CPPPATH=[
     "#",
     "#msgq",
-    "#third_party",
-    "#third_party/json11",
-    "#third_party/linux/include",
-    "#third_party/acados/include",
-    "#third_party/acados/include/blasfeo/include",
-    "#third_party/acados/include/hpipm/include",
-    "#third_party/catch2/include",
+    acados_include_dirs,
     [x.INCLUDE_DIR for x in pkgs],
   ],
   LIBPATH=[
     "#common",
     "#msgq_repo",
-    "#third_party",
     "#selfdrive/pandad",
     "#rednose/helpers",
-    f"#third_party/acados/{arch}/lib",
     [x.LIB_DIR for x in pkgs],
   ],
   RPATH=[],
@@ -174,16 +175,6 @@ if not GetOption('verbose'):
   ):
     env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
 
-# progress output
-node_interval = 5
-node_count = 0
-def progress_function(node):
-  global node_count
-  node_count += node_interval
-  sys.stderr.write("progress: %d\n" % node_count)
-if os.environ.get('SCONS_PROGRESS'):
-  Progress(progress_function, interval=node_interval)
-
 # ********** Cython build environment **********
 envCython = env.Clone()
 envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
@@ -199,14 +190,24 @@ else:
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch')
+Export('env', 'arch', 'acados')
 
 # Setup cache dir
 default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
 cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
+cache_size_limit = 4e9 if "CI" in os.environ else 2e9
 CacheDir(cache_dir)
 Clean(["."], cache_dir)
 
+def prune_cache_dir(target=None, source=None, env=None):
+  cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime)
+  cache_size = sum(os.path.getsize(f) for f in cache_files)
+  for f in cache_files:
+    if cache_size < cache_size_limit:
+      break
+    cache_size -= os.path.getsize(f)
+    os.unlink(f)
+
 # ********** start building stuff **********
 
 # Build common module
@@ -242,9 +243,6 @@ SConscript([
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
 
-# Build openpilot
-SConscript(['third_party/SConscript'])
-
 # Build selfdrive
 SConscript([
   'selfdrive/pandad/SConscript',
@@ -257,8 +255,8 @@ SConscript([
 
 SConscript(['sunnypilot/SConscript'])
 
-# Build tools
-if arch != "larch64":
+# Build desktop-only tools
+if GetOption('extras') and arch != "larch64":
   SConscript([
     'tools/replay/SConscript',
     'tools/cabana/SConscript',
@@ -267,3 +265,37 @@ if arch != "larch64":
 
 
 env.CompilationDatabase('compile_commands.json')
+
+# progress output
+def count_scons_nodes(nodes):
+  seen = set()
+  stack = list(nodes)
+
+  while stack:
+    node = stack.pop().disambiguate()
+    if node in seen:
+      continue
+    seen.add(node)
+    executor = node.get_executor()
+    if executor is not None:
+      stack += executor.get_all_prerequisites() + executor.get_all_children()
+
+  return len(seen)
+
+progress_interval = 5
+progress_count = 0
+progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
+
+def progress_function(node):
+  global progress_count
+  if progress_count >= progress_total:
+    return
+  progress_count = min(progress_count + progress_interval, progress_total)
+  progress = round(100. * progress_count / progress_total, 1)
+  sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
+  if progress == 100. and sys.stderr.isatty():
+    sys.stderr.write("\n")
+  sys.stderr.flush()
+
+Progress(progress_function, interval=progress_interval)
+AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)