scons build cleanups (#37981)

* simpler progress

* lil less

* cleanup

* handle cache in scons

* no j

* lil more

* rm atexit

* fix?

* cleanup

.github/workflows/tests.yaml

@@ -107,7 +107,7 @@ jobs:
         submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
-      run: scons -j$(nproc)
+      run: scons
     - name: Run unit tests
       timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
       run: |
@@ -130,7 +130,7 @@ jobs:
         submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
-      run: scons -j$(nproc)
+      run: scons
     - name: Run replay
       timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
       continue-on-error: ${{ github.ref == 'refs/heads/master' }}
@@ -197,7 +197,7 @@ jobs:
         submodules: true
     - run: ./tools/op.sh setup
     - name: Build openpilot
-      run: scons -j$(nproc)
+      run: scons
     - name: Driving test
       timeout-minutes: 2
       run: |
@@ -218,7 +218,7 @@ jobs:
           submodules: true
       - run: ./tools/op.sh setup
       - name: Build openpilot
-        run: scons -j$(nproc)
+        run: scons
       - name: Create UI Report
         run: |
           source selfdrive/test/setup_xvfb.sh

SConstruct

@@ -14,7 +14,7 @@ SCons.Warnings.warningAsException(True)
 
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2))))
 
 AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
 AddOption('--verbose', action='store_true', default=False, help='show full build commands')
@@ -174,16 +174,6 @@ if not GetOption('verbose'):
   ):
     env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
 
-# progress output
-node_interval = 5
-node_count = 0
-def progress_function(node):
-  global node_count
-  node_count += node_interval
-  sys.stderr.write("progress: %d\n" % node_count)
-if os.environ.get('SCONS_PROGRESS'):
-  Progress(progress_function, interval=node_interval)
-
 # ********** Cython build environment **********
 envCython = env.Clone()
 envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
@@ -203,9 +193,19 @@ Export('env', 'arch')
 
 # Setup cache dir
 cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
+cache_size_limit = 4e9 if "CI" in os.environ else 2e9
 CacheDir(cache_dir)
 Clean(["."], cache_dir)
 
+def prune_cache_dir(target=None, source=None, env=None):
+  cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime)
+  cache_size = sum(os.path.getsize(f) for f in cache_files)
+  for f in cache_files:
+    if cache_size < cache_size_limit:
+      break
+    cache_size -= os.path.getsize(f)
+    os.unlink(f)
+
 # ********** start building stuff **********
 
 # Build common module
@@ -264,3 +264,37 @@ if GetOption('extras') and arch != "larch64":
 
 
 env.CompilationDatabase('compile_commands.json')
+
+# progress output
+def count_scons_nodes(nodes):
+  seen = set()
+  stack = list(nodes)
+
+  while stack:
+    node = stack.pop().disambiguate()
+    if node in seen:
+      continue
+    seen.add(node)
+    executor = node.get_executor()
+    if executor is not None:
+      stack += executor.get_all_prerequisites() + executor.get_all_children()
+
+  return len(seen)
+
+progress_interval = 5
+progress_count = 0
+progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
+
+def progress_function(node):
+  global progress_count
+  if progress_count >= progress_total:
+    return
+  progress_count = min(progress_count + progress_interval, progress_total)
+  progress = round(100. * progress_count / progress_total, 1)
+  sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
+  if progress == 100. and sys.stderr.isatty():
+    sys.stderr.write("\n")
+  sys.stderr.flush()
+
+Progress(progress_function, interval=progress_interval)
+AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)

docs/how-to/turn-the-speed-blue.md

@@ -20,7 +20,7 @@ source .venv/bin/activate
 
 Then, compile openpilot:
 ```bash
-scons -j$(nproc)
+scons
 ```
 
 ## 2. Run replay

release/build_release.sh

@@ -46,14 +46,14 @@ git commit -a -m "openpilot v$VERSION release"
 
 # Build
 export PYTHONPATH="$BUILD_DIR"
-scons -j$(nproc) --minimal
+op build
 
 if [ -z "$PANDA_DEBUG_BUILD" ]; then
   # release panda fw
-  CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/
+  CERT=/data/pandaextra/certs/release RELEASE=1 scons panda/
 else
   # build with ALLOW_DEBUG=1 to enable features like experimental longitudinal
-  scons -j$(nproc) panda/
+  scons panda/
 fi
 
 # Ensure no submodules in release

selfdrive/test/scons_build_test.sh

@@ -13,7 +13,7 @@ cd $BASEDIR
 
 cd $BASEDIR/opendbc_repo/
 scons --clean
-scons --no-cache --random -j$(nproc)
+scons --no-cache --random
 if ! scons -q; then
   echo "FAILED: all build products not up to date after first pass."
   exit 1

selfdrive/test/test_updated.py

@@ -50,7 +50,7 @@ class TestUpdated:
       f"git clone {BASEDIR} {self.git_remote_dir}",
       f"git clone {self.git_remote_dir} {self.basedir}",
       f"cd {self.basedir} && git submodule init && git submodule update",
-      f"cd {self.basedir} && scons -j{os.cpu_count()} cereal/ common/"
+      f"cd {self.basedir} && scons cereal/ common/"
     ])
 
     self.params = Params(os.path.join(self.basedir, "persist/params"))

system/camerad/test/debug.sh

@@ -8,7 +8,7 @@ echo 0xfffdbfff | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
 #echo 0 | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
 
 sudo dmesg -C
-scons -u -j8 --minimal .
+scons -u --minimal .
 export DEBUG_FRAMES=1
 export DISABLE_ROAD=1 DISABLE_WIDE_ROAD=1
 #export DISABLE_DRIVER=1

system/manager/build.py

@@ -1,41 +1,27 @@
 #!/usr/bin/env python3
 import os
 import subprocess
-from pathlib import Path
 
 # NOTE: Do NOT import anything here that needs be built (e.g. params)
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.spinner import Spinner
 from openpilot.common.text_window import TextWindow
-from openpilot.common.swaglog import cloudlog, add_file_handler
 from openpilot.system.hardware import HARDWARE, AGNOS
-from openpilot.system.version import get_build_metadata
 
-MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
-CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
-
-TOTAL_SCONS_NODES = 2705
-MAX_BUILD_PROGRESS = 100
-
-def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
+def build() -> None:
+  spinner = Spinner()
+  spinner.update_progress(0, 100)
   env = os.environ.copy()
-  env['SCONS_PROGRESS'] = "1"
-  nproc = os.cpu_count()
-  if nproc is None:
-    nproc = 2
 
-  extra_args = ["--minimal"] if minimal else []
-
-  if AGNOS:
   HARDWARE.set_power_save(False)
+  if AGNOS:
     os.sched_setaffinity(0, range(8))  # ensure we can use the isolcpus cores
 
-  # building with all cores can result in using too
-  # much memory, so retry with less parallelism
+  # building with all cores can result in using too much memory, so retry serially
   compile_output: list[bytes] = []
-  for n in (nproc, nproc/2, 1):
+  for parallelism in ([], ["-j4"], ["-j1"]):
     compile_output.clear()
-    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+    scons: subprocess.Popen = subprocess.Popen(["scons", *parallelism], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
     assert scons.stderr is not None
 
     # Read progress from stderr and update spinner
@@ -48,8 +34,8 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
 
         prefix = b'progress: '
         if line.startswith(prefix):
-          i = int(line[len(prefix):])
-          spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
+          progress = float(line[len(prefix):])
+          spinner.update_progress(100 * min(1., progress / 100.), 100.)
         elif len(line):
           compile_output.append(line)
           print(line.decode('utf8', 'replace'))
@@ -66,8 +52,6 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
 
     # Build failed log errors
     error_s = b"\n".join(compile_output).decode('utf8', 'replace')
-    add_file_handler(cloudlog)
-    cloudlog.error("scons build failed\n" + error_s)
 
     # Show TextWindow
     spinner.close()
@@ -76,19 +60,5 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
         t.wait_for_exit()
     exit(1)
 
-  # enforce max cache size
-  cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
-  cache_files.sort(key=lambda f: f.stat().st_mtime)
-  cache_size = sum(f.stat().st_size for f in cache_files)
-  for f in cache_files:
-    if cache_size < MAX_CACHE_SIZE:
-      break
-    cache_size -= f.stat().st_size
-    f.unlink()
-
-
 if __name__ == "__main__":
-  spinner = Spinner()
-  spinner.update_progress(0, 100)
-  build_metadata = get_build_metadata()
-  build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS)
+  build()

tools/cabana/cabana

@@ -33,6 +33,6 @@ fi
 
 # Build _cabana
 cd "$ROOT"
-scons -j4 tools/cabana/_cabana cereal/messaging/bridge
+scons tools/cabana/_cabana cereal/messaging/bridge
 
 exec "$DIR/_cabana" "$@"