build.py: retry with less threads (#29543)

* build.py: retry with less threads

* cleanup

* fix no output

* cleanup
old-commit-hash: bb1b641812353e9723cad0aa6f844466c67d9c6b

selfdrive/manager/build.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 import os
 import subprocess
-import textwrap
 from pathlib import Path
+from typing import List
 
 # NOTE: Do NOT import anything here that needs be built (e.g. params)
 from openpilot.common.basedir import BASEDIR
@@ -19,57 +19,59 @@ TOTAL_SCONS_NODES = 2560
 MAX_BUILD_PROGRESS = 100
 PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
 
-
 def build(spinner: Spinner, dirty: bool = False) -> None:
   env = os.environ.copy()
   env['SCONS_PROGRESS'] = "1"
   nproc = os.cpu_count()
-  j_flag = "" if nproc is None else f"-j{nproc - 1}"
+  if nproc is None:
+    nproc = 2
 
-  scons: subprocess.Popen = subprocess.Popen(["scons", j_flag, "--cache-populate"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
-  assert scons.stderr is not None
+  # building with all cores can result in using too
+  # much memory, so retry with less parallelism
+  compile_output: List[bytes] = []
+  for n in (nproc, nproc/2, 1):
+    compile_output.clear()
+    scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+    assert scons.stderr is not None
 
-  compile_output = []
+    # Read progress from stderr and update spinner
+    while scons.poll() is None:
+      try:
+        line = scons.stderr.readline()
+        if line is None:
+          continue
+        line = line.rstrip()
 
-  # Read progress from stderr and update spinner
-  while scons.poll() is None:
-    try:
-      line = scons.stderr.readline()
-      if line is None:
-        continue
-      line = line.rstrip()
+        prefix = b'progress: '
+        if line.startswith(prefix):
+          i = int(line[len(prefix):])
+          spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
+        elif len(line):
+          compile_output.append(line)
+          print(line.decode('utf8', 'replace'))
+      except Exception:
+        pass
 
-      prefix = b'progress: '
-      if line.startswith(prefix):
-        i = int(line[len(prefix):])
-        spinner.update_progress(MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES), 100.)
-      elif len(line):
-        compile_output.append(line)
-        print(line.decode('utf8', 'replace'))
-    except Exception:
-      pass
+    if scons.returncode == 0:
+      break
 
   if scons.returncode != 0:
     # Read remaining output
-    r = scons.stderr.read().split(b'\n')
-    compile_output += r
+    if scons.stderr is not None:
+      compile_output += scons.stderr.read().split(b'\n')
 
     # Build failed log errors
-    errors = [line.decode('utf8', 'replace') for line in compile_output
-              if any(err in line for err in [b'error: ', b'not found, needed by target'])]
-    error_s = "\n".join(errors)
+    error_s = b"\n".join(compile_output).decode('utf8', 'replace')
     add_file_handler(cloudlog)
     cloudlog.error("scons build failed\n" + error_s)
 
     # Show TextWindow
     spinner.close()
     if not os.getenv("CI"):
-      error_s = "\n \n".join("\n".join(textwrap.wrap(e, 65)) for e in errors)
       with TextWindow("openpilot failed to build\n \n" + error_s) as t:
         t.wait_for_exit()
     exit(1)
 
-
   # enforce max cache size
   cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
   cache_files.sort(key=lambda f: f.stat().st_mtime)