release: speedup builds (#38644)

openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript

@@ -35,6 +35,7 @@ build_files = [f'{gen}/acados_solver_long.c'] + casadi_model + casadi_cost_y + c
 
 # extra generated files used to trigger a rebuild
 generated_files = [
+  'acados_ocp_long.json',
   f'{gen}/Makefile',
 
   f'{gen}/main_long.c',

tools/release/build_release.sh

@@ -2,15 +2,14 @@
 set -e
 set -x
 
-# git diff --name-status origin/release3-staging | grep "^A" | less
-
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-
 cd $DIR
 
 BUILD_DIR=/data/openpilot
 SOURCE_DIR="$(git rev-parse --show-toplevel)"
 
+export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
+
 if [ -z "$RELEASE_BRANCH" ]; then
   echo "RELEASE_BRANCH is not set"
   exit 1
@@ -23,29 +22,24 @@ BUILD_BRANCH=release-mici-staging
 source $DIR/identity.sh
 
 echo "[-] Setting up repo T=$SECONDS"
-rm -rf $BUILD_DIR
-mkdir -p $BUILD_DIR
+if ! git -C "$SOURCE_DIR" worktree remove --force "$BUILD_DIR" 2>/dev/null; then
+  rm -rf $BUILD_DIR
+fi
+git -C "$SOURCE_DIR" worktree prune
+git -C "$SOURCE_DIR" worktree add --detach --no-checkout "$BUILD_DIR"
 cd $BUILD_DIR
-git init
-git remote add origin git@github.com:commaai/openpilot.git
-git checkout --orphan $BUILD_BRANCH
+git update-ref -d "refs/heads/$BUILD_BRANCH"
+git symbolic-ref HEAD "refs/heads/$BUILD_BRANCH"
+git read-tree --empty
 
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
-cp -pR --parents $(./tools/release/release_files.py) $BUILD_DIR/
+./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" --
 
 # in the directory
 cd $BUILD_DIR
 
-rm -f panda/board/obj/panda.bin.signed
-rm -f panda/board/obj/panda_h7.bin.signed
-
-VERSION=$(cat openpilot/common/version.h | awk -F[\"-]  '{print $2}')
-echo "[-] committing version $VERSION T=$SECONDS"
-git add -f .
-git commit -a -m "openpilot v$VERSION release"
-
 # use the full CPU available for speeding up the build.
 # openpilot resets the CPU frequencies when test_onroad.py runs below.
 for policy in /sys/devices/system/cpu/cpufreq/policy*; do
@@ -54,9 +48,6 @@ for policy in /sys/devices/system/cpu/cpufreq/policy*; do
   echo "$hardware_max" | sudo tee "$policy/scaling_max_freq" >/dev/null
 done
 
-# Build and test before launch_chffrplus.sh creates the on-device package
-# symlinks. SConstruct uses the same package roots for build subprocesses.
-export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
 scons
 if [ -n "$INCLUDE_BIG_MODEL" ]; then
   test -f openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl.chunkmanifest
@@ -83,7 +74,6 @@ find . -name '*.a' -delete
 find . -name '*.o' -delete
 find . -name '*.os' -delete
 find . -name '*.pyc' -delete
-find . -name 'moc_*' -delete
 find . -name '__pycache__' -delete
 rm -rf .sconsign.dblite Jenkinsfile tools/release/
 rm -f openpilot/selfdrive/modeld/models/*.onnx*
@@ -91,9 +81,11 @@ rm -f openpilot/selfdrive/modeld/models/*.onnx*
 # Mark as prebuilt release
 touch prebuilt
 
+VERSION=$(cat openpilot/common/version.h | awk -F[\"-]  '{print $2}')
 # Add built files to git
-git add -f .
-git commit --amend -m "openpilot v$VERSION"
+# writing larger objects is faster than compressing them on-device
+git -c core.compression=0 add -f .
+git -c core.compression=0 -c gc.auto=0 commit -m "openpilot v$VERSION"
 
 # Run tests
 cd $BUILD_DIR

tools/release/build_stripped.sh

@@ -30,20 +30,14 @@ git submodule deinit -f --all
 git rm -rf --cached .
 find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
 
-# cleanup before the copy
-cd $SOURCE_DIR
-git clean -xdff
-git submodule foreach --recursive git clean -xdff
-
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
-./tools/release/release_files.py | xargs -d '\n' cp -pR --parents -t "$TARGET_DIR"
+./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$TARGET_DIR" --
 
 # in the directory
 cd $TARGET_DIR
 rm -rf .git/modules/
-rm -f panda/board/obj/panda.bin.signed
 
 find openpilot/selfdrive/modeld/models -name '*.onnx' -size +95M -exec ./openpilot/common/file_chunker.py {} \;
 
@@ -57,9 +51,10 @@ echo -n "$GIT_HASH" > git_src_commit
 echo -n "$GIT_COMMIT_DATE" > git_src_commit_date
 
 echo "[-] committing version $VERSION T=$SECONDS"
-git add -f .
+# writing larger objects is faster than compressing them on-device
+git -c core.compression=0 add -f .
 git status
-git commit -a -m "openpilot v$VERSION release
+git -c core.compression=0 commit -a -m "openpilot v$VERSION release
 
 date: $DATETIME
 master commit: $GIT_HASH

tools/release/release_files.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
 import os
 import re
-from pathlib import Path
+import subprocess
+import sys
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 ROOT = os.path.abspath(os.path.join(HERE, "../.."))
@@ -25,11 +26,12 @@ whitelist: list[str] = [
 ]
 
 if __name__ == "__main__":
-  for f in Path(ROOT).rglob("**/*"):
-    if not (f.is_file() or f.is_symlink()):
+  tracked_files = subprocess.check_output(["git", "ls-files", "-z", "--recurse-submodules"], cwd=ROOT).split(b"\0")
+  for tracked_file in tracked_files:
+    if not tracked_file:
       continue
 
-    rf = str(f.relative_to(ROOT))
+    rf = os.fsdecode(tracked_file)
     if not os.getenv("INCLUDE_BIG_MODEL") and rf.startswith("openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"):
       continue
     blacklisted = any(re.search(p, rf) for p in blacklist)
@@ -37,4 +39,4 @@ if __name__ == "__main__":
     if blacklisted and not whitelisted:
       continue
 
-    print(rf)
+    sys.stdout.buffer.write(tracked_file + b"\0")