diff --git a/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript b/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript index 636ef0fb21..fa249765bc 100644 --- a/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript +++ b/openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript @@ -35,6 +35,7 @@ build_files = [f'{gen}/acados_solver_long.c'] + casadi_model + casadi_cost_y + c # extra generated files used to trigger a rebuild generated_files = [ + 'acados_ocp_long.json', f'{gen}/Makefile', f'{gen}/main_long.c', diff --git a/tools/release/build_release.sh b/tools/release/build_release.sh index 5dca1626ca..4bc5dd2e68 100755 --- a/tools/release/build_release.sh +++ b/tools/release/build_release.sh @@ -2,15 +2,14 @@ set -e set -x -# git diff --name-status origin/release3-staging | grep "^A" | less - DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - cd $DIR BUILD_DIR=/data/openpilot SOURCE_DIR="$(git rev-parse --show-toplevel)" +export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo" + if [ -z "$RELEASE_BRANCH" ]; then echo "RELEASE_BRANCH is not set" exit 1 @@ -23,29 +22,24 @@ BUILD_BRANCH=release-mici-staging source $DIR/identity.sh echo "[-] Setting up repo T=$SECONDS" -rm -rf $BUILD_DIR -mkdir -p $BUILD_DIR +if ! git -C "$SOURCE_DIR" worktree remove --force "$BUILD_DIR" 2>/dev/null; then + rm -rf $BUILD_DIR +fi +git -C "$SOURCE_DIR" worktree prune +git -C "$SOURCE_DIR" worktree add --detach --no-checkout "$BUILD_DIR" cd $BUILD_DIR -git init -git remote add origin git@github.com:commaai/openpilot.git -git checkout --orphan $BUILD_BRANCH +git update-ref -d "refs/heads/$BUILD_BRANCH" +git symbolic-ref HEAD "refs/heads/$BUILD_BRANCH" +git read-tree --empty # do the files copy echo "[-] copying files T=$SECONDS" cd $SOURCE_DIR -cp -pR --parents $(./tools/release/release_files.py) $BUILD_DIR/ +./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" -- # in the directory cd $BUILD_DIR -rm -f panda/board/obj/panda.bin.signed -rm -f panda/board/obj/panda_h7.bin.signed - -VERSION=$(cat openpilot/common/version.h | awk -F[\"-] '{print $2}') -echo "[-] committing version $VERSION T=$SECONDS" -git add -f . -git commit -a -m "openpilot v$VERSION release" - # use the full CPU available for speeding up the build. # openpilot resets the CPU frequencies when test_onroad.py runs below. for policy in /sys/devices/system/cpu/cpufreq/policy*; do @@ -54,9 +48,6 @@ for policy in /sys/devices/system/cpu/cpufreq/policy*; do echo "$hardware_max" | sudo tee "$policy/scaling_max_freq" >/dev/null done -# Build and test before launch_chffrplus.sh creates the on-device package -# symlinks. SConstruct uses the same package roots for build subprocesses. -export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo" scons if [ -n "$INCLUDE_BIG_MODEL" ]; then test -f openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl.chunkmanifest @@ -83,7 +74,6 @@ find . -name '*.a' -delete find . -name '*.o' -delete find . -name '*.os' -delete find . -name '*.pyc' -delete -find . -name 'moc_*' -delete find . -name '__pycache__' -delete rm -rf .sconsign.dblite Jenkinsfile tools/release/ rm -f openpilot/selfdrive/modeld/models/*.onnx* @@ -91,9 +81,11 @@ rm -f openpilot/selfdrive/modeld/models/*.onnx* # Mark as prebuilt release touch prebuilt +VERSION=$(cat openpilot/common/version.h | awk -F[\"-] '{print $2}') # Add built files to git -git add -f . -git commit --amend -m "openpilot v$VERSION" +# writing larger objects is faster than compressing them on-device +git -c core.compression=0 add -f . +git -c core.compression=0 -c gc.auto=0 commit -m "openpilot v$VERSION" # Run tests cd $BUILD_DIR diff --git a/tools/release/build_stripped.sh b/tools/release/build_stripped.sh index 6c1ba4e097..ba4c847375 100755 --- a/tools/release/build_stripped.sh +++ b/tools/release/build_stripped.sh @@ -30,20 +30,14 @@ git submodule deinit -f --all git rm -rf --cached . find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \; -# cleanup before the copy -cd $SOURCE_DIR -git clean -xdff -git submodule foreach --recursive git clean -xdff - # do the files copy echo "[-] copying files T=$SECONDS" cd $SOURCE_DIR -./tools/release/release_files.py | xargs -d '\n' cp -pR --parents -t "$TARGET_DIR" +./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$TARGET_DIR" -- # in the directory cd $TARGET_DIR rm -rf .git/modules/ -rm -f panda/board/obj/panda.bin.signed find openpilot/selfdrive/modeld/models -name '*.onnx' -size +95M -exec ./openpilot/common/file_chunker.py {} \; @@ -57,9 +51,10 @@ echo -n "$GIT_HASH" > git_src_commit echo -n "$GIT_COMMIT_DATE" > git_src_commit_date echo "[-] committing version $VERSION T=$SECONDS" -git add -f . +# writing larger objects is faster than compressing them on-device +git -c core.compression=0 add -f . git status -git commit -a -m "openpilot v$VERSION release +git -c core.compression=0 commit -a -m "openpilot v$VERSION release date: $DATETIME master commit: $GIT_HASH diff --git a/tools/release/release_files.py b/tools/release/release_files.py index 1558f04a26..dd42125337 100755 --- a/tools/release/release_files.py +++ b/tools/release/release_files.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import os import re -from pathlib import Path +import subprocess +import sys HERE = os.path.abspath(os.path.dirname(__file__)) ROOT = os.path.abspath(os.path.join(HERE, "../..")) @@ -25,11 +26,12 @@ whitelist: list[str] = [ ] if __name__ == "__main__": - for f in Path(ROOT).rglob("**/*"): - if not (f.is_file() or f.is_symlink()): + tracked_files = subprocess.check_output(["git", "ls-files", "-z", "--recurse-submodules"], cwd=ROOT).split(b"\0") + for tracked_file in tracked_files: + if not tracked_file: continue - rf = str(f.relative_to(ROOT)) + rf = os.fsdecode(tracked_file) if not os.getenv("INCLUDE_BIG_MODEL") and rf.startswith("openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"): continue blacklisted = any(re.search(p, rf) for p in blacklist) @@ -37,4 +39,4 @@ if __name__ == "__main__": if blacklisted and not whitelisted: continue - print(rf) + sys.stdout.buffer.write(tracked_file + b"\0")