mawei/dragonpilot
1
0
Fork 0
mirror of https://github.com/dragonpilot/dragonpilot.git synced 2026-06-24 23:42:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

jenkins: use build.py to manage cache size (#30440)

Browse Source 
* jenkins: use build.py to manage cache size

* label

* set pp

* double

* cleanup

* non mimimal build on PC

---------

Co-authored-by: Justin Newberry <justin@comma.ai>
This commit is contained in:
Adeeb Shihadeh
2023-11-10 16:55:25 -08:00
committed by GitHub
parent 2629417320
commit 5f7143df02
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Jenkinsfile
Vendored
+2 -2
View File
@@ -85,7 +85,7 @@ def pcStage(String stageName, Closure body) {
checkout scm
def dockerArgs = '--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache';
def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE}";
docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .").inside(dockerArgs) {
timeout(time: 20, unit: 'MINUTES') {
try {
@@ -228,7 +228,7 @@ node {
},
'car tests': {
pcStage("car tests") {
sh "scons -j30"
sh label: "build", script: "selfdrive/manager/build.py"
sh label: "test_models.py", script: "INTERNAL_SEG_CNT=250 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \
pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py"
sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py"
selfdrive/manager/build.py
+5 -3
View File
@@ -19,19 +19,21 @@ TOTAL_SCONS_NODES = 2560
MAX_BUILD_PROGRESS = 100
PREBUILT = os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
def build(spinner: Spinner, dirty: bool = False) -> None:
def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
nproc = os.cpu_count()
if nproc is None:
nproc = 2
extra_args = ["--minimal"] if minimal else []
# building with all cores can result in using too
# much memory, so retry with less parallelism
compile_output: List[bytes] = []
for n in (nproc, nproc/2, 1):
compile_output.clear()
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", "--minimal"], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
scons: subprocess.Popen = subprocess.Popen(["scons", f"-j{int(n)}", "--cache-populate", *extra_args], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
assert scons.stderr is not None
# Read progress from stderr and update spinner
@@ -86,4 +88,4 @@ def build(spinner: Spinner, dirty: bool = False) -> None:
if __name__ == "__main__" and not PREBUILT:
spinner = Spinner()
spinner.update_progress(0, 100)
build(spinner, is_dirty())
build(spinner, is_dirty(), minimal = AGNOS)