From 503688baf1127fa4ceedc8a85370eaa379292a91 Mon Sep 17 00:00:00 2001 From: coder351 <170117454+coder351@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:38:22 +0500 Subject: [PATCH] pyenv to uv based python and venv (#303) * switch pyenv to uv * resolve issues * remove uv upgrade * switch to uv * remove comment * remove python version * remove python3.12-dev * remove empty line * use python3.pc * test github workflow * switch to pyqt-5 + use uv python for pkgconfig * remove references to pyenv * add python2 * [upload] * resolve docker warnings * [upload] * setup uv to use /usr/local * use uv to install pyqt5 * use uv to install pip * use --seed to setup pip * test build using --no-deps * test build pyqt5 using pyproject.toml * revert pyqt5 from pyproject.toml * [upload] * use PyQt5 wheel * revert duplicate openpilot_dependencies.sh installed in agnos-base * add sudo * revert dev deps purge * remove openpilot_dependencies.sh from agnos-base * add -y to apt-get * add pyproject opencl dep * add gcc-arm-none-eabi dep * test removing dev dependencies * test removing dev dependencies * test purge using noninteractive * test autoremove dependencies * revert autoremove * add pip to handle pip install * add new line * rebuild * use uv python 3.12.3 * test github runner * revert github runner * rebuild * set pc path to python 3.12 * rebuild pyqt5 in Docker * test rebuild pyqt5 * rebuild pyqt5 * rebuild pyqt5 * test build pyqt5 wheel * resolve venv * test rebuild pyqt5 wheel * resolve build error * use pip to build wheel * use prebuilt pyqt5 wheel * move dbus-python deps * resolve PR comments * test remove dbus-python deps * remove dbus-python deps * remove pyqt5 pre-built wheel * test uv python venv * add quotes * add shims * rebuild * remove XDG_DATA_HOME * rebuild * add uv shim * chmod +x uv * set UV_NO_CACHE=1 * remove trailing whitespace * resolve uv device issues * rebuild --- Dockerfile.agnos | 30 +++++++++++--------- userspace/compile-pyqt5.sh | 6 +--- userspace/files/brightnessd.service | 2 +- userspace/files/phantom_touch_logger.service | 2 +- userspace/files/power_drop_monitor.service | 2 +- userspace/files/profile | 27 ++++-------------- userspace/openpilot_python_dependencies.sh | 23 ++++++--------- userspace/usr/comma/shims/pip | 4 +-- userspace/usr/comma/shims/uv | 18 ++++++++++++ userspace/usr/comma/sound/sound_init.sh | 2 +- 10 files changed, 54 insertions(+), 62 deletions(-) create mode 100755 userspace/usr/comma/shims/uv diff --git a/Dockerfile.agnos b/Dockerfile.agnos index 15b9fad..16b7e6f 100644 --- a/Dockerfile.agnos +++ b/Dockerfile.agnos @@ -86,29 +86,31 @@ RUN cd /tmp && apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./capn COPY --from=agnos-compiler-ffmpeg /tmp/ffmpeg.deb /tmp/ffmpeg.deb RUN cd /tmp && apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./ffmpeg.deb +ARG XDG_DATA_HOME="/usr/local" +ARG PATH="$XDG_DATA_HOME/.cargo/bin:$PATH" + # Use pre-compiled PyQt5 wheel COPY --from=agnos-compiler-pyqt5 /tmp/PyQt5-5.15.9/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl /tmp/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl -RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \ - export PYENV_ROOT="/usr/local/pyenv" && \ - eval "$(pyenv init -)" && \ - eval "$(pyenv virtualenv-init -)" && \ - export MAKEFLAGS="-j$(nproc)" && \ - pip install pyqt5-sip==12.12.1 && \ - pip install /tmp/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl --verbose --config-settings --confirm-license= && \ - pyenv rehash +RUN export MAKEFLAGS="-j$(nproc)" && \ + source $XDG_DATA_HOME/venv/bin/activate && \ + uv pip install pyqt5-sip==12.12.1 && \ + uv pip install /tmp/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl --verbose --config-settings="--confirm-license=" # Install openpilot python packages RUN mkdir -p /tmp/agnos/uv COPY ./userspace/pyproject.toml ./userspace/uv.lock /tmp/agnos/uv -RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \ - export PYENV_ROOT="/usr/local/pyenv" && \ - eval "$(pyenv init -)" && \ - eval "$(pyenv virtualenv-init -)" && \ +RUN source $XDG_DATA_HOME/venv/bin/activate && \ cd /tmp/agnos/uv && \ export PYOPENCL_CL_PRETEND_VERSION="2.0" && \ - MAKEFLAGS="-j$(nproc)" UV_NO_CACHE=1 uv pip install --no-cache-dir --system . && \ - pyenv rehash + pc="$(python -c "import sysconfig;print(sysconfig.get_config_vars('installed_base')[0])")" && \ + pcpath=$pc"/lib/pkgconfig" && \ + export PKG_CONFIG_PATH="$pcpath:$PKG_CONFIG_PATH" && \ + pypcpath=$pcpath"/python-3.12.pc" && \ + sed -i 's|prefix=/install|prefix='$pc'|' $pypcpath && \ + pypcpathembed=$pcpath"/python-3.12-embed.pc" && \ + sed -i 's|prefix=/install|prefix='$pc'|' $pypcpathembed && \ + MAKEFLAGS="-j$(nproc)" UV_NO_CACHE=1 uv pip install --no-cache-dir . # Install nice to haves COPY ./userspace/install_extras.sh /tmp/agnos/ diff --git a/userspace/compile-pyqt5.sh b/userspace/compile-pyqt5.sh index 9742bca..d7de955 100755 --- a/userspace/compile-pyqt5.sh +++ b/userspace/compile-pyqt5.sh @@ -1,10 +1,6 @@ #!/bin/bash -e -# Use pyenv venv -export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" -export PYENV_ROOT="/usr/local/pyenv" -eval "$(pyenv init -)" -eval "$(pyenv virtualenv-init -)" +source /usr/local/venv/bin/activate # Build PyQt5 wheel cd /tmp diff --git a/userspace/files/brightnessd.service b/userspace/files/brightnessd.service index 1dc9636..18beace 100644 --- a/userspace/files/brightnessd.service +++ b/userspace/files/brightnessd.service @@ -3,7 +3,7 @@ Description=Limit display brightness with uptime [Service] Restart=on-failure -ExecStart=/usr/local/pyenv/shims/python -u /usr/comma/brightnessd.py +ExecStart=/usr/local/venv/bin/python -u /usr/comma/brightnessd.py [Install] WantedBy=multi-user.target diff --git a/userspace/files/phantom_touch_logger.service b/userspace/files/phantom_touch_logger.service index 68ca8f1..4d88ea0 100644 --- a/userspace/files/phantom_touch_logger.service +++ b/userspace/files/phantom_touch_logger.service @@ -4,7 +4,7 @@ After=multi-user.target [Service] User=comma Restart=no -ExecStart=/usr/local/pyenv/shims/python -u /usr/comma/phantom_touch_logger.py +ExecStart=/usr/local/venv/bin/python -u /usr/comma/phantom_touch_logger.py [Install] WantedBy=multi-user.target diff --git a/userspace/files/power_drop_monitor.service b/userspace/files/power_drop_monitor.service index 63bb4e8..9501551 100644 --- a/userspace/files/power_drop_monitor.service +++ b/userspace/files/power_drop_monitor.service @@ -3,7 +3,7 @@ After=multi-user.target [Service] Restart=always -ExecStart=/usr/local/pyenv/shims/python -u /usr/comma/power_drop_monitor.py +ExecStart=/usr/local/venv/bin/python -u /usr/comma/power_drop_monitor.py [Install] WantedBy=multi-user.target diff --git a/userspace/files/profile b/userspace/files/profile index d54026e..de6139d 100644 --- a/userspace/files/profile +++ b/userspace/files/profile @@ -32,33 +32,16 @@ export QT_QPA_PLATFORM="wayland-egl" # python setup export PYTHONPATH="/data/pythonpath" -export PATH="/usr/local/pyenv/bin:$PATH" -export PYENV_VERSION="3.11.4" -export PYENV_ROOT="/usr/local/pyenv" -# output of "pyenv init - --no-rehash" -export PATH="/usr/local/pyenv/shims:${PATH}" -export PYENV_SHELL=bash -source '/usr/local/pyenv/libexec/../completions/pyenv.bash' -pyenv() { - local command - command="${1:-}" - if [ "$#" -gt 0 ]; then - shift - fi - - case "$command" in - activate|deactivate|rehash|shell) - eval "$(pyenv "sh-$command" "$@")";; - *) - command pyenv "$command" "$@";; - esac -} +export PATH="/usr/local/.cargo/bin:$PATH" +export UV_PYTHON_INSTALL_DIR="/usr/local/uv/python" +export UV_PYTHON_PREFERENCE=only-managed +export UV_LINK_MODE=copy +source /usr/local/venv/bin/activate # custom shims export PATH="/usr/comma/shims:${PATH}" - # nice AT command helper atc() { mmcli -m any --command="${@}" diff --git a/userspace/openpilot_python_dependencies.sh b/userspace/openpilot_python_dependencies.sh index 4897e01..1054a8f 100755 --- a/userspace/openpilot_python_dependencies.sh +++ b/userspace/openpilot_python_dependencies.sh @@ -1,21 +1,14 @@ #!/bin/bash -e -echo "Installing python for openpilot" +echo "installing uv..." -# Install pyenv -export PYENV_ROOT="/usr/local/pyenv" -curl https://pyenv.run | bash -export PATH="$PYENV_ROOT/bin:$PATH" -eval "$(pyenv init -)" +export XDG_DATA_HOME="/usr/local" +export CARGO_HOME="$XDG_DATA_HOME/.cargo" -PYTHON_VERSION="3.11.4" -if [ "$(uname -p)" == "aarch64" ]; then - pyenv install --verbose $PYTHON_VERSION -else - MAKEFLAGS="-j1" MAKE_OPTS="-j1" taskset --cpu-list 0 pyenv install --verbose $PYTHON_VERSION -fi +curl -LsSf https://astral.sh/uv/install.sh | sh +eval ". $CARGO_HOME/env" -echo "Setting global python version" -pyenv global $PYTHON_VERSION +PYTHON_VERSION="3.12.3" -pip3 install --no-cache-dir --upgrade pip uv +# uv requires virtual env either managed or system before installing dependencies +uv venv $XDG_DATA_HOME/venv --seed --python-preference only-managed --python=$PYTHON_VERSION diff --git a/userspace/usr/comma/shims/pip b/userspace/usr/comma/shims/pip index 1eeac50..6c85e87 100755 --- a/userspace/usr/comma/shims/pip +++ b/userspace/usr/comma/shims/pip @@ -1,6 +1,6 @@ #!/usr/bin/bash -PIP_PATH=/usr/local/pyenv/shims/pip +PIP_PATH=/usr/local/venv/bin/pip export TMPDIR=/tmp/pip-tmp # setup large tmp for install @@ -14,5 +14,5 @@ sudo TMPDIR=$TMPDIR PIP_NO_CACHE_DIR=1 $PIP_PATH "$@" # cleanup sudo mount -o ro,remount / -rm -rf $TMPDIR +sudo rm -rf $TMPDIR sudo mount -o remount /tmp diff --git a/userspace/usr/comma/shims/uv b/userspace/usr/comma/shims/uv new file mode 100755 index 0000000..f9e2250 --- /dev/null +++ b/userspace/usr/comma/shims/uv @@ -0,0 +1,18 @@ +#!/usr/bin/bash + +UV_PATH=/usr/local/.cargo/bin/uv +export TMPDIR=/tmp/uv-tmp + +# setup large tmp for install +mkdir -p $TMPDIR +sudo mount -o remount,size=2G /tmp +sudo mount -o rw,remount / +sudo resize2fs $(findmnt -n -o SOURCE /) &>/dev/null || sudo resize2fs $(findmnt -n -o SOURCE /) + +# run command +sudo -E TMPDIR=$TMPDIR UV_NO_CACHE=1 $UV_PATH "$@" + +# cleanup +sudo mount -o ro,remount / +sudo rm -rf $TMPDIR +sudo mount -o remount /tmp diff --git a/userspace/usr/comma/sound/sound_init.sh b/userspace/usr/comma/sound/sound_init.sh index 7158827..c8b7129 100755 --- a/userspace/usr/comma/sound/sound_init.sh +++ b/userspace/usr/comma/sound/sound_init.sh @@ -16,4 +16,4 @@ echo "sound card online" /usr/comma/sound/tinymix set "TERT_MI2S_TX Channels" Two # setup the amplifier registers -/usr/local/pyenv/shims/python /usr/comma/sound/amplifier.py +/usr/local/venv/bin/python /usr/comma/sound/amplifier.py