pyenv to uv based python and venv (#303)

* switch pyenv to uv

* resolve issues

* remove uv upgrade

* switch to uv

* remove comment

* remove python version

* remove python3.12-dev

* remove empty line

* use python3.pc

* test github workflow

* switch to pyqt-5 + use uv python for pkgconfig

* remove references to pyenv

* add python2

* [upload]

* resolve docker warnings

* [upload]

* setup uv to use /usr/local

* use uv to install pyqt5

* use uv to install pip

* use --seed to setup pip

* test build using --no-deps

* test build pyqt5 using pyproject.toml

* revert pyqt5 from pyproject.toml

* [upload]

* use PyQt5 wheel

* revert duplicate openpilot_dependencies.sh installed in agnos-base

* add sudo

* revert dev deps purge

* remove openpilot_dependencies.sh from agnos-base

* add -y to apt-get

* add pyproject opencl dep

* add gcc-arm-none-eabi dep

* test removing dev dependencies

* test removing dev dependencies

* test purge using noninteractive

* test autoremove dependencies

* revert autoremove

* add pip to handle pip install

* add new line

* rebuild

* use uv python 3.12.3

* test github runner

* revert github runner

* rebuild

* set pc path to python 3.12

* rebuild pyqt5 in Docker

* test rebuild pyqt5

* rebuild pyqt5

* rebuild pyqt5

* test build pyqt5 wheel

* resolve venv

* test rebuild pyqt5 wheel

* resolve build error

* use pip to build wheel

* use prebuilt pyqt5 wheel

* move dbus-python deps

* resolve PR comments

* test remove dbus-python deps

* remove dbus-python deps

* remove pyqt5 pre-built wheel

* test uv python venv

* add quotes

* add shims

* rebuild

* remove XDG_DATA_HOME

* rebuild

* add uv shim

* chmod +x uv

* set UV_NO_CACHE=1

* remove trailing whitespace

* resolve uv device issues

* rebuild

Dockerfile.agnos

@@ -86,29 +86,31 @@ RUN cd /tmp && apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./capn
 COPY --from=agnos-compiler-ffmpeg /tmp/ffmpeg.deb /tmp/ffmpeg.deb
 RUN cd /tmp && apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./ffmpeg.deb
 
+ARG XDG_DATA_HOME="/usr/local"
+ARG PATH="$XDG_DATA_HOME/.cargo/bin:$PATH"
+
 # Use pre-compiled PyQt5 wheel
 COPY --from=agnos-compiler-pyqt5 /tmp/PyQt5-5.15.9/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl /tmp/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl
 
-RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \
-    export PYENV_ROOT="/usr/local/pyenv" && \
-    eval "$(pyenv init -)" && \
-    eval "$(pyenv virtualenv-init -)" && \
-    export MAKEFLAGS="-j$(nproc)" && \
-    pip install pyqt5-sip==12.12.1 && \
-    pip install /tmp/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl --verbose --config-settings --confirm-license= && \
-    pyenv rehash
+RUN export MAKEFLAGS="-j$(nproc)" && \
+    source $XDG_DATA_HOME/venv/bin/activate && \
+    uv pip install pyqt5-sip==12.12.1 && \
+    uv pip install /tmp/PyQt5-5.15.9-cp38-abi3-manylinux_2_17_aarch64.whl --verbose --config-settings="--confirm-license="
 
 # Install openpilot python packages
 RUN mkdir -p /tmp/agnos/uv
 COPY ./userspace/pyproject.toml ./userspace/uv.lock /tmp/agnos/uv
-RUN export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH" && \
-    export PYENV_ROOT="/usr/local/pyenv" && \
-    eval "$(pyenv init -)" && \
-    eval "$(pyenv virtualenv-init -)" && \
+RUN source $XDG_DATA_HOME/venv/bin/activate && \
     cd /tmp/agnos/uv && \
     export PYOPENCL_CL_PRETEND_VERSION="2.0" && \
-    MAKEFLAGS="-j$(nproc)" UV_NO_CACHE=1 uv pip install --no-cache-dir --system . && \
-    pyenv rehash
+    pc="$(python -c "import sysconfig;print(sysconfig.get_config_vars('installed_base')[0])")" && \
+    pcpath=$pc"/lib/pkgconfig" && \
+    export PKG_CONFIG_PATH="$pcpath:$PKG_CONFIG_PATH" && \
+    pypcpath=$pcpath"/python-3.12.pc" && \
+    sed -i 's|prefix=/install|prefix='$pc'|' $pypcpath && \
+    pypcpathembed=$pcpath"/python-3.12-embed.pc" && \
+    sed -i 's|prefix=/install|prefix='$pc'|' $pypcpathembed && \
+    MAKEFLAGS="-j$(nproc)" UV_NO_CACHE=1 uv pip install --no-cache-dir .
 
 # Install nice to haves
 COPY ./userspace/install_extras.sh /tmp/agnos/

userspace/compile-pyqt5.sh

@@ -1,10 +1,6 @@
 #!/bin/bash -e
 
-# Use pyenv venv
-export PATH="/usr/local/pyenv/bin:/usr/local/pyenv/shims:$PATH"
-export PYENV_ROOT="/usr/local/pyenv"
-eval "$(pyenv init -)"
-eval "$(pyenv virtualenv-init -)"
+source /usr/local/venv/bin/activate
 
 # Build PyQt5 wheel
 cd /tmp

userspace/files/brightnessd.service

@@ -3,7 +3,7 @@ Description=Limit display brightness with uptime
 
 [Service]
 Restart=on-failure
-ExecStart=/usr/local/pyenv/shims/python -u /usr/comma/brightnessd.py
+ExecStart=/usr/local/venv/bin/python -u /usr/comma/brightnessd.py
 
 [Install]
 WantedBy=multi-user.target

userspace/files/phantom_touch_logger.service

@@ -4,7 +4,7 @@ After=multi-user.target
 [Service]
 User=comma
 Restart=no
-ExecStart=/usr/local/pyenv/shims/python -u /usr/comma/phantom_touch_logger.py
+ExecStart=/usr/local/venv/bin/python -u /usr/comma/phantom_touch_logger.py
 
 [Install]
 WantedBy=multi-user.target

userspace/files/power_drop_monitor.service

@@ -3,7 +3,7 @@ After=multi-user.target
 
 [Service]
 Restart=always
-ExecStart=/usr/local/pyenv/shims/python -u /usr/comma/power_drop_monitor.py
+ExecStart=/usr/local/venv/bin/python -u /usr/comma/power_drop_monitor.py
 
 [Install]
 WantedBy=multi-user.target

userspace/files/profile

@@ -32,33 +32,16 @@ export QT_QPA_PLATFORM="wayland-egl"
 
 # python setup
 export PYTHONPATH="/data/pythonpath"
-export PATH="/usr/local/pyenv/bin:$PATH"
-export PYENV_VERSION="3.11.4"
-export PYENV_ROOT="/usr/local/pyenv"
 
-# output of "pyenv init - --no-rehash"
-export PATH="/usr/local/pyenv/shims:${PATH}"
-export PYENV_SHELL=bash
-source '/usr/local/pyenv/libexec/../completions/pyenv.bash'
-pyenv() {
-  local command
-  command="${1:-}"
-  if [ "$#" -gt 0 ]; then
-    shift
-  fi
-
-  case "$command" in
-  activate|deactivate|rehash|shell)
-    eval "$(pyenv "sh-$command" "$@")";;
-  *)
-    command pyenv "$command" "$@";;
-  esac
-}
+export PATH="/usr/local/.cargo/bin:$PATH"
+export UV_PYTHON_INSTALL_DIR="/usr/local/uv/python"
+export UV_PYTHON_PREFERENCE=only-managed
+export UV_LINK_MODE=copy
+source /usr/local/venv/bin/activate
 
 # custom shims
 export PATH="/usr/comma/shims:${PATH}"
 
-
 # nice AT command helper
 atc() {
   mmcli -m any --command="${@}"

userspace/openpilot_python_dependencies.sh

@@ -1,21 +1,14 @@
 #!/bin/bash -e
 
-echo "Installing python for openpilot"
+echo "installing uv..."
 
-# Install pyenv
-export PYENV_ROOT="/usr/local/pyenv"
-curl https://pyenv.run | bash
-export PATH="$PYENV_ROOT/bin:$PATH"
-eval "$(pyenv init -)"
+export XDG_DATA_HOME="/usr/local"
+export CARGO_HOME="$XDG_DATA_HOME/.cargo"
 
-PYTHON_VERSION="3.11.4"
-if [ "$(uname -p)" == "aarch64" ]; then
-  pyenv install --verbose $PYTHON_VERSION
-else
-  MAKEFLAGS="-j1" MAKE_OPTS="-j1" taskset --cpu-list 0 pyenv install --verbose $PYTHON_VERSION
-fi
+curl -LsSf https://astral.sh/uv/install.sh | sh
+eval ". $CARGO_HOME/env"
 
-echo "Setting global python version"
-pyenv global $PYTHON_VERSION
+PYTHON_VERSION="3.12.3"
 
-pip3 install --no-cache-dir --upgrade pip uv
+# uv requires virtual env either managed or system before installing dependencies
+uv venv $XDG_DATA_HOME/venv --seed --python-preference only-managed --python=$PYTHON_VERSION

userspace/usr/comma/shims/pip

@@ -1,6 +1,6 @@
 #!/usr/bin/bash
 
-PIP_PATH=/usr/local/pyenv/shims/pip
+PIP_PATH=/usr/local/venv/bin/pip
 export TMPDIR=/tmp/pip-tmp
 
 # setup large tmp for install
@@ -14,5 +14,5 @@ sudo TMPDIR=$TMPDIR PIP_NO_CACHE_DIR=1 $PIP_PATH "$@"
 
 # cleanup
 sudo mount -o ro,remount /
-rm -rf $TMPDIR
+sudo rm -rf $TMPDIR
 sudo mount -o remount /tmp

userspace/usr/comma/shims/uv

@@ -0,0 +1,18 @@
+#!/usr/bin/bash
+
+UV_PATH=/usr/local/.cargo/bin/uv
+export TMPDIR=/tmp/uv-tmp
+
+# setup large tmp for install
+mkdir -p $TMPDIR
+sudo mount -o remount,size=2G /tmp
+sudo mount -o rw,remount /
+sudo resize2fs $(findmnt -n -o SOURCE /) &>/dev/null || sudo resize2fs $(findmnt -n -o SOURCE /)
+
+# run command
+sudo -E TMPDIR=$TMPDIR UV_NO_CACHE=1 $UV_PATH "$@"
+
+# cleanup
+sudo mount -o ro,remount /
+sudo rm -rf $TMPDIR
+sudo mount -o remount /tmp

userspace/usr/comma/sound/sound_init.sh

@@ -16,4 +16,4 @@ echo "sound card online"
 /usr/comma/sound/tinymix set "TERT_MI2S_TX Channels" Two
 
 # setup the amplifier registers
-/usr/local/pyenv/shims/python /usr/comma/sound/amplifier.py
+/usr/local/venv/bin/python /usr/comma/sound/amplifier.py