From 5af3f3215737a7741941232f51d4bd7b6b010cfa Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 23 Feb 2026 16:56:58 -0800 Subject: [PATCH] simplify setup (#37358) * simplify setup * lil more * simplify dockedr * just run setup there: * don't need that junk * lil more --- .github/workflows/prebuilt.yaml | 2 +- .github/workflows/release.yaml | 13 +-- Dockerfile.openpilot | 36 +++++-- Dockerfile.openpilot_base | 42 -------- selfdrive/test/docker_build.sh | 16 ++-- selfdrive/test/docker_common.sh | 18 ---- tools/install_python_dependencies.sh | 29 ------ tools/install_ubuntu_dependencies.sh | 99 ------------------- tools/mac_setup.sh | 20 ---- tools/op.sh | 6 +- tools/setup_dependencies.sh | 137 +++++++++++++++++++++++++++ tools/ubuntu_setup.sh | 10 -- 12 files changed, 180 insertions(+), 248 deletions(-) delete mode 100644 Dockerfile.openpilot_base delete mode 100644 selfdrive/test/docker_common.sh delete mode 100755 tools/install_python_dependencies.sh delete mode 100755 tools/install_ubuntu_dependencies.sh delete mode 100755 tools/mac_setup.sh create mode 100755 tools/setup_dependencies.sh delete mode 100755 tools/ubuntu_setup.sh diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml index c0a2baa8e3..ecf1e8503a 100644 --- a/.github/workflows/prebuilt.yaml +++ b/.github/workflows/prebuilt.yaml @@ -6,7 +6,7 @@ on: env: DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: selfdrive/test/docker_build.sh prebuilt + BUILD: selfdrive/test/docker_build.sh jobs: build_prebuilt: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 159902d519..4d731965d7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,20 +7,12 @@ on: jobs: build_masterci: name: build master-ci - env: - ImageOS: ubuntu24 - container: - image: ghcr.io/commaai/openpilot-base:latest runs-on: ubuntu-latest if: github.repository == 'commaai/openpilot' permissions: checks: read contents: write steps: - - name: Install wait-on-check-action dependencies - run: | - sudo apt-get update - sudo apt-get install -y libyaml-dev - name: Wait for green check mark if: ${{ github.event_name == 'schedule' }} uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc @@ -34,9 +26,6 @@ jobs: with: submodules: true fetch-depth: 0 - - name: Pull LFS - run: | - git config --global --add safe.directory '*' - git lfs pull + - uses: ./.github/workflows/setup-with-retry - name: Push master-ci run: BRANCH=__nightly release/build_stripped.sh diff --git a/Dockerfile.openpilot b/Dockerfile.openpilot index 106a06e3a2..72d874b022 100644 --- a/Dockerfile.openpilot +++ b/Dockerfile.openpilot @@ -1,14 +1,38 @@ -FROM ghcr.io/commaai/openpilot-base:latest +FROM ubuntu:24.04 ENV PYTHONUNBUFFERED=1 -ENV OPENPILOT_PATH=/home/batman/openpilot +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends sudo tzdata locales && \ + rm -rf /var/lib/apt/lists/* +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute + +ARG USER=batman +ARG USER_UID=1001 +RUN useradd -m -s /bin/bash -u $USER_UID $USER +RUN usermod -aG sudo $USER +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER $USER + +ENV OPENPILOT_PATH=/home/$USER/openpilot RUN mkdir -p ${OPENPILOT_PATH} WORKDIR ${OPENPILOT_PATH} -COPY . ${OPENPILOT_PATH}/ +COPY --chown=$USER . ${OPENPILOT_PATH}/ -ENV UV_BIN="/home/batman/.local/bin/" -ENV PATH="$UV_BIN:$PATH" -RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc) +ENV UV_BIN="/home/$USER/.local/bin/" +ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv +ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH" +RUN tools/setup_dependencies.sh && \ + sudo rm -rf /var/lib/apt/lists/* + +USER root +RUN git config --global --add safe.directory '*' diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base deleted file mode 100644 index 6ea1450bee..0000000000 --- a/Dockerfile.openpilot_base +++ /dev/null @@ -1,42 +0,0 @@ -FROM ubuntu:24.04 - -ENV PYTHONUNBUFFERED=1 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ - apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 - -COPY tools/install_ubuntu_dependencies.sh /tmp/tools/ -RUN /tmp/tools/install_ubuntu_dependencies.sh && \ - rm -rf /var/lib/apt/lists/* /tmp/* - -ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute -ENV QTWEBENGINE_DISABLE_SANDBOX=1 - -RUN dbus-uuidgen > /etc/machine-id - -ARG USER=batman -ARG USER_UID=1001 -RUN useradd -m -s /bin/bash -u $USER_UID $USER -RUN usermod -aG sudo $USER -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -USER $USER - -COPY --chown=$USER pyproject.toml uv.lock /home/$USER -COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/ - -ENV VIRTUAL_ENV=/home/$USER/.venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN cd /home/$USER && \ - tools/install_python_dependencies.sh && \ - rm -rf tools/ pyproject.toml uv.lock .cache - -USER root -RUN sudo git config --global --add safe.directory /tmp/openpilot diff --git a/selfdrive/test/docker_build.sh b/selfdrive/test/docker_build.sh index 4d58a1507c..8d1fa82249 100755 --- a/selfdrive/test/docker_build.sh +++ b/selfdrive/test/docker_build.sh @@ -1,12 +1,14 @@ #!/usr/bin/env bash set -e -# To build sim and docs, you can run the following to mount the scons cache to the same place as in CI: -# mkdir -p .ci_cache/scons_cache -# sudo mount --bind /tmp/scons_cache/ .ci_cache/scons_cache - SCRIPT_DIR=$(dirname "$0") OPENPILOT_DIR=$SCRIPT_DIR/../../ + +DOCKER_IMAGE=openpilot +DOCKER_FILE=Dockerfile.openpilot +DOCKER_REGISTRY=ghcr.io/commaai +COMMIT_SHA=$(git rev-parse HEAD) + if [ -n "$TARGET_ARCHITECTURE" ]; then PLATFORM="linux/$TARGET_ARCHITECTURE" TAG_SUFFIX="-$TARGET_ARCHITECTURE" @@ -15,9 +17,11 @@ else TAG_SUFFIX="" fi -source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX" +LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX +REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG +REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA -DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR +DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR if [ -n "$PUSH_IMAGE" ]; then docker push $REMOTE_TAG diff --git a/selfdrive/test/docker_common.sh b/selfdrive/test/docker_common.sh deleted file mode 100644 index 2887fff74b..0000000000 --- a/selfdrive/test/docker_common.sh +++ /dev/null @@ -1,18 +0,0 @@ -if [ "$1" = "base" ]; then - export DOCKER_IMAGE=openpilot-base - export DOCKER_FILE=Dockerfile.openpilot_base -elif [ "$1" = "prebuilt" ]; then - export DOCKER_IMAGE=openpilot-prebuilt - export DOCKER_FILE=Dockerfile.openpilot -else - echo "Invalid docker build image: '$1'" - exit 1 -fi - -export DOCKER_REGISTRY=ghcr.io/commaai -export COMMIT_SHA=$(git rev-parse HEAD) - -TAG_SUFFIX=$2 -LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX -REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG -REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA diff --git a/tools/install_python_dependencies.sh b/tools/install_python_dependencies.sh deleted file mode 100755 index 4454845fcd..0000000000 --- a/tools/install_python_dependencies.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Increase the pip timeout to handle TimeoutError -export PIP_DEFAULT_TIMEOUT=200 - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -ROOT="$DIR"/../ -cd "$ROOT" - -if ! command -v "uv" > /dev/null 2>&1; then - echo "installing uv..." - curl -LsSf --retry 5 --retry-delay 5 --retry-all-errors https://astral.sh/uv/install.sh | sh - UV_BIN="$HOME/.local/bin" - PATH="$UV_BIN:$PATH" -fi - -echo "updating uv..." -# ok to fail, can also fail due to installing with brew -uv self update || true - -echo "installing python packages..." -uv sync --frozen --all-extras -source .venv/bin/activate - -if [[ "$(uname)" == 'Darwin' ]]; then - touch "$ROOT"/.env - echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> "$ROOT"/.env -fi diff --git a/tools/install_ubuntu_dependencies.sh b/tools/install_ubuntu_dependencies.sh deleted file mode 100755 index 6793cf2d19..0000000000 --- a/tools/install_ubuntu_dependencies.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -set -e - -SUDO="" - -# Use sudo if not root -if [[ ! $(id -u) -eq 0 ]]; then - if [[ -z $(which sudo) ]]; then - echo "Please install sudo or run as root" - exit 1 - fi - SUDO="sudo" -fi - -# Check if stdin is open -if [ -t 0 ]; then - INTERACTIVE=1 -fi - -# Install common packages -function install_ubuntu_common_requirements() { - $SUDO apt-get update - - # normal stuff, mostly for the bare docker image - $SUDO apt-get install -y --no-install-recommends \ - ca-certificates \ - build-essential \ - curl \ - libssl-dev \ - libcurl4-openssl-dev \ - locales \ - git \ - xvfb - - $SUDO apt-get install -y --no-install-recommends \ - libgles2-mesa-dev \ - libjpeg-dev \ - libncurses5-dev \ - libzstd-dev \ - gettext -} - -# Install Ubuntu 24.04 LTS packages -function install_ubuntu_lts_latest_requirements() { - install_ubuntu_common_requirements - - $SUDO apt-get install -y --no-install-recommends \ - python3-dev \ - python3-venv -} - -# Detect OS using /etc/os-release file -if [ -f "/etc/os-release" ]; then - source /etc/os-release - case "$VERSION_CODENAME" in - "jammy" | "kinetic" | "noble") - install_ubuntu_lts_latest_requirements - ;; - *) - echo "$ID $VERSION_ID is unsupported. This setup script is written for Ubuntu 24.04." - read -p "Would you like to attempt installation anyway? " -n 1 -r - echo "" - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi - install_ubuntu_lts_latest_requirements - esac - - if [[ -d "/etc/udev/rules.d/" ]]; then - # Setup jungle udev rules - $SUDO tee /etc/udev/rules.d/12-panda_jungle.rules > /dev/null < /dev/null < /dev/null </dev/null && pwd )" -ROOT="$(cd $DIR/../ && pwd)" - -if [[ $SHELL == "/bin/zsh" ]]; then - RC_FILE="$HOME/.zshrc" -elif [[ $SHELL == "/bin/bash" ]]; then - RC_FILE="$HOME/.bash_profile" -fi - -# install python dependencies -$DIR/install_python_dependencies.sh -echo "[ ] installed python dependencies t=$SECONDS" - -echo -echo "---- OPENPILOT SETUP DONE ----" -echo "Open a new shell or configure your active shell env by running:" -echo "source $RC_FILE" diff --git a/tools/op.sh b/tools/op.sh index 8c41926e0c..966d4ba433 100755 --- a/tools/op.sh +++ b/tools/op.sh @@ -216,11 +216,7 @@ function op_setup() { echo "Installing dependencies..." st="$(date +%s)" - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - SETUP_SCRIPT="tools/ubuntu_setup.sh" - elif [[ "$OSTYPE" == "darwin"* ]]; then - SETUP_SCRIPT="tools/mac_setup.sh" - fi + SETUP_SCRIPT="tools/setup_dependencies.sh" if ! $OPENPILOT_ROOT/$SETUP_SCRIPT; then echo -e " ↳ [${RED}✗${NC}] Dependencies installation failed!" loge "ERROR_DEPENDENCIES_INSTALLATION" diff --git a/tools/setup_dependencies.sh b/tools/setup_dependencies.sh new file mode 100755 index 0000000000..6dd1f294c1 --- /dev/null +++ b/tools/setup_dependencies.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +ROOT="$(cd "$DIR/../" && pwd)" + +function install_ubuntu_deps() { + SUDO="" + + if [[ ! $(id -u) -eq 0 ]]; then + if [[ -z $(which sudo) ]]; then + echo "Please install sudo or run as root" + exit 1 + fi + SUDO="sudo" + fi + + # Detect OS using /etc/os-release file + if [ -f "/etc/os-release" ]; then + source /etc/os-release + case "$VERSION_CODENAME" in + "jammy" | "kinetic" | "noble") + ;; + *) + echo "$ID $VERSION_ID is unsupported. This setup script is written for Ubuntu 24.04." + read -p "Would you like to attempt installation anyway? " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + ;; + esac + else + echo "No /etc/os-release in the system. Make sure you're running on Ubuntu, or similar." + exit 1 + fi + + $SUDO apt-get update + + # normal stuff, mostly for the bare docker image + $SUDO apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + curl \ + libssl-dev \ + libcurl4-openssl-dev \ + locales \ + git \ + xvfb + + $SUDO apt-get install -y --no-install-recommends \ + python3-dev \ + libgles2-mesa-dev \ + libjpeg-dev \ + libncurses5-dev \ + libzstd-dev \ + gettext + + if [[ -d "/etc/udev/rules.d/" ]]; then + # Setup jungle udev rules + $SUDO tee /etc/udev/rules.d/12-panda_jungle.rules > /dev/null < /dev/null < /dev/null < /dev/null 2>&1; then + echo "installing uv..." + curl -LsSf --retry 5 --retry-delay 5 --retry-all-errors https://astral.sh/uv/install.sh | sh + UV_BIN="$HOME/.local/bin" + PATH="$UV_BIN:$PATH" + fi + + echo "updating uv..." + # ok to fail, can also fail due to installing with brew + uv self update || true + + echo "installing python packages..." + uv sync --frozen --all-extras + source .venv/bin/activate + + if [[ "$(uname)" == 'Darwin' ]]; then + touch "$ROOT"/.env + echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> "$ROOT"/.env + fi +} + +# --- Main --- + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + install_ubuntu_deps + echo "[ ] installed system dependencies t=$SECONDS" +elif [[ "$OSTYPE" == "darwin"* ]]; then + if [[ $SHELL == "/bin/zsh" ]]; then + RC_FILE="$HOME/.zshrc" + elif [[ $SHELL == "/bin/bash" ]]; then + RC_FILE="$HOME/.bash_profile" + fi +fi + +if [ -f "$ROOT/pyproject.toml" ]; then + install_python_deps + echo "[ ] installed python dependencies t=$SECONDS" +fi + +if [[ "$OSTYPE" == "darwin"* ]] && [[ -n "${RC_FILE:-}" ]]; then + echo + echo "---- OPENPILOT SETUP DONE ----" + echo "Open a new shell or configure your active shell env by running:" + echo "source $RC_FILE" +fi diff --git a/tools/ubuntu_setup.sh b/tools/ubuntu_setup.sh deleted file mode 100755 index be4cfb68fa..0000000000 --- a/tools/ubuntu_setup.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# NOTE: this is used in a docker build, so do not run any scripts here. - -"$DIR"/install_ubuntu_dependencies.sh -"$DIR"/install_python_dependencies.sh