From f856a66bfce0aaeba4ba92dcba0f218c7579cab8 Mon Sep 17 00:00:00 2001 From: Robin Reckmann Date: Thu, 12 Sep 2024 05:36:48 +0900 Subject: [PATCH] Build independent packages first (#354) * Move package compilation to separate dockerfile * Add tag * Revert "Add tag" This reverts commit 6ed675b5db217e3ffaa45711444742127d2d3ee9. * Export packages from compiler docker * Add packages folder to gitignore * Add newline at EOF * Keep cleaning apt * Minimize compiler image * Keep compilation in main dockerfile * Revert some more * Remove wip part for now * Move build requirements installation to build scripts * Remove newlines * lint * Compile and package modemmanager * Disable unrelated modems * Add runtime dependencies * [Upload] * Add missing package to fix networking * Move package install to mm for now --- Dockerfile.agnos | 94 +++++++++++++++++++--------- userspace/compile-capnp.sh | 11 +++- userspace/compile-ffmpeg.sh | 29 +++++++-- userspace/compile-modemmanager.sh | 100 +++++++++++++++++++++--------- userspace/compile-pyqt5.sh | 25 +++++++- userspace/compile-qtwayland5.sh | 30 ++++++++- 6 files changed, 220 insertions(+), 69 deletions(-) diff --git a/Dockerfile.agnos b/Dockerfile.agnos index 7f498ec..32cb9f9 100644 --- a/Dockerfile.agnos +++ b/Dockerfile.agnos @@ -1,5 +1,52 @@ # check=error=true +# ################## # +# ### Package ##### # +# ### Compilation ## # +# ################## # +FROM ubuntu:24.04 AS agnos-compiler + +# Common packages +RUN apt-get update && apt-get install -yq --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + curl \ + checkinstall \ + git \ + pkg-config \ + wget + +# capnproto +FROM agnos-compiler AS agnos-compiler-capnp +COPY ./userspace/compile-capnp.sh /tmp/agnos/ +RUN /tmp/agnos/compile-capnp.sh + +# ffmpeg +FROM agnos-compiler AS agnos-compiler-ffmpeg +COPY ./userspace/compile-ffmpeg.sh /tmp/agnos/ +RUN /tmp/agnos/compile-ffmpeg.sh + +# ModemManager +FROM agnos-compiler AS agnos-compiler-modemmanager +COPY ./userspace/compile-modemmanager.sh /tmp/agnos/ +RUN /tmp/agnos/compile-modemmanager.sh + +# PyQt5 +FROM agnos-compiler AS agnos-compiler-pyqt5 +COPY ./userspace/qtwayland/*.deb /tmp/agnos/ +COPY ./userspace/openpilot_python_dependencies.sh /tmp/agnos/ +RUN /tmp/agnos/openpilot_python_dependencies.sh +COPY ./userspace/compile-pyqt5.sh /tmp/agnos/ +RUN /tmp/agnos/compile-pyqt5.sh + +# qtwayland5 +FROM agnos-compiler AS agnos-compiler-qtwayland5 +COPY ./userspace/qtwayland/*.deb /tmp/agnos/ +COPY ./userspace/compile-qtwayland5.sh /tmp/agnos/ +COPY ./userspace/qtwayland/patch* /tmp/agnos/ +RUN /tmp/agnos/compile-qtwayland5.sh + # ################## # # ###### Base ###### # # ################## # @@ -37,31 +84,6 @@ RUN apt-get -o Dpkg::Options::="--force-overwrite" install -yq \ /tmp/agnos/libssl1.1_1.1.1f-1ubuntu2.22_arm64.deb \ /tmp/agnos/libffi6_3.2.1-8_arm64.deb -# ################## # -# #### Compiler #### # -# ################## # -FROM agnos-base AS agnos-compiler - -RUN apt-fast update && apt-fast install --no-install-recommends -yq checkinstall - -# Individual compiling images -FROM agnos-compiler AS agnos-compiler-pyqt5 -COPY ./userspace/compile-pyqt5.sh /tmp/agnos/ -RUN /tmp/agnos/compile-pyqt5.sh - -FROM agnos-compiler AS agnos-compiler-capnp -COPY ./userspace/compile-capnp.sh /tmp/agnos/ -RUN /tmp/agnos/compile-capnp.sh - -FROM agnos-compiler AS agnos-compiler-ffmpeg -COPY ./userspace/compile-ffmpeg.sh /tmp/agnos/ -RUN /tmp/agnos/compile-ffmpeg.sh - -FROM agnos-compiler AS agnos-compiler-qtwayland5 -COPY ./userspace/compile-qtwayland5.sh /tmp/agnos/ -COPY ./userspace/qtwayland/patch* /tmp/agnos/ -RUN /tmp/agnos/compile-qtwayland5.sh - # ################### # # ###### AGNOS ###### # # ################### # @@ -74,9 +96,25 @@ COPY ./userspace/hardware_setup.sh /tmp/agnos RUN /tmp/agnos/hardware_setup.sh RUN mv /data/persist /system/ && rm -rf /data/* -# Build ModemManager -COPY ./userspace/compile-modemmanager.sh /tmp/agnos/ -RUN /tmp/agnos/compile-modemmanager.sh +# Pre-compiled libqmi and modemmanager +COPY --from=agnos-compiler-modemmanager /tmp/libqmi.deb /tmp/modemmanager.deb /tmp/ +RUN cd /tmp && \ + apt-get update && \ + apt-get install -yq --no-install-recommends \ + gir1.2-qmi-1.0 \ + libglib2.0-dev \ + libqmi-glib5 \ + libc6 \ + libglib2.0-0t64 \ + libgudev-1.0-0 \ + libmm-glib0 \ + libpolkit-gobject-1-0 \ + libsystemd0 \ + polkitd \ + polkitd-pkla \ + mobile-broadband-provider-info && \ + apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./libqmi.deb && \ + apt-get -o Dpkg::Options::="--force-overwrite" install -yq ./modemmanager.deb # Pre-compiled capnp (must be before python install) COPY --from=agnos-compiler-capnp /tmp/capnproto.deb /tmp/capnproto.deb diff --git a/userspace/compile-capnp.sh b/userspace/compile-capnp.sh index 279a516..b3203ba 100755 --- a/userspace/compile-capnp.sh +++ b/userspace/compile-capnp.sh @@ -1,8 +1,15 @@ #!/bin/bash -e -# Install capnproto -cd /tmp VERSION=1.0.2 + +# Install build requirements +apt-get update && apt-get install -yq --no-install-recommends \ + libc6-dev \ + libssl-dev \ + zlib1g-dev + +# Build capnproto +cd /tmp wget https://capnproto.org/capnproto-c++-${VERSION}.tar.gz tar xvf capnproto-c++-${VERSION}.tar.gz cd capnproto-c++-${VERSION} diff --git a/userspace/compile-ffmpeg.sh b/userspace/compile-ffmpeg.sh index d007f82..f908563 100755 --- a/userspace/compile-ffmpeg.sh +++ b/userspace/compile-ffmpeg.sh @@ -1,10 +1,31 @@ #!/bin/bash -e -# Install ffmpeg (the one from the ubuntu repos doesn't work with our libOpenCL) +VERSION=4.2.2 + +# Install build requirements +apt-get update && apt-get install -yq --no-install-recommends \ + libass-dev \ + libfreetype6-dev \ + libgnutls28-dev \ + libmp3lame-dev \ + libsdl2-dev \ + libtool \ + libva-dev \ + libvdpau-dev \ + libvorbis-dev \ + libxcb1-dev \ + libxcb-shm0-dev \ + libxcb-xfixes0-dev \ + liblzma-dev \ + rsync \ + texinfo \ + zlib1g-dev + +# Build ffmpeg (the one from the ubuntu repos doesn't work with our libOpenCL) cd /tmp -wget https://ffmpeg.org/releases/ffmpeg-4.2.2.tar.bz2 -tar xvf ffmpeg-4.2.2.tar.bz2 -cd ffmpeg-4.2.2 +wget https://ffmpeg.org/releases/ffmpeg-${VERSION}.tar.bz2 +tar xvf ffmpeg-${VERSION}.tar.bz2 +cd ffmpeg-${VERSION} # avoid makeinfo: error parsing ./doc/t2h.pm: Undefined subroutine &Texinfo::Config::set_from_init_file called at ./doc/t2h.pm line 24. # with --disable-htmlpages diff --git a/userspace/compile-modemmanager.sh b/userspace/compile-modemmanager.sh index 1cd8587..ba5160e 100755 --- a/userspace/compile-modemmanager.sh +++ b/userspace/compile-modemmanager.sh @@ -3,41 +3,48 @@ set -e MM_VERSION="1.22.0" LIBQMI_VERSION="1.34.0" -PROVIDER_INFO_VERSION="20230416" cd /tmp -apt-fast update -apt-fast install -y --no-install-recommends automake autoconf build-essential cmake +# meson support for checkinstall +git clone https://github.com/keithbowes/meson-install.git -# TODO: clean up these build time dependencies -apt-fast install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel ninja-build meson -export PATH=$PATH:/root/.local/bin - -# build mobile-broadband-provider-info -apt-fast install -y --no-install-recommends xsltproc -git clone -b $PROVIDER_INFO_VERSION --depth 1 https://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git -cd mobile-broadband-provider-info -./autogen.sh -./configure -make install - -# build libqmi -cd /tmp -apt-fast install -y --no-install-recommends libgudev-1.0-dev gobject-introspection libgirepository1.0-dev help2man bash-completion python-is-python3 +# libqmi +apt-get update && apt-get install -yq --no-install-recommends \ + bash-completion \ + gobject-introspection \ + gtk-doc-tools \ + help2man \ + libgirepository1.0-dev \ + libglib2.0-dev \ + libgudev-1.0-dev \ + meson \ + ninja-build \ git clone -b $LIBQMI_VERSION --depth 1 https://gitlab.freedesktop.org/mobile-broadband/libqmi.git cd libqmi meson setup build --prefix=/usr --libdir=/usr/lib/aarch64-linux-gnu -Dmbim_qmux=false -Dqrtr=false ninja -C build -ninja -C build install -# build ModemManager +cd build +checkinstall -yD --install=no --fstrans=no --pkgname=libqmi /tmp/meson-install/meson-install +mv libqmi*.deb /tmp/libqmi.deb + +# ModemManager cd /tmp -apt install -y --no-install-recommends gettext libpolkit-gobject-1-dev libdbus-1-dev libsystemd-dev polkitd-pkla git clone -b $MM_VERSION --depth 1 https://gitlab.freedesktop.org/mobile-broadband/ModemManager.git +apt-get install -y --no-install-recommends \ + cmake \ + gettext \ + libdbus-1-dev \ + libpolkit-gobject-1-dev \ + libsystemd-dev \ + udev + +apt-get -o Dpkg::Options::="--force-overwrite" install -yq /tmp/libqmi.deb + cd ModemManager meson setup build \ --prefix=/usr \ @@ -47,16 +54,49 @@ meson setup build \ -Dqmi=true \ -Dmbim=false \ -Dqrtr=false \ - -Dplugin_foxconn=disabled \ - -Dplugin_dell=disabled \ + -Dplugin_generic=enabled \ -Dplugin_altair_lte=disabled \ - -Dplugin_fibocom=disabled + -Dplugin_anydata=disabled \ + -Dplugin_broadmobi=disabled \ + -Dplugin_cinterion=disabled \ + -Dplugin_dell=disabled \ + -Dplugin_dlink=disabled \ + -Dplugin_fibocom=disabled \ + -Dplugin_foxconn=disabled \ + -Dplugin_gosuncn=disabled \ + -Dplugin_haier=disabled \ + -Dplugin_huawei=disabled \ + -Dplugin_intel=disabled \ + -Dplugin_iridium=disabled \ + -Dplugin_linktop=disabled \ + -Dplugin_longcheer=disabled \ + -Dplugin_mbm=disabled \ + -Dplugin_motorola=disabled \ + -Dplugin_mtk=disabled \ + -Dplugin_nokia=disabled \ + -Dplugin_nokia_icera=disabled \ + -Dplugin_novatel=disabled \ + -Dplugin_novatel_lte=disabled \ + -Dplugin_option=disabled \ + -Dplugin_option_hso=disabled \ + -Dplugin_pantech=disabled \ + -Dplugin_qcom_soc=disabled \ + -Dplugin_quectel=enabled \ + -Dplugin_samsung=disabled \ + -Dplugin_sierra_legacy=disabled \ + -Dplugin_sierra=disabled \ + -Dplugin_simtech=disabled \ + -Dplugin_telit=disabled \ + -Dplugin_thuraya=disabled \ + -Dplugin_tplink=disabled \ + -Dplugin_ublox=disabled \ + -Dplugin_via=disabled \ + -Dplugin_wavecom=disabled \ + -Dplugin_x22x=disabled \ + -Dplugin_zte=disabled ninja -C build -ninja -C build install -# remove plugins we don't use, makes probing faster -mkdir -p /tmp/mm-plugins -mv /usr/lib/aarch64-linux-gnu/ModemManager/libmm-*.so /tmp/mm-plugins -cp /tmp/mm-plugins/*generic* /usr/lib/aarch64-linux-gnu/ModemManager/ -cp /tmp/mm-plugins/*quectel* /usr/lib/aarch64-linux-gnu/ModemManager/ +cd build +checkinstall -yD --install=no --fstrans=no --pkgname=modemmanager /tmp/meson-install/meson-install +mv modemmanager*.deb /tmp/modemmanager.deb diff --git a/userspace/compile-pyqt5.sh b/userspace/compile-pyqt5.sh index d7de955..92d18c2 100755 --- a/userspace/compile-pyqt5.sh +++ b/userspace/compile-pyqt5.sh @@ -1,12 +1,31 @@ #!/bin/bash -e +VERSION=5.15.9 + +# Install build dependencies +apt-get update && apt-get install -yq --no-install-recommends \ + libdbus-1-3 \ + libgles-dev \ + libglib2.0-dev \ + libharfbuzz-dev \ + libpcre2-dev \ + libpng-dev \ + libpulse-dev + +apt-get -o Dpkg::Options::="--force-overwrite" install -yq \ + /tmp/agnos/qt-5.12.8.deb \ + /tmp/agnos/libwayland-1.9.0-1.deb \ + /tmp/agnos/libicu66_66.1-2ubuntu2.1_arm64.deb \ + /tmp/agnos/libssl1.1_1.1.1f-1ubuntu2.22_arm64.deb \ + /tmp/agnos/libffi6_3.2.1-8_arm64.deb + source /usr/local/venv/bin/activate # Build PyQt5 wheel cd /tmp -wget https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-5.15.9.tar.gz -tar xf PyQt5-5.15.9.tar.gz -cd PyQt5-5.15.9 +wget https://files.pythonhosted.org/packages/5c/46/b4b6eae1e24d9432905ef1d4e7c28b6610e28252527cdc38f2a75997d8b5/PyQt5-${VERSION}.tar.gz +tar xf PyQt5-${VERSION}.tar.gz +cd PyQt5-${VERSION} export MAKEFLAGS="-j$(nproc)" pip wheel -w . --verbose --config-settings --confirm-license= . diff --git a/userspace/compile-qtwayland5.sh b/userspace/compile-qtwayland5.sh index de33571..33cac04 100755 --- a/userspace/compile-qtwayland5.sh +++ b/userspace/compile-qtwayland5.sh @@ -1,9 +1,35 @@ #!/bin/bash -e +VERSION=5.12.9 + +# Install build requirements +dpkg --add-architecture armhf + +apt-get update && apt-get install -yq --no-install-recommends \ + libc6:armhf \ + libdbus-1-3 \ + libegl-dev \ + libexpat1:armhf \ + libfontconfig-dev \ + libfreetype6 \ + libgles-dev \ + libglib2.0-dev \ + libharfbuzz-dev \ + libpcre2-dev \ + libwayland-dev \ + libxcomposite-dev \ + libxkbcommon-dev \ + zlib1g-dev + +apt-get -o Dpkg::Options::="--force-overwrite" install -yq \ + /tmp/agnos/qt-5.12.8.deb \ + /tmp/agnos/libwayland-1.9.0-1.deb \ + /tmp/agnos/libicu66_66.1-2ubuntu2.1_arm64.deb \ + /tmp/agnos/libssl1.1_1.1.1f-1ubuntu2.22_arm64.deb \ + /tmp/agnos/libffi6_3.2.1-8_arm64.deb + # Patched qtwayland that outputs a fixed screen size # Clone qtwayland submodule, checkout, apply patch, qmake, make -VERSION=5.12.9 - cd /tmp git clone --branch v${VERSION} --depth 1 https://github.com/qt/qtwayland.git cd qtwayland