From e5fae93540fe157f0c9ed045fb279125e9cdc10d Mon Sep 17 00:00:00 2001 From: Andrei Radulescu Date: Thu, 25 Jul 2024 20:47:56 +0300 Subject: [PATCH] Build kernel in docker (#247) * build kernel in container * remove ci deps * cleaning * more cleaning * even more cleaning * README.md update for masOS Case-sensitive required volume * README.md more info about macOS * README.md more info about macOS * README.md more info about macOS - M-series vs Intel * README.md more info about macOS - M-series vs Intel * extract_tools should install and pull lfs no matter the ARCH * raise attention on macOS Case-sensitive volume * --privileged not needed in build_kernel.sh * remove docker and git checks * set -e * always build image * updated host user in container * also works with sudo/root * removed deps in ci since they are not needed anymore * agnos-meta-builder * show macOS support only if current path is not on a Case-sensitive APFS volume * more succint * just exit * fixing caching issues * cache kernel out * cleaning * debug out * ccache-action no needed anymore * debug * rebuild * remove out cache * revert ccache action * create-symlink not needed since it's building in the container * simpler ccache key name * cleaning * rebuild * git ignore .ccache --------- Co-authored-by: Adeeb Shihadeh --- .github/workflows/build.yaml | 15 +--- .gitignore | 1 + Dockerfile.builder | 35 +++++++++ Dockerfile.sparsify | 20 ------ README.md | 12 ++++ build_kernel.sh | 135 +++++++++++++++++++++++------------ build_system.sh | 9 ++- tools/extract_tools.sh | 38 +++++----- 8 files changed, 163 insertions(+), 102 deletions(-) create mode 100644 Dockerfile.builder delete mode 100644 Dockerfile.sparsify diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 40e41e6..7d07db0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,7 +36,7 @@ jobs: fi echo EOF } | tee -a $GITHUB_ENV - + - name: Get kernel submodule ref id: kernel-submodule run: echo "ref=$(git ls-tree HEAD | awk '$4 == "agnos-kernel-sdm845"' | awk '{print $3}')" | tee -a $GITHUB_OUTPUT @@ -51,17 +51,8 @@ jobs: - name: ccache uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92 with: - create-symlink: true - key: kernel-ccache-${{ steps.kernel-submodule.outputs.ref }} - restore-keys: kernel-ccache- - - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y bc img2simg - - - name: Install python2 - uses: gabrielfalcao/pyenv-action@2f49ca7587f9d0663d13f1147b78d3361417eaf7 - with: - default: '2.7.18' + key: kernel-${{ steps.kernel-submodule.outputs.ref }} + restore-keys: kernel- - name: Build kernel run: ./build_kernel.sh diff --git a/.gitignore b/.gitignore index e47228c..8fe6efc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ output/ build/ tmp/ qdl/ +.ccache/ .qemu_registered .vscode diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 0000000..7b661fc --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,35 @@ +FROM ubuntu:20.04 + +ARG UNAME +ARG UID +ARG GID + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python2 \ + build-essential \ + libssl-dev \ + bc \ + python-is-python2 \ + openssl \ + ccache \ + android-sdk-libsparse-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \ + userdel -r `getent passwd ${UID} | cut -d : -f 1` > /dev/null 2>&1; \ + groupdel -f `getent group ${GID} | cut -d : -f 1` > /dev/null 2>&1; \ + groupadd -g ${GID} -o ${UNAME} && \ + useradd -u $UID -g $GID ${UNAME} \ +;fi + +RUN CCACHE_PATH=$(which ccache) && \ + ln -s $CCACHE_PATH /usr/local/bin/gcc && \ + ln -s $CCACHE_PATH /usr/local/bin/g++ && \ + ln -s $CCACHE_PATH /usr/local/bin/cc && \ + ln -s $CCACHE_PATH /usr/local/bin/c++ && \ + ln -s $CCACHE_PATH /usr/local/bin/clang + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/Dockerfile.sparsify b/Dockerfile.sparsify deleted file mode 100644 index c3d4f30..0000000 --- a/Dockerfile.sparsify +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:24.04 - -ARG UNAME -ARG UID -ARG GID - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y android-sdk-libsparse-utils && \ - rm -rf /var/lib/apt/lists/* - -RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \ - userdel -r `getent passwd ${UID} | cut -d : -f 1` > /dev/null 2>&1; \ - groupdel -f `getent group ${GID} | cut -d : -f 1` > /dev/null 2>&1; \ - groupadd -g ${GID} -o ${UNAME} && \ - useradd -u $UID -g $GID ${UNAME}; \ - fi - -ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/README.md b/README.md index 729ba4c..a744cae 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,18 @@ Validating changes: * [CI](https://github.com/commaai/agnos-builder/blob/master/.github/workflows/build.yaml) ensures the kernel and system builds work (and pushes the images for you to download) * [this](https://github.com/commaai/agnos-builder/blob/master/internal/README.md) is the checklist we go through before shipping new AGNOS releases to openpilot +### macOS + +Building the kernel on macOS requires `agnos-builder` to be in a [Case-sensitive](https://support.apple.com/lv-lv/guide/disk-utility/dsku19ed921c/mac) filesystem. + +Use this to set it up: +``` +diskutil apfs addVolume "Case-sensitive APFS" agnos +``` +* replace `` with the `synthesized` disk in `diskutil list` that includes your main volume (e.g. Macintosh HD) - usually `disk3` on M-series Macs, `disk1` on Intel Macs +* `agnos` is the name of the new volume - can be replaced with an arbitrary name +* the volume is mounted automatically in `/Volumes/agnos` + ## Contributing Join us in the `#dev-agnos` channel on [Discord](https://discord.comma.ai). diff --git a/build_kernel.sh b/build_kernel.sh index 6df7b85..7fcfbc1 100755 --- a/build_kernel.sh +++ b/build_kernel.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/bin/bash +set -e DEFCONFIG=tici_defconfig @@ -10,62 +11,102 @@ OUTPUT_DIR=$DIR/output BOOT_IMG=./boot.img cd $DIR +if [[ "$(uname)" == 'Darwin' ]]; then + BASE_VOLUME_PATH=$(echo $DIR | grep -o "^/Volumes/[^/]*" || echo "/") + if ! diskutil info -plist $BASE_VOLUME_PATH | grep -q "Case-sensitive APFS"; then + echo "--------------- macOS support ---------------" + echo "Ensure you are in a Case-sensitive APFS volume to build the AGNOS kernel." + echo "https://github.com/commaai/agnos-builder?tab=readme-ov-file#macos" + echo "-------------------------------------------------" + exit 1 + fi +fi + +# Setup kernel build container +echo "Building agnos-meta-builder docker image" +docker build -f Dockerfile.builder -t agnos-meta-builder $DIR \ + --build-arg UNAME=$(id -nu) \ + --build-arg UID=$(id -u) \ + --build-arg GID=$(id -g) +echo "Starting agnos-meta-builder container" +CONTAINER_ID=$(docker run -d -v $DIR:$DIR -w $DIR agnos-meta-builder) + +# Cleanup container on exit +trap "echo \"Cleaning up container:\"; \ +docker container rm -f $CONTAINER_ID" EXIT + # Clone kernel if not done already if git submodule status --cached agnos-kernel-sdm845/ | grep "^-"; then + echo "Cloning agnos-kernel-sdm845" git submodule update --init agnos-kernel-sdm845 fi -cd agnos-kernel-sdm845 -# Build parameters -ARCH=$(uname -m) -if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "aarch64" ]; then - export CROSS_COMPILE=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu- - export CC=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-gcc - export LD=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-ld.bfd +$DIR/tools/extract_tools.sh - $DIR/tools/extract_tools.sh -fi -export ARCH=arm64 +build_kernel() { + cd agnos-kernel-sdm845 -# these do anything? -export KCFLAGS="-w" + # Build parameters + ARCH=$(uname -m) + if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "aarch64" ]; then + export CROSS_COMPILE=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu- + export CC=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-gcc + export LD=$TOOLS/aarch64-linux-gnu-gcc/bin/aarch64-linux-gnu-ld.bfd + fi -# Load defconfig and build kernel -echo "-- First make --" -make $DEFCONFIG O=out -echo "-- Second make: $(nproc --all) cores --" -make -j$(nproc --all) O=out # Image.gz-dtb + # Build arm64 arch + export ARCH=arm64 -# Turn on if you want perf -# LDFLAGS=-static make -j$(nproc --all) -C tools/perf + # Set ccache dir + export CCACHE_DIR=$DIR/.ccache -# Copy over Image.gz-dtb -mkdir -p $TMP_DIR -cd $TMP_DIR -cp $DIR/agnos-kernel-sdm845/out/arch/arm64/boot/Image.gz-dtb . + # Avoid LINUX_COMPILE_HOST to change on every run thus invalidating cache + # https://patchwork.kernel.org/project/linux-kbuild/patch/1302015561-21047-8-git-send-email-mmarek@suse.cz/ + export KBUILD_BUILD_HOST="docker" -# Make boot image -$TOOLS/mkbootimg \ - --kernel Image.gz-dtb \ - --ramdisk /dev/null \ - --cmdline "console=ttyMSM0,115200n8 quiet loglevel=3 earlycon=msm_geni_serial,0xA84000 androidboot.hardware=qcom androidboot.console=ttyMSM0 ehci-hcd.park=3 lpm_levels.sleep_disabled=1 service_locator.enable=1 androidboot.selinux=permissive firmware_class.path=/lib/firmware/updates net.ifnames=0 dyndbg=\"\"" \ - --pagesize 4096 \ - --base 0x80000000 \ - --kernel_offset 0x8000 \ - --ramdisk_offset 0x8000 \ - --tags_offset 0x100 \ - --output $BOOT_IMG.nonsecure + # Disable all warnings + export KCFLAGS="-w" -# le signing -openssl dgst -sha256 -binary $BOOT_IMG.nonsecure > $BOOT_IMG.sha256 -openssl pkeyutl -sign -in $BOOT_IMG.sha256 -inkey $DIR/vble-qti.key -out $BOOT_IMG.sig -pkeyopt digest:sha256 -pkeyopt rsa_padding_mode:pkcs1 -dd if=/dev/zero of=$BOOT_IMG.sig.padded bs=2048 count=1 -dd if=$BOOT_IMG.sig of=$BOOT_IMG.sig.padded conv=notrunc -cat $BOOT_IMG.nonsecure $BOOT_IMG.sig.padded > $BOOT_IMG + # Load defconfig and build kernel + echo "-- First make --" + make $DEFCONFIG O=out + echo "-- Second make: $(nproc --all) cores --" + make -j$(nproc --all) O=out # Image.gz-dtb -# Copy to output dir -mkdir -p $OUTPUT_DIR -mv $BOOT_IMG $OUTPUT_DIR/ -cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/snd-soc-sdm845.ko $OUTPUT_DIR/ -cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/codecs/snd-soc-wcd9xxx.ko $OUTPUT_DIR/ -cp $DIR/agnos-kernel-sdm845/out/drivers/staging/qcacld-3.0/wlan.ko $OUTPUT_DIR/ + # Turn on if you want perf + # LDFLAGS=-static make -j$(nproc --all) -C tools/perf + + # Copy over Image.gz-dtb + mkdir -p $TMP_DIR + cd $TMP_DIR + cp $DIR/agnos-kernel-sdm845/out/arch/arm64/boot/Image.gz-dtb . + + # Make boot image + $TOOLS/mkbootimg \ + --kernel Image.gz-dtb \ + --ramdisk /dev/null \ + --cmdline "console=ttyMSM0,115200n8 quiet loglevel=3 earlycon=msm_geni_serial,0xA84000 androidboot.hardware=qcom androidboot.console=ttyMSM0 ehci-hcd.park=3 lpm_levels.sleep_disabled=1 service_locator.enable=1 androidboot.selinux=permissive firmware_class.path=/lib/firmware/updates net.ifnames=0 dyndbg=\"\"" \ + --pagesize 4096 \ + --base 0x80000000 \ + --kernel_offset 0x8000 \ + --ramdisk_offset 0x8000 \ + --tags_offset 0x100 \ + --output $BOOT_IMG.nonsecure + + # le signing + openssl dgst -sha256 -binary $BOOT_IMG.nonsecure > $BOOT_IMG.sha256 + openssl pkeyutl -sign -in $BOOT_IMG.sha256 -inkey $DIR/vble-qti.key -out $BOOT_IMG.sig -pkeyopt digest:sha256 -pkeyopt rsa_padding_mode:pkcs1 + dd if=/dev/zero of=$BOOT_IMG.sig.padded bs=2048 count=1 + dd if=$BOOT_IMG.sig of=$BOOT_IMG.sig.padded conv=notrunc + cat $BOOT_IMG.nonsecure $BOOT_IMG.sig.padded > $BOOT_IMG + + # Copy to output dir + mkdir -p $OUTPUT_DIR + mv $BOOT_IMG $OUTPUT_DIR/ + cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/snd-soc-sdm845.ko $OUTPUT_DIR/ + cp $DIR/agnos-kernel-sdm845/out/techpack/audio/asoc/codecs/snd-soc-wcd9xxx.ko $OUTPUT_DIR/ + cp $DIR/agnos-kernel-sdm845/out/drivers/staging/qcacld-3.0/wlan.ko $OUTPUT_DIR/ +} + +# Run build_kernel in container +docker exec -u $(id -nu) $CONTAINER_ID bash -c "set -e; export DEFCONFIG=$DEFCONFIG DIR=$DIR TOOLS=$TOOLS TMP_DIR=$TMP_DIR OUTPUT_DIR=$OUTPUT_DIR BOOT_IMG=$BOOT_IMG; $(declare -f build_kernel); build_kernel" diff --git a/build_system.sh b/build_system.sh index 0cde0dd..5588614 100755 --- a/build_system.sh +++ b/build_system.sh @@ -49,14 +49,13 @@ echo "Creating agnos-builder container" CONTAINER_ID=$(docker container create --entrypoint /bin/bash agnos-builder:latest) # Setup mount container for macOS and CI support (namespace.so) -echo "Building agnos-mount docker image" -docker build -f Dockerfile.sparsify -t agnos-mount $DIR \ +echo "Building agnos-meta-builder docker image" +docker build -f Dockerfile.builder -t agnos-meta-builder $DIR \ --build-arg UNAME=$(id -nu) \ --build-arg UID=$(id -u) \ --build-arg GID=$(id -g) - -echo "Starting agnos-mount container" -MOUNT_CONTAINER_ID=$(docker run -d --privileged -v $DIR:$DIR agnos-mount) +echo "Starting agnos-meta-builder container" +MOUNT_CONTAINER_ID=$(docker run -d --privileged -v $DIR:$DIR agnos-meta-builder) # Cleanup containers on possible exit trap "echo \"Cleaning up containers:\"; \ diff --git a/tools/extract_tools.sh b/tools/extract_tools.sh index edfc28c..ba86f79 100755 --- a/tools/extract_tools.sh +++ b/tools/extract_tools.sh @@ -1,7 +1,5 @@ #!/bin/bash -e -echo "Extracting tools..." - git lfs &> /dev/null || { echo "ERROR: git lfs not installed" exit 1 @@ -13,11 +11,6 @@ ROOT=$DIR/.. cd $DIR -LINARO_GCC=aarch64-linux-gnu-gcc -GOOGLE_GCC_4_9=aarch64-linux-android-4.9 -EDK2_LLVM=llvm-arm-toolchain-ship -SEC_IMAGE=SecImage - # grep for `-`, which stands for LFS pointer git lfs ls-files | awk '{print $2}' | grep "-" &>/dev/null && { echo "Pulling git lfs objects..." @@ -27,23 +20,32 @@ git lfs ls-files | awk '{print $2}' | grep "-" &>/dev/null && { cd $DIR } +LINARO_GCC=aarch64-linux-gnu-gcc +GOOGLE_GCC_4_9=aarch64-linux-android-4.9 +EDK2_LLVM=llvm-arm-toolchain-ship +SEC_IMAGE=SecImage + LINARO_GCC_TARBALL=$LINARO_GCC.tar.gz GOOGLE_GCC_4_9_TARBALL=$GOOGLE_GCC_4_9.tar.gz EDK2_LLVM_TARBALL=$EDK2_LLVM.tar.gz SEC_IMAGE_TARBALL=$SEC_IMAGE.tar.gz -if [ ! -d $LINARO_GCC ]; then - tar -xzf $LINARO_GCC_TARBALL &>/dev/null -fi +ARCH=$(uname -m) +if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "aarch64" ]; then + echo "Extracting tools..." + if [ ! -d $LINARO_GCC ]; then + tar -xzf $LINARO_GCC_TARBALL &>/dev/null + fi -if [ ! -d $GOOGLE_GCC_4_9 ]; then - tar -xzf $GOOGLE_GCC_4_9_TARBALL &>/dev/null -fi + if [ ! -d $GOOGLE_GCC_4_9 ]; then + tar -xzf $GOOGLE_GCC_4_9_TARBALL &>/dev/null + fi -if [ ! -d $EDK2_LLVM ]; then - tar -xzf $EDK2_LLVM_TARBALL &>/dev/null -fi + if [ ! -d $EDK2_LLVM ]; then + tar -xzf $EDK2_LLVM_TARBALL &>/dev/null + fi -if [ ! -d $SEC_IMAGE ]; then - tar -xzf $SEC_IMAGE_TARBALL &>/dev/null + if [ ! -d $SEC_IMAGE ]; then + tar -xzf $SEC_IMAGE_TARBALL &>/dev/null + fi fi