diff --git a/build_system.sh b/build_system.sh index 05eee17..0d5f5f6 100755 --- a/build_system.sh +++ b/build_system.sh @@ -16,7 +16,8 @@ OUTPUT_DIR="$DIR/output" ROOTFS_DIR="$BUILD_DIR/agnos-rootfs" ROOTFS_IMAGE="$BUILD_DIR/system.img.raw" ROOTFS_IMAGE_SIZE=10G -SPARSE_IMAGE="$BUILD_DIR/system.img" +SPARSE_IMAGE="$OUTPUT_DIR/system.img" +SKIP_CHUNKS_IMAGE="$OUTPUT_DIR/system-skip-chunks.img" # Create temp dir if non-existent mkdir -p $BUILD_DIR $OUTPUT_DIR @@ -87,7 +88,13 @@ sudo umount -l $ROOTFS_DIR # Sparsify echo "Sparsify image" -img2simg $ROOTFS_IMAGE $SPARSE_IMAGE -mv $SPARSE_IMAGE $OUTPUT_DIR +TMP_SPARSE="$(mktemp)" +img2simg $ROOTFS_IMAGE $TMP_SPARSE +mv $TMP_SPARSE $SPARSE_IMAGE + +# Make image with skipped chunks +TMP_SKIP="$(mktemp)" +$DIR/tools/simg2dontcare.py $SPARSE_IMAGE $TMP_SKIP +mv $TMP_SKIP $SKIP_CHUNKS_IMAGE echo "Done!" diff --git a/flash_system.sh b/flash_system.sh index f449106..9719924 100755 --- a/flash_system.sh +++ b/flash_system.sh @@ -1,29 +1,12 @@ #!/bin/bash -e +set -e -# Config -OUTPUT_DIR="output" -ROOTFS_IMAGE="system.img" - -# Log colors -GREEN="\033[0;32m" -NO_COLOR='\033[0m' - -# Make sure we're in the correct spot DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd $DIR -# Flash system fastboot --set-active=a - -if [ -d $HOME/openpilot/provisioning ]; then - DIMG=$(mktemp) - $DIR/tools/simg2dontcare.py $OUTPUT_DIR/$ROOTFS_IMAGE $DIMG - fastboot erase system_a - fastboot flash system_a $DIMG -else - fastboot flash system_a $OUTPUT_DIR/$ROOTFS_IMAGE -fi - +fastboot erase system_a +fastboot flash system_a $OUTPUT_DIR/system-skip-chunks.img fastboot continue -echo -e "${GREEN}Done!" +echo "Done!" diff --git a/internal/build_and_package.sh b/internal/build_and_package.sh index ff6f8f8..2ec5bf3 100755 --- a/internal/build_and_package.sh +++ b/internal/build_and_package.sh @@ -19,5 +19,5 @@ fi ./build_kernel.sh ./build_system.sh -internal/package_ota.sh +internal/package_ota.py internal/ota_push.sh staging diff --git a/internal/ota_push.sh b/internal/ota_push.sh index f329059..443fcba 100755 --- a/internal/ota_push.sh +++ b/internal/ota_push.sh @@ -39,6 +39,14 @@ process_file() { local NAME=$1 local HASH_RAW=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .hash_raw") upload_file "$NAME-$HASH_RAW.img.xz" + upload_file "$NAME-$HASH_RAW.img.gz" + + local ALT_URL=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .alt.url") + if [ "$ALT_URL" != "null" ]; then + local ALT_FILE_NAME=$(basename $ALT_URL) + upload_file $ALT_FILE_NAME + upload_file {$ALT_FILE_NAME%.xz}.gz + fi # if [ "$NAME" == "system" ]; then # local CAIBX_FILE_NAME="system-$HASH_RAW.caibx" diff --git a/internal/package_ota.py b/internal/package_ota.py new file mode 100755 index 0000000..f6ad7f1 --- /dev/null +++ b/internal/package_ota.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import json +import os +import subprocess +from copy import deepcopy +from pathlib import Path +from tempfile import NamedTemporaryFile + +ROOT = Path(__file__).parent.parent +OUTPUT_DIR = ROOT / "output" +OTA_OUTPUT_DIR = OUTPUT_DIR / "ota" +FIRMWARE_DIR = ROOT / "agnos-firmware" + +AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate") +AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging") + + +def checksum(fn): + return subprocess.check_output(["sha256sum", fn]).decode().split()[0] + + +def compress(fin, fout) -> None: + subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True) + subprocess.check_call(f"gzip -vc {fin} > {fout.with_suffix('.gz')}", shell=True) + + +def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None): + print(name) + hash_raw = hash = checksum(fn) + size = fn.stat().st_size + print(f" {size} bytes, hash {hash}") + + if sparse: + with NamedTemporaryFile() as tmp_f: + print(" converting sparse image to raw") + subprocess.check_call(["simg2img", fn, tmp_f.name]) + hash_raw = checksum(tmp_f.name) + size = Path(tmp_f.name).stat().st_size + print(f" {size} bytes, hash {hash} (raw)") + + print(" compressing") + xz_fn = OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img.xz" + compress(fn, xz_fn) + + ret = { + "name": name, + "url": "{remote_url}/" + xz_fn.name, + "hash": hash, + "hash_raw": hash_raw, + "size": size, + "sparse": sparse, + "full_check": full_check, + "has_ab": has_ab, + } + + if alt is not None: + print(" calculating alt") + alt_hash = checksum(alt) + alt_size = alt.stat().st_size + print(f" {alt_size} bytes, hash {alt_hash} (alt)") + + print(" compressing alt") + alt_xz_fn = OTA_OUTPUT_DIR / f"{alt.stem}-{hash_raw}.img.xz" + compress(alt, alt_xz_fn) + + ret["alt"] = { + "hash": alt_hash, + "url": "{remote_url}/" + alt_xz_fn.name, + } + + return ret + + +if __name__ == "__main__": + OTA_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + files = [ + process_file(OUTPUT_DIR / "boot.img", "boot"), + process_file(FIRMWARE_DIR / "abl.bin", "abl"), + process_file(FIRMWARE_DIR / "xbl.bin", "xbl"), + process_file(FIRMWARE_DIR / "xbl_config.bin", "xbl_config"), + process_file(FIRMWARE_DIR / "devcfg.bin", "devcfg"), + process_file(FIRMWARE_DIR / "aop.bin", "aop"), + process_file(OUTPUT_DIR / "system.img", "system", sparse=True, full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"), + ] + configs = [ + (AGNOS_UPDATE_URL, "ota.json"), + (AGNOS_STAGING_UPDATE_URL, "ota-staging.json"), + ] + + for remote_url, output_fn in configs: + processed_files = [] + for f in deepcopy(files): + f["url"] = f["url"].format(remote_url=remote_url) + if "alt" in f: + f["alt"]["url"] = f["alt"]["url"].format(remote_url=remote_url) + processed_files.append(f) + + with open(OTA_OUTPUT_DIR / output_fn, "w") as out: + json.dump(processed_files, out, indent=2) + + print("Done") diff --git a/internal/package_ota.sh b/internal/package_ota.sh deleted file mode 100755 index 01d2da8..0000000 --- a/internal/package_ota.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -e - -# Make sure we're in the correct spot -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - -ROOT="$DIR/../" -TMP_DIR="/tmp/agnos-builder-tmp" -OUTPUT_DIR="$ROOT/output" -OTA_OUTPUT_DIR="$OUTPUT_DIR/ota" -FIRMWARE_DIR="$ROOT/agnos-firmware" - -AGNOS_UPDATE_URL=${AGNOS_UPDATE_URL:-https://commadist.azureedge.net/agnosupdate} -AGNOS_STAGING_UPDATE_URL=${AGNOS_STAGING_UPDATE_URL:-https://commadist.azureedge.net/agnosupdate-staging} -OUTPUT_JSON="$OTA_OUTPUT_DIR/ota.json" -OUTPUT_STAGING_JSON="$OTA_OUTPUT_DIR/ota-staging.json" - -# Make sure archive dir is empty -rm -rf $OTA_OUTPUT_DIR && mkdir -p $OTA_OUTPUT_DIR - -process_file() { - local FILE=$1 - local NAME=$2 - local SPARSE=${3:-false} - local FULL_CHECK=${4:-true} - local HAS_AB=${5:-true} - - echo "Hashing $NAME..." - local HASH=$(sha256sum $FILE | cut -c 1-64) - local SIZE=$(wc -c < $FILE) - echo " $HASH ($SIZE bytes)" - - local HASH_RAW=$HASH - if [ "$NAME" == "system" ]; then - echo "Converting system to raw..." - local FILE_RAW=/tmp/system.img.raw - simg2img $FILE $FILE_RAW - - echo "Hashing system raw..." - HASH_RAW=$(sha256sum $FILE_RAW | cut -c 1-64) - SIZE=$(wc -c < $FILE_RAW) - echo " $HASH_RAW ($SIZE bytes) (raw)" - - - # echo "Creating system casync files" - # casync make --compression=xz --store $OTA_OUTPUT_DIR/system-$HASH_RAW $OTA_OUTPUT_DIR/system-$HASH_RAW.caibx $FILE_RAW - - rm $FILE_RAW - fi - - echo "Compressing $NAME..." - local FILENAME=$NAME-$HASH_RAW.img.xz - local ARCHIVE=$OTA_OUTPUT_DIR/$FILENAME - xz -vc $FILE > $ARCHIVE - - cat < /dev/null - { - "name": "$NAME", - "hash": "$HASH", - "hash_raw": "$HASH_RAW", - "size": $SIZE, - "sparse": $SPARSE, - "full_check": $FULL_CHECK, - "has_ab": $HAS_AB, -EOF - - cat <> $OUTPUT_JSON - "url": "$AGNOS_UPDATE_URL/$FILENAME" -EOF - cat <> $OUTPUT_STAGING_JSON - "url": "$AGNOS_STAGING_UPDATE_URL/$FILENAME" -EOF - -# if [ $NAME == "system" ]; then -# cat <> $OUTPUT_JSON -# "casync_caibx": "$AGNOS_UPDATE_URL/$NAME-$HASH_RAW.caibx", -# "casync_store": "$AGNOS_UPDATE_URL/$NAME-$HASH_RAW" -# EOF -# cat <> $OUTPUT_STAGING_JSON -# "casync_caibx": "$AGNOS_STAGING_UPDATE_URL/$NAME-$HASH_RAW.caibx", -# "casync_store": "$AGNOS_STAGING_UPDATE_URL/$NAME-$HASH_RAW" -# EOF -# fi - - cat < /dev/null - }, -EOF -} - -cd $ROOT - -echo "[" > $OUTPUT_JSON -echo "[" > $OUTPUT_STAGING_JSON - -process_file "$OUTPUT_DIR/boot.img" boot -process_file "$FIRMWARE_DIR/abl.bin" abl -process_file "$FIRMWARE_DIR/xbl.bin" xbl -process_file "$FIRMWARE_DIR/xbl_config.bin" xbl_config -process_file "$FIRMWARE_DIR/devcfg.bin" devcfg -process_file "$FIRMWARE_DIR/aop.bin" aop -process_file "$OUTPUT_DIR/system.img" system true false true - -# remove trailing comma -sed -i "$ s/.$//" $OUTPUT_JSON $OUTPUT_STAGING_JSON - -echo "]" >> $OUTPUT_JSON -echo "]" >> $OUTPUT_STAGING_JSON - -echo "Done!" diff --git a/internal/pull_ota.sh b/internal/pull_ota.sh new file mode 100755 index 0000000..aafc54b --- /dev/null +++ b/internal/pull_ota.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e + +# Make sure we're in the correct directory +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + +# Constants +OUTPUT_DIR="$DIR/../output" + +if [ "$1" == "" ]; then + echo "Supply the URL to the OTA JSON as first argument!" + exit 1 +fi + +OTA_JSON=$(mktemp) +wget $1 -O $OTA_JSON + +mkdir -p $OUTPUT_DIR +cd $OUTPUT_DIR + +download_image() { + local name=$1 + local alt=${2:-""} + + local url=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$name\") | $alt.url") + if [ "$url" == "null" ]; then + return + fi + + local hash_raw=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$name\") | .hash_raw") + local file_name=$(basename $url .xz) + file_name=${file_name//-$hash_raw/} + + echo "Downloading $file_name..." + curl $url | xz -d > $file_name +} + +for name in boot system; do + download_image $name + download_image $name ".alt" +done + +echo "Done!"