From 5e5a350981d28720fb5d325ecaf14b3a1f566a5e Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Sun, 2 Feb 2025 19:55:34 -0800 Subject: [PATCH] manifest with all partitions (#443) * all * ota * all manif * ab * remove these * hash * userdata * ontheflyisthebest * on the fly * dont need * remove * handle fill * comment * all * all img * rename * read * sparse image * tool * no root * userdata * userdata * need this unfort * fix * fix * nop * remove * nop * this too * ota check * better for push * fix * again * good now --- Dockerfile.builder | 1 + build_system.sh | 4 +- firmware.json | 52 ------- firmware/.gitattributes | 1 + firmware/__init__.py | 51 ------ firmware/{abl.bin => abl.img} | 0 firmware/{aop.bin => aop.img} | 0 firmware/{bluetooth.bin => bluetooth.img} | 0 firmware/{cache.bin => cache.img} | 0 firmware/{cmnlib.bin => cmnlib.img} | 0 firmware/{cmnlib64.bin => cmnlib64.img} | 0 firmware/{devcfg.bin => devcfg.img} | 0 firmware/{devinfo.bin => devinfo.img} | 0 firmware/{dsp.bin => dsp.img} | 0 firmware/{gpt_main_0.bin => gpt_main_0.img} | 0 firmware/{gpt_main_1.bin => gpt_main_1.img} | 0 firmware/{gpt_main_2.bin => gpt_main_2.img} | 0 firmware/{gpt_main_3.bin => gpt_main_3.img} | 0 firmware/{gpt_main_4.bin => gpt_main_4.img} | 0 firmware/{gpt_main_5.bin => gpt_main_5.img} | 0 firmware/{hyp.bin => hyp.img} | 0 firmware/{keymaster.bin => keymaster.img} | 0 firmware/{limits.bin => limits.img} | 0 firmware/{logfs.bin => logfs.img} | 0 firmware/{modem.bin => modem.img} | 0 firmware/{persist.bin => persist.img} | 0 firmware/{qupfw.bin => qupfw.img} | 0 firmware/{splash.bin => splash.img} | 0 firmware/{splash_cc.bin => splash_cc.img} | 0 firmware/{storsec.bin => storsec.img} | 0 firmware/{systemrw.bin => systemrw.img} | 0 firmware/{tz.bin => tz.img} | 0 firmware/{xbl.bin => xbl.img} | 0 firmware/{xbl_config.bin => xbl_config.img} | 0 flash_all.sh | 4 +- scripts/build_and_package.sh | 1 + scripts/build_userdata.sh | 38 +++++ scripts/ota_push.sh | 4 +- scripts/package_ota.py | 163 ++++++++++++++++---- 39 files changed, 182 insertions(+), 137 deletions(-) delete mode 100644 firmware.json delete mode 100644 firmware/__init__.py rename firmware/{abl.bin => abl.img} (100%) rename firmware/{aop.bin => aop.img} (100%) rename firmware/{bluetooth.bin => bluetooth.img} (100%) rename firmware/{cache.bin => cache.img} (100%) rename firmware/{cmnlib.bin => cmnlib.img} (100%) rename firmware/{cmnlib64.bin => cmnlib64.img} (100%) rename firmware/{devcfg.bin => devcfg.img} (100%) rename firmware/{devinfo.bin => devinfo.img} (100%) rename firmware/{dsp.bin => dsp.img} (100%) rename firmware/{gpt_main_0.bin => gpt_main_0.img} (100%) rename firmware/{gpt_main_1.bin => gpt_main_1.img} (100%) rename firmware/{gpt_main_2.bin => gpt_main_2.img} (100%) rename firmware/{gpt_main_3.bin => gpt_main_3.img} (100%) rename firmware/{gpt_main_4.bin => gpt_main_4.img} (100%) rename firmware/{gpt_main_5.bin => gpt_main_5.img} (100%) rename firmware/{hyp.bin => hyp.img} (100%) rename firmware/{keymaster.bin => keymaster.img} (100%) rename firmware/{limits.bin => limits.img} (100%) rename firmware/{logfs.bin => logfs.img} (100%) rename firmware/{modem.bin => modem.img} (100%) rename firmware/{persist.bin => persist.img} (100%) rename firmware/{qupfw.bin => qupfw.img} (100%) rename firmware/{splash.bin => splash.img} (100%) rename firmware/{splash_cc.bin => splash_cc.img} (100%) rename firmware/{storsec.bin => storsec.img} (100%) rename firmware/{systemrw.bin => systemrw.img} (100%) rename firmware/{tz.bin => tz.img} (100%) rename firmware/{xbl.bin => xbl.img} (100%) rename firmware/{xbl_config.bin => xbl_config.img} (100%) create mode 100755 scripts/build_userdata.sh diff --git a/Dockerfile.builder b/Dockerfile.builder index 206c2f2..e32c9be 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -18,6 +18,7 @@ RUN apt-get update && \ openssl \ ccache \ libcap2-bin \ + android-sdk-libsparse-utils \ && rm -rf /var/lib/apt/lists/* RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \ diff --git a/build_system.sh b/build_system.sh index c2f7862..a438836 100755 --- a/build_system.sh +++ b/build_system.sh @@ -136,7 +136,7 @@ exec_as_root bash -c "set -e; export ROOTFS_DIR=$ROOTFS_DIR GIT_HASH=$GIT_HASH; echo "Unmount filesystem" exec_as_root umount -l $ROOTFS_DIR -# Copy system image to output -cp $ROOTFS_IMAGE $OUT_IMAGE +# Sparsify system image +exec_as_user img2simg $ROOTFS_IMAGE $OUT_IMAGE echo "Done!" diff --git a/firmware.json b/firmware.json deleted file mode 100644 index 82c78ab..0000000 --- a/firmware.json +++ /dev/null @@ -1,52 +0,0 @@ -[ - { - "name": "xbl", - "url": "https://commadist.azureedge.net/agnosupdate/xbl-468f1ad6ab55e198647ff9191f91bd2918db9c0a3e27bae5673b4c5575c1254c.img.xz", - "hash": "468f1ad6ab55e198647ff9191f91bd2918db9c0a3e27bae5673b4c5575c1254c", - "hash_raw": "468f1ad6ab55e198647ff9191f91bd2918db9c0a3e27bae5673b4c5575c1254c", - "size": 3282256, - "sparse": false, - "full_check": true, - "has_ab": true - }, - { - "name": "abl", - "url": "https://commadist.azureedge.net/agnosupdate/abl-32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6.img.xz", - "hash": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6", - "hash_raw": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6", - "size": 274432, - "sparse": false, - "full_check": true, - "has_ab": true - }, - { - "name": "xbl_config", - "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-92b675dc2862ed15c732d91d9eb307d7e852e349217db8bee8f8829db543686b.img.xz", - "hash": "92b675dc2862ed15c732d91d9eb307d7e852e349217db8bee8f8829db543686b", - "hash_raw": "92b675dc2862ed15c732d91d9eb307d7e852e349217db8bee8f8829db543686b", - "size": 98124, - "sparse": false, - "full_check": true, - "has_ab": true - }, - { - "name": "devcfg", - "url": "https://commadist.azureedge.net/agnosupdate/devcfg-225b24ea7b1d2fee7f7d2da21386920ddacac2e33e9e938168436292f4eae180.img.xz", - "hash": "225b24ea7b1d2fee7f7d2da21386920ddacac2e33e9e938168436292f4eae180", - "hash_raw": "225b24ea7b1d2fee7f7d2da21386920ddacac2e33e9e938168436292f4eae180", - "size": 40336, - "sparse": false, - "full_check": true, - "has_ab": true - }, - { - "name": "aop", - "url": "https://commadist.azureedge.net/agnosupdate/aop-f0fcf7611d0890a72984f15a516dd37fa532dfcb70d428a8406838cf74ce23d5.img.xz", - "hash": "f0fcf7611d0890a72984f15a516dd37fa532dfcb70d428a8406838cf74ce23d5", - "hash_raw": "f0fcf7611d0890a72984f15a516dd37fa532dfcb70d428a8406838cf74ce23d5", - "size": 184364, - "sparse": false, - "full_check": true, - "has_ab": true - } -] \ No newline at end of file diff --git a/firmware/.gitattributes b/firmware/.gitattributes index 4edd5ac..ec07d0f 100644 --- a/firmware/.gitattributes +++ b/firmware/.gitattributes @@ -1 +1,2 @@ *.bin filter=lfs diff=lfs merge=lfs -text +*.img filter=lfs diff=lfs merge=lfs -text diff --git a/firmware/__init__.py b/firmware/__init__.py deleted file mode 100644 index aca86c1..0000000 --- a/firmware/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -# Structure: (lun, partition_name, start_sector, num_sectors, checksum) -PARTITION_TABLES = [ - (0, 'gpt_main_0', 0, 6, '8928A31FD9EE20F8703649F89833EBA9B55E84B6415E67799C777B163C95A0BD'), - (1, 'gpt_main_1', 0, 6, 'FE8EF7653DB588D7420A625920CA06927DFCB0ED8AFF3E3A1C74A52A24398BA6'), - (2, 'gpt_main_2', 0, 6, '5CCFC7240C8CBFA2F1A018A2E376CF274A6BAF858C9BFE71951D8E28CAB53C21'), - (3, 'gpt_main_3', 0, 6, 'C707979FA21E89519328F4F30C2B21C9C453401CA8303F914C1873D410A95159'), - (4, 'gpt_main_4', 0, 6, 'E9405DCD785DBE79412184E1894A9C51AB7DEB33BB612166C4C42A3D2BF42A0E'), - (5, 'gpt_main_5', 0, 6, '21AE965F05B2FA8D02E04F1EB74718F9779864F6EACDEB859757D6435E8CCCE3'), -] - -# Structure: (lun, partition_name, filename, checksum) -# All zero-filled partitions are not in this list, since the device gets erased anyways -QDL_FLASH_ARRAY = [ - (0, 'persist', 'persist.bin', '9814B07851292F510F3794B767489F38AB379A99F0EA75DC620AD2D3A496D54D'), - (0, 'systemrw', 'systemrw.bin', '8CE150CA38EF64A0885FC2FE816E5B63BAE8ADB4DF5D809C5B318E6996366C7E'), - (0, 'cache', 'cache.bin', 'EBFBAAA2F96DC4E5FEA4F126364E5BF5B3B44C12CBC753B62FDD8BAAB82F70B4'), - (1, 'xbl_a', 'xbl.bin', '7EA3D6A4AB5396D2C47E135B4E027EDDA270064914305CF5B98363779F53A890'), - (1, 'xbl_config_a', 'xbl_config.bin', '04E478D8C604CBA3DE571C56A846474D0DEA5270462C18034229123BAE2EA645'), - (2, 'xbl_b', 'xbl.bin', '7EA3D6A4AB5396D2C47E135B4E027EDDA270064914305CF5B98363779F53A890'), - (2, 'xbl_config_b', 'xbl_config.bin', '04E478D8C604CBA3DE571C56A846474D0DEA5270462C18034229123BAE2EA645'), - (4, 'abl_a', 'abl.bin', '262AC633281044C13FA861C244485E940BB92ADECCF14657C9160DF52C360AED'), - (4, 'abl_b', 'abl.bin', '262AC633281044C13FA861C244485E940BB92ADECCF14657C9160DF52C360AED'), - (4, 'aop_a', 'aop.bin', '29DA7A7F7973001B82DCBCCDA5069F2F8A27BD8850DC25F857BA7919AC06E83E'), - (4, 'aop_b', 'aop.bin', '29DA7A7F7973001B82DCBCCDA5069F2F8A27BD8850DC25F857BA7919AC06E83E'), - (4, 'bluetooth_a', 'bluetooth.bin', '9BB766D2D2CE0CC4491664B3010FE1EF62F8FFC1E362D55F78E48C4141F75533'), - (4, 'bluetooth_b', 'bluetooth.bin', '9BB766D2D2CE0CC4491664B3010FE1EF62F8FFC1E362D55F78E48C4141F75533'), - (4, 'cmnlib64_a', 'cmnlib64.bin', '1A876BD151BB9635F18719C4A17F953079DE6E11D3EAEC800968FC75669E0DC3'), - (4, 'cmnlib64_b', 'cmnlib64.bin', '1A876BD151BB9635F18719C4A17F953079DE6E11D3EAEC800968FC75669E0DC3'), - (4, 'cmnlib_a', 'cmnlib.bin', '63DF823E8A5FAE01D66CB2B8C20F0D2DDB5C5F2425E5D0992A64676273BA1C82'), - (4, 'cmnlib_b', 'cmnlib.bin', '63DF823E8A5FAE01D66CB2B8C20F0D2DDB5C5F2425E5D0992A64676273BA1C82'), - (4, 'devcfg_a', 'devcfg.bin', '8DE276D6179915B01B9568D18EB69EB18F1E0D06CA99AEC290080B005AA23612'), - (4, 'devcfg_b', 'devcfg.bin', '8DE276D6179915B01B9568D18EB69EB18F1E0D06CA99AEC290080B005AA23612'), - (4, 'devinfo', 'devinfo.bin', '143869C499A7E878FBEAB756E9C53074195770CC41D6D0D10E45C043141389A3'), - (4, 'dsp_a', 'dsp.bin', '4B15FBD2F45581F1553F33F01649E450B24AA19D5DEFF2AC7DCB16A534D9C248'), - (4, 'dsp_b', 'dsp.bin', '4B15FBD2F45581F1553F33F01649E450B24AA19D5DEFF2AC7DCB16A534D9C248'), - (4, 'hyp_a', 'hyp.bin', 'FF5ECE6A4E3D2B4D898C77FFE193FC8BBC8ACEBE78263996ECF52373D8088927'), - (4, 'hyp_b', 'hyp.bin', 'FF5ECE6A4E3D2B4D898C77FFE193FC8BBC8ACEBE78263996ECF52373D8088927'), - (4, 'keymaster_a', 'keymaster.bin', '5C968C76F29B9A4D66FBE57E639BAC6B7A2C83B1758E25ABBAF5D276B8A6AF04'), - (4, 'keymaster_b', 'keymaster.bin', '5C968C76F29B9A4D66FBE57E639BAC6B7A2C83B1758E25ABBAF5D276B8A6AF04'), - (4, 'limits', 'limits.bin', '94951A0F7AA55FB6CB975535CE4EBBFE6D695F04CB5424677B01C10DFA2E94E1'), - (4, 'logfs', 'logfs.bin', 'B8B5AC87F3D954404FC7ECBDD9EE3B5B0CF5691E5006E6EC55DB4C899FF61220'), - (4, 'modem_a', 'modem.bin', 'A3D014F0896D77A2DF7E5A80A70F43A51A047B9D03CFC675B6F0E31A6ECC4994'), - (4, 'modem_b', 'modem.bin', 'A3D014F0896D77A2DF7E5A80A70F43A51A047B9D03CFC675B6F0E31A6ECC4994'), - (4, 'qupfw_a', 'qupfw.bin', '64CC7C29D5D69B04267452B8B4DDBA9F4809E68F476FC162CA283F58537AFE4A'), - (4, 'qupfw_b', 'qupfw.bin', '64CC7C29D5D69B04267452B8B4DDBA9F4809E68F476FC162CA283F58537AFE4A'), - (4, 'splash', 'splash.bin', '5C61260048F22EDE6E6343FABB27F6FF73F9271F4751A01AAF7ABF097AFC1F08'), - (4, 'storsec_a', 'storsec.bin', '4494D86F68B125FBF2C004C824B1C6DBE71E61A65D2A1CC7DB13C553EDCB3FCE'), - (4, 'storsec_b', 'storsec.bin', '4494D86F68B125FBF2C004C824B1C6DBE71E61A65D2A1CC7DB13C553EDCB3FCE'), - (4, 'tz_a', 'tz.bin', 'E9443BF187641661BFA6C96702B9AB0156E72FB7482500F8799BA9EE2503CB16'), - (4, 'tz_b', 'tz.bin', 'E9443BF187641661BFA6C96702B9AB0156E72FB7482500F8799BA9EE2503CB16'), -] diff --git a/firmware/abl.bin b/firmware/abl.img similarity index 100% rename from firmware/abl.bin rename to firmware/abl.img diff --git a/firmware/aop.bin b/firmware/aop.img similarity index 100% rename from firmware/aop.bin rename to firmware/aop.img diff --git a/firmware/bluetooth.bin b/firmware/bluetooth.img similarity index 100% rename from firmware/bluetooth.bin rename to firmware/bluetooth.img diff --git a/firmware/cache.bin b/firmware/cache.img similarity index 100% rename from firmware/cache.bin rename to firmware/cache.img diff --git a/firmware/cmnlib.bin b/firmware/cmnlib.img similarity index 100% rename from firmware/cmnlib.bin rename to firmware/cmnlib.img diff --git a/firmware/cmnlib64.bin b/firmware/cmnlib64.img similarity index 100% rename from firmware/cmnlib64.bin rename to firmware/cmnlib64.img diff --git a/firmware/devcfg.bin b/firmware/devcfg.img similarity index 100% rename from firmware/devcfg.bin rename to firmware/devcfg.img diff --git a/firmware/devinfo.bin b/firmware/devinfo.img similarity index 100% rename from firmware/devinfo.bin rename to firmware/devinfo.img diff --git a/firmware/dsp.bin b/firmware/dsp.img similarity index 100% rename from firmware/dsp.bin rename to firmware/dsp.img diff --git a/firmware/gpt_main_0.bin b/firmware/gpt_main_0.img similarity index 100% rename from firmware/gpt_main_0.bin rename to firmware/gpt_main_0.img diff --git a/firmware/gpt_main_1.bin b/firmware/gpt_main_1.img similarity index 100% rename from firmware/gpt_main_1.bin rename to firmware/gpt_main_1.img diff --git a/firmware/gpt_main_2.bin b/firmware/gpt_main_2.img similarity index 100% rename from firmware/gpt_main_2.bin rename to firmware/gpt_main_2.img diff --git a/firmware/gpt_main_3.bin b/firmware/gpt_main_3.img similarity index 100% rename from firmware/gpt_main_3.bin rename to firmware/gpt_main_3.img diff --git a/firmware/gpt_main_4.bin b/firmware/gpt_main_4.img similarity index 100% rename from firmware/gpt_main_4.bin rename to firmware/gpt_main_4.img diff --git a/firmware/gpt_main_5.bin b/firmware/gpt_main_5.img similarity index 100% rename from firmware/gpt_main_5.bin rename to firmware/gpt_main_5.img diff --git a/firmware/hyp.bin b/firmware/hyp.img similarity index 100% rename from firmware/hyp.bin rename to firmware/hyp.img diff --git a/firmware/keymaster.bin b/firmware/keymaster.img similarity index 100% rename from firmware/keymaster.bin rename to firmware/keymaster.img diff --git a/firmware/limits.bin b/firmware/limits.img similarity index 100% rename from firmware/limits.bin rename to firmware/limits.img diff --git a/firmware/logfs.bin b/firmware/logfs.img similarity index 100% rename from firmware/logfs.bin rename to firmware/logfs.img diff --git a/firmware/modem.bin b/firmware/modem.img similarity index 100% rename from firmware/modem.bin rename to firmware/modem.img diff --git a/firmware/persist.bin b/firmware/persist.img similarity index 100% rename from firmware/persist.bin rename to firmware/persist.img diff --git a/firmware/qupfw.bin b/firmware/qupfw.img similarity index 100% rename from firmware/qupfw.bin rename to firmware/qupfw.img diff --git a/firmware/splash.bin b/firmware/splash.img similarity index 100% rename from firmware/splash.bin rename to firmware/splash.img diff --git a/firmware/splash_cc.bin b/firmware/splash_cc.img similarity index 100% rename from firmware/splash_cc.bin rename to firmware/splash_cc.img diff --git a/firmware/storsec.bin b/firmware/storsec.img similarity index 100% rename from firmware/storsec.bin rename to firmware/storsec.img diff --git a/firmware/systemrw.bin b/firmware/systemrw.img similarity index 100% rename from firmware/systemrw.bin rename to firmware/systemrw.img diff --git a/firmware/tz.bin b/firmware/tz.img similarity index 100% rename from firmware/tz.bin rename to firmware/tz.img diff --git a/firmware/xbl.bin b/firmware/xbl.img similarity index 100% rename from firmware/xbl.bin rename to firmware/xbl.img diff --git a/firmware/xbl_config.bin b/firmware/xbl_config.img similarity index 100% rename from firmware/xbl_config.bin rename to firmware/xbl_config.img diff --git a/flash_all.sh b/flash_all.sh index ade4e93..cb580af 100755 --- a/flash_all.sh +++ b/flash_all.sh @@ -5,8 +5,8 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd $DIR for part in aop abl xbl xbl_config devcfg; do - tools/edl w ${part}_a $DIR/firmware/$part.bin - tools/edl w ${part}_b $DIR/firmware/$part.bin + tools/edl w ${part}_a $DIR/firmware/$part.img + tools/edl w ${part}_b $DIR/firmware/$part.img done ./flash_kernel.sh diff --git a/scripts/build_and_package.sh b/scripts/build_and_package.sh index 370343a..308bbbc 100755 --- a/scripts/build_and_package.sh +++ b/scripts/build_and_package.sh @@ -6,6 +6,7 @@ cd $DIR/.. ./build_kernel.sh ./build_system.sh +scripts/build_userdata.sh scripts/package_ota.py # push to azure on an internal machine diff --git a/scripts/build_userdata.sh b/scripts/build_userdata.sh new file mode 100755 index 0000000..77546fe --- /dev/null +++ b/scripts/build_userdata.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +OUTPUT_DIR=$DIR/../output +GIT_BRANCH=release3-staging + +function create_image() { + IMAGE_SIZE=$1 + + WORKDIR=$(mktemp -d) + MNTDIR=$WORKDIR/mnt + USERDATA_IMAGE=$WORKDIR/raw.img + + sudo umount $MNTDIR 2> /dev/null || true + rm -rf $WORKDIR + mkdir $WORKDIR + cd $WORKDIR + + fallocate -l $IMAGE_SIZE $USERDATA_IMAGE + mkfs.ext4 $USERDATA_IMAGE + + mkdir $MNTDIR + sudo mount $USERDATA_IMAGE $MNTDIR + sudo git clone --branch=$GIT_BRANCH --depth=1 https://github.com/commaai/openpilot.git $MNTDIR/openpilot.cache + echo "clone done for $(sudo cat $MNTDIR/openpilot.cache/common/version.h)" + sudo umount $MNTDIR + + echo "Sparsify" + img2simg $USERDATA_IMAGE $OUTPUT_DIR/userdata_${sz}.img +} + +for sz in 30 89 90; do + echo "Building ${sz}GB userdata image" + create_image ${sz}G +done + +echo "Done!" diff --git a/scripts/ota_push.sh b/scripts/ota_push.sh index eab733a..a3aca13 100755 --- a/scripts/ota_push.sh +++ b/scripts/ota_push.sh @@ -12,12 +12,12 @@ DATA_ACCOUNT="commadist" # Parse input FOUND=0 if [ "$1" == "production" ]; then - OTA_JSON="$OTA_DIR/ota.json" + OTA_JSON="$OTA_DIR/all-partitions.json" DATA_CONTAINER="agnosupdate" FOUND=1 fi if [ "$1" == "staging" ]; then - OTA_JSON="$OTA_DIR/ota-staging.json" + OTA_JSON="$OTA_DIR/all-partitions-staging.json" DATA_CONTAINER="agnosupdate-staging" FOUND=1 fi diff --git a/scripts/package_ota.py b/scripts/package_ota.py index ea4e7ba..7069dec 100755 --- a/scripts/package_ota.py +++ b/scripts/package_ota.py @@ -3,9 +3,11 @@ import json import os import hashlib import shutil +import struct import subprocess from copy import deepcopy from pathlib import Path +from collections import namedtuple ROOT = Path(__file__).parent.parent OUTPUT_DIR = ROOT / "output" @@ -13,15 +15,106 @@ FIRMWARE_DIR = ROOT / "firmware" OTA_OUTPUT_DIR = OUTPUT_DIR / "ota" BUILD_DIR = ROOT / "build" +SECTOR_SIZE = 4096 + AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate") AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging") -def checksum(fn): +GPT = namedtuple('GPT', ['lun', 'name', 'path', 'start_sector', 'num_sectors', 'has_ab', 'ota', 'full_check', 'sparse']) +GPTS = [ + GPT(0, 'gpt_main_0', FIRMWARE_DIR / 'gpt_main_0.img', 0, 6, False, False, True, False), + GPT(1, 'gpt_main_1', FIRMWARE_DIR / 'gpt_main_1.img', 0, 6, False, False, True, False), + GPT(2, 'gpt_main_2', FIRMWARE_DIR / 'gpt_main_2.img', 0, 6, False, False, True, False), + GPT(3, 'gpt_main_3', FIRMWARE_DIR / 'gpt_main_3.img', 0, 6, False, False, True, False), + GPT(4, 'gpt_main_4', FIRMWARE_DIR / 'gpt_main_4.img', 0, 6, False, False, True, False), + GPT(5, 'gpt_main_5', FIRMWARE_DIR / 'gpt_main_5.img', 0, 6, False, False, True, False), +] + +Partition = namedtuple('Partition', ['name', 'path', 'has_ab', 'ota', 'full_check', 'sparse']) +PARTITIONS = [ + Partition('persist', FIRMWARE_DIR / 'persist.img', False, False, True, False), + Partition('systemrw', FIRMWARE_DIR / 'systemrw.img', False, False, True, False), + Partition('cache', FIRMWARE_DIR / 'cache.img', False, False, True, False), + Partition('xbl', FIRMWARE_DIR / 'xbl.img', True, True, True, False), + Partition('xbl_config', FIRMWARE_DIR / 'xbl_config.img', True, True, True, False), + Partition('abl', FIRMWARE_DIR / 'abl.img', True, True, True, False), + Partition('aop', FIRMWARE_DIR / 'aop.img', True, True, True, False), + Partition('bluetooth', FIRMWARE_DIR / 'bluetooth.img', True, False, True, False), + Partition('cmnlib64', FIRMWARE_DIR / 'cmnlib64.img', True, False, True, False), + Partition('cmnlib', FIRMWARE_DIR / 'cmnlib.img', True, False, True, False), + Partition('devcfg', FIRMWARE_DIR / 'devcfg.img', True, True, True, False), + Partition('devinfo', FIRMWARE_DIR / 'devinfo.img', False, False, True, False), + Partition('dsp', FIRMWARE_DIR / 'dsp.img', True, False, True, False), + Partition('hyp', FIRMWARE_DIR / 'hyp.img', True, False, True, False), + Partition('keymaster', FIRMWARE_DIR / 'keymaster.img', True, False, True, False), + Partition('limits', FIRMWARE_DIR / 'limits.img', False, False, True, False), + Partition('logfs', FIRMWARE_DIR / 'logfs.img', False, False, True, False), + Partition('modem', FIRMWARE_DIR / 'modem.img', True, False, True, False), + Partition('qupfw', FIRMWARE_DIR / 'qupfw.img', True, False, True, False), + Partition('splash', FIRMWARE_DIR / 'splash.img', False, False, True, False), + Partition('storsec', FIRMWARE_DIR / 'storsec.img', True, False, True, False), + Partition('tz', FIRMWARE_DIR / 'tz.img', True, False, True, False), + Partition('boot', OUTPUT_DIR / 'boot.img', True, True, True, False), + Partition('system', OUTPUT_DIR / 'system.img', True, True, False, True), + Partition('userdata_90', OUTPUT_DIR / 'userdata_90.img', False, False, True, True), + Partition('userdata_89', OUTPUT_DIR / 'userdata_89.img', False, False, True, True), + Partition('userdata_30', OUTPUT_DIR / 'userdata_30.img', False, False, True, True), +] + + +def file_checksum(fn): sha256 = hashlib.sha256() with open(fn, 'rb') as f: for chunk in iter(lambda: f.read(4096), b""): sha256.update(chunk) - return sha256.hexdigest() + return sha256 + +def ondevice_checksum_sparse(fn, ota): + with open(fn, 'rb') as data_source: + dat = data_source.read(28) + header = struct.unpack(" None: # since system.img is a squashfs now, we don't rely on this compression. @@ -30,34 +123,49 @@ def compress(fin, fout) -> None: subprocess.check_call(f"xz -0 -T0 -vc {fin} > {fout}", shell=True) -def process_file(fn, name, full_check=True, has_ab=True): - print(name) - hash_raw = hash = checksum(fn) - size = fn.stat().st_size - print(f" {size} bytes, hash {hash}") +def process_file(entry): + size = entry.path.stat().st_size + print(f"\n{entry.name} {size} bytes") + + sha256 = file_checksum(entry.path) + hash = hash_raw = sha256.hexdigest() + + if struct.unpack("