mirror of
https://github.com/commaai/agnos-builder.git
synced 2026-09-29 02:43:50 +08:00
include simg2dontcare image in manifest (#170)
Co-authored-by: Cameron Clough <cameronjclough@gmail.com>
This commit is contained in:
Adeeb Shihadeh
+10
-3
@@ -16,7 +16,8 @@ OUTPUT_DIR="$DIR/output"
|
||||
ROOTFS_DIR="$BUILD_DIR/agnos-rootfs"
|
||||
ROOTFS_IMAGE="$BUILD_DIR/system.img.raw"
|
||||
ROOTFS_IMAGE_SIZE=10G
|
||||
SPARSE_IMAGE="$BUILD_DIR/system.img"
|
||||
SPARSE_IMAGE="$OUTPUT_DIR/system.img"
|
||||
SKIP_CHUNKS_IMAGE="$OUTPUT_DIR/system-skip-chunks.img"
|
||||
|
||||
# Create temp dir if non-existent
|
||||
mkdir -p $BUILD_DIR $OUTPUT_DIR
|
||||
@@ -87,7 +88,13 @@ sudo umount -l $ROOTFS_DIR
|
||||
|
||||
# Sparsify
|
||||
echo "Sparsify image"
|
||||
img2simg $ROOTFS_IMAGE $SPARSE_IMAGE
|
||||
mv $SPARSE_IMAGE $OUTPUT_DIR
|
||||
TMP_SPARSE="$(mktemp)"
|
||||
img2simg $ROOTFS_IMAGE $TMP_SPARSE
|
||||
mv $TMP_SPARSE $SPARSE_IMAGE
|
||||
|
||||
# Make image with skipped chunks
|
||||
TMP_SKIP="$(mktemp)"
|
||||
$DIR/tools/simg2dontcare.py $SPARSE_IMAGE $TMP_SKIP
|
||||
mv $TMP_SKIP $SKIP_CHUNKS_IMAGE
|
||||
|
||||
echo "Done!"
|
||||
|
||||
+4
-21
@@ -1,29 +1,12 @@
|
||||
#!/bin/bash -e
|
||||
set -e
|
||||
|
||||
# Config
|
||||
OUTPUT_DIR="output"
|
||||
ROOTFS_IMAGE="system.img"
|
||||
|
||||
# Log colors
|
||||
GREEN="\033[0;32m"
|
||||
NO_COLOR='\033[0m'
|
||||
|
||||
# Make sure we're in the correct spot
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
cd $DIR
|
||||
|
||||
# Flash system
|
||||
fastboot --set-active=a
|
||||
|
||||
if [ -d $HOME/openpilot/provisioning ]; then
|
||||
DIMG=$(mktemp)
|
||||
$DIR/tools/simg2dontcare.py $OUTPUT_DIR/$ROOTFS_IMAGE $DIMG
|
||||
fastboot erase system_a
|
||||
fastboot flash system_a $DIMG
|
||||
else
|
||||
fastboot flash system_a $OUTPUT_DIR/$ROOTFS_IMAGE
|
||||
fi
|
||||
|
||||
fastboot erase system_a
|
||||
fastboot flash system_a $OUTPUT_DIR/system-skip-chunks.img
|
||||
fastboot continue
|
||||
|
||||
echo -e "${GREEN}Done!"
|
||||
echo "Done!"
|
||||
|
||||
@@ -19,5 +19,5 @@ fi
|
||||
|
||||
./build_kernel.sh
|
||||
./build_system.sh
|
||||
internal/package_ota.sh
|
||||
internal/package_ota.py
|
||||
internal/ota_push.sh staging
|
||||
|
||||
@@ -39,6 +39,14 @@ process_file() {
|
||||
local NAME=$1
|
||||
local HASH_RAW=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .hash_raw")
|
||||
upload_file "$NAME-$HASH_RAW.img.xz"
|
||||
upload_file "$NAME-$HASH_RAW.img.gz"
|
||||
|
||||
local ALT_URL=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .alt.url")
|
||||
if [ "$ALT_URL" != "null" ]; then
|
||||
local ALT_FILE_NAME=$(basename $ALT_URL)
|
||||
upload_file $ALT_FILE_NAME
|
||||
upload_file {$ALT_FILE_NAME%.xz}.gz
|
||||
fi
|
||||
|
||||
# if [ "$NAME" == "system" ]; then
|
||||
# local CAIBX_FILE_NAME="system-$HASH_RAW.caibx"
|
||||
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
OUTPUT_DIR = ROOT / "output"
|
||||
OTA_OUTPUT_DIR = OUTPUT_DIR / "ota"
|
||||
FIRMWARE_DIR = ROOT / "agnos-firmware"
|
||||
|
||||
AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate")
|
||||
AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging")
|
||||
|
||||
|
||||
def checksum(fn):
|
||||
return subprocess.check_output(["sha256sum", fn]).decode().split()[0]
|
||||
|
||||
|
||||
def compress(fin, fout) -> None:
|
||||
subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True)
|
||||
subprocess.check_call(f"gzip -vc {fin} > {fout.with_suffix('.gz')}", shell=True)
|
||||
|
||||
|
||||
def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None):
|
||||
print(name)
|
||||
hash_raw = hash = checksum(fn)
|
||||
size = fn.stat().st_size
|
||||
print(f" {size} bytes, hash {hash}")
|
||||
|
||||
if sparse:
|
||||
with NamedTemporaryFile() as tmp_f:
|
||||
print(" converting sparse image to raw")
|
||||
subprocess.check_call(["simg2img", fn, tmp_f.name])
|
||||
hash_raw = checksum(tmp_f.name)
|
||||
size = Path(tmp_f.name).stat().st_size
|
||||
print(f" {size} bytes, hash {hash} (raw)")
|
||||
|
||||
print(" compressing")
|
||||
xz_fn = OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img.xz"
|
||||
compress(fn, xz_fn)
|
||||
|
||||
ret = {
|
||||
"name": name,
|
||||
"url": "{remote_url}/" + xz_fn.name,
|
||||
"hash": hash,
|
||||
"hash_raw": hash_raw,
|
||||
"size": size,
|
||||
"sparse": sparse,
|
||||
"full_check": full_check,
|
||||
"has_ab": has_ab,
|
||||
}
|
||||
|
||||
if alt is not None:
|
||||
print(" calculating alt")
|
||||
alt_hash = checksum(alt)
|
||||
alt_size = alt.stat().st_size
|
||||
print(f" {alt_size} bytes, hash {alt_hash} (alt)")
|
||||
|
||||
print(" compressing alt")
|
||||
alt_xz_fn = OTA_OUTPUT_DIR / f"{alt.stem}-{hash_raw}.img.xz"
|
||||
compress(alt, alt_xz_fn)
|
||||
|
||||
ret["alt"] = {
|
||||
"hash": alt_hash,
|
||||
"url": "{remote_url}/" + alt_xz_fn.name,
|
||||
}
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
OTA_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
files = [
|
||||
process_file(OUTPUT_DIR / "boot.img", "boot"),
|
||||
process_file(FIRMWARE_DIR / "abl.bin", "abl"),
|
||||
process_file(FIRMWARE_DIR / "xbl.bin", "xbl"),
|
||||
process_file(FIRMWARE_DIR / "xbl_config.bin", "xbl_config"),
|
||||
process_file(FIRMWARE_DIR / "devcfg.bin", "devcfg"),
|
||||
process_file(FIRMWARE_DIR / "aop.bin", "aop"),
|
||||
process_file(OUTPUT_DIR / "system.img", "system", sparse=True, full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"),
|
||||
]
|
||||
configs = [
|
||||
(AGNOS_UPDATE_URL, "ota.json"),
|
||||
(AGNOS_STAGING_UPDATE_URL, "ota-staging.json"),
|
||||
]
|
||||
|
||||
for remote_url, output_fn in configs:
|
||||
processed_files = []
|
||||
for f in deepcopy(files):
|
||||
f["url"] = f["url"].format(remote_url=remote_url)
|
||||
if "alt" in f:
|
||||
f["alt"]["url"] = f["alt"]["url"].format(remote_url=remote_url)
|
||||
processed_files.append(f)
|
||||
|
||||
with open(OTA_OUTPUT_DIR / output_fn, "w") as out:
|
||||
json.dump(processed_files, out, indent=2)
|
||||
|
||||
print("Done")
|
||||
@@ -1,108 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
|
||||
# Make sure we're in the correct spot
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
|
||||
ROOT="$DIR/../"
|
||||
TMP_DIR="/tmp/agnos-builder-tmp"
|
||||
OUTPUT_DIR="$ROOT/output"
|
||||
OTA_OUTPUT_DIR="$OUTPUT_DIR/ota"
|
||||
FIRMWARE_DIR="$ROOT/agnos-firmware"
|
||||
|
||||
AGNOS_UPDATE_URL=${AGNOS_UPDATE_URL:-https://commadist.azureedge.net/agnosupdate}
|
||||
AGNOS_STAGING_UPDATE_URL=${AGNOS_STAGING_UPDATE_URL:-https://commadist.azureedge.net/agnosupdate-staging}
|
||||
OUTPUT_JSON="$OTA_OUTPUT_DIR/ota.json"
|
||||
OUTPUT_STAGING_JSON="$OTA_OUTPUT_DIR/ota-staging.json"
|
||||
|
||||
# Make sure archive dir is empty
|
||||
rm -rf $OTA_OUTPUT_DIR && mkdir -p $OTA_OUTPUT_DIR
|
||||
|
||||
process_file() {
|
||||
local FILE=$1
|
||||
local NAME=$2
|
||||
local SPARSE=${3:-false}
|
||||
local FULL_CHECK=${4:-true}
|
||||
local HAS_AB=${5:-true}
|
||||
|
||||
echo "Hashing $NAME..."
|
||||
local HASH=$(sha256sum $FILE | cut -c 1-64)
|
||||
local SIZE=$(wc -c < $FILE)
|
||||
echo " $HASH ($SIZE bytes)"
|
||||
|
||||
local HASH_RAW=$HASH
|
||||
if [ "$NAME" == "system" ]; then
|
||||
echo "Converting system to raw..."
|
||||
local FILE_RAW=/tmp/system.img.raw
|
||||
simg2img $FILE $FILE_RAW
|
||||
|
||||
echo "Hashing system raw..."
|
||||
HASH_RAW=$(sha256sum $FILE_RAW | cut -c 1-64)
|
||||
SIZE=$(wc -c < $FILE_RAW)
|
||||
echo " $HASH_RAW ($SIZE bytes) (raw)"
|
||||
|
||||
|
||||
# echo "Creating system casync files"
|
||||
# casync make --compression=xz --store $OTA_OUTPUT_DIR/system-$HASH_RAW $OTA_OUTPUT_DIR/system-$HASH_RAW.caibx $FILE_RAW
|
||||
|
||||
rm $FILE_RAW
|
||||
fi
|
||||
|
||||
echo "Compressing $NAME..."
|
||||
local FILENAME=$NAME-$HASH_RAW.img.xz
|
||||
local ARCHIVE=$OTA_OUTPUT_DIR/$FILENAME
|
||||
xz -vc $FILE > $ARCHIVE
|
||||
|
||||
cat <<EOF | tee -a $OUTPUT_JSON $OUTPUT_STAGING_JSON > /dev/null
|
||||
{
|
||||
"name": "$NAME",
|
||||
"hash": "$HASH",
|
||||
"hash_raw": "$HASH_RAW",
|
||||
"size": $SIZE,
|
||||
"sparse": $SPARSE,
|
||||
"full_check": $FULL_CHECK,
|
||||
"has_ab": $HAS_AB,
|
||||
EOF
|
||||
|
||||
cat <<EOF >> $OUTPUT_JSON
|
||||
"url": "$AGNOS_UPDATE_URL/$FILENAME"
|
||||
EOF
|
||||
cat <<EOF >> $OUTPUT_STAGING_JSON
|
||||
"url": "$AGNOS_STAGING_UPDATE_URL/$FILENAME"
|
||||
EOF
|
||||
|
||||
# if [ $NAME == "system" ]; then
|
||||
# cat <<EOF >> $OUTPUT_JSON
|
||||
# "casync_caibx": "$AGNOS_UPDATE_URL/$NAME-$HASH_RAW.caibx",
|
||||
# "casync_store": "$AGNOS_UPDATE_URL/$NAME-$HASH_RAW"
|
||||
# EOF
|
||||
# cat <<EOF >> $OUTPUT_STAGING_JSON
|
||||
# "casync_caibx": "$AGNOS_STAGING_UPDATE_URL/$NAME-$HASH_RAW.caibx",
|
||||
# "casync_store": "$AGNOS_STAGING_UPDATE_URL/$NAME-$HASH_RAW"
|
||||
# EOF
|
||||
# fi
|
||||
|
||||
cat <<EOF | tee -a $OUTPUT_JSON $OUTPUT_STAGING_JSON > /dev/null
|
||||
},
|
||||
EOF
|
||||
}
|
||||
|
||||
cd $ROOT
|
||||
|
||||
echo "[" > $OUTPUT_JSON
|
||||
echo "[" > $OUTPUT_STAGING_JSON
|
||||
|
||||
process_file "$OUTPUT_DIR/boot.img" boot
|
||||
process_file "$FIRMWARE_DIR/abl.bin" abl
|
||||
process_file "$FIRMWARE_DIR/xbl.bin" xbl
|
||||
process_file "$FIRMWARE_DIR/xbl_config.bin" xbl_config
|
||||
process_file "$FIRMWARE_DIR/devcfg.bin" devcfg
|
||||
process_file "$FIRMWARE_DIR/aop.bin" aop
|
||||
process_file "$OUTPUT_DIR/system.img" system true false true
|
||||
|
||||
# remove trailing comma
|
||||
sed -i "$ s/.$//" $OUTPUT_JSON $OUTPUT_STAGING_JSON
|
||||
|
||||
echo "]" >> $OUTPUT_JSON
|
||||
echo "]" >> $OUTPUT_STAGING_JSON
|
||||
|
||||
echo "Done!"
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/bash -e
|
||||
|
||||
# Make sure we're in the correct directory
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
|
||||
# Constants
|
||||
OUTPUT_DIR="$DIR/../output"
|
||||
|
||||
if [ "$1" == "" ]; then
|
||||
echo "Supply the URL to the OTA JSON as first argument!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OTA_JSON=$(mktemp)
|
||||
wget $1 -O $OTA_JSON
|
||||
|
||||
mkdir -p $OUTPUT_DIR
|
||||
cd $OUTPUT_DIR
|
||||
|
||||
download_image() {
|
||||
local name=$1
|
||||
local alt=${2:-""}
|
||||
|
||||
local url=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$name\") | $alt.url")
|
||||
if [ "$url" == "null" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
local hash_raw=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$name\") | .hash_raw")
|
||||
local file_name=$(basename $url .xz)
|
||||
file_name=${file_name//-$hash_raw/}
|
||||
|
||||
echo "Downloading $file_name..."
|
||||
curl $url | xz -d > $file_name
|
||||
}
|
||||
|
||||
for name in boot system; do
|
||||
download_image $name
|
||||
download_image $name ".alt"
|
||||
done
|
||||
|
||||
echo "Done!"
|
||||
Reference in New Issue
Block a user