From 147dce2e8f10b5d6297fbebaba2dbb48c1cbdc58 Mon Sep 17 00:00:00 2001
From: Andrei Radulescu <andi.radulescu@gmail.com>
Date: Fri, 13 Sep 2024 17:51:18 +0300
Subject: [PATCH] remove system-skip-chunks alt image (#367)

---
 Dockerfile.builder     |  1 -
 build_system.sh        | 10 ------
 scripts/ota_push.sh    |  6 ----
 scripts/package_ota.py | 22 ++----------
 scripts/pull_ota.sh    |  4 +--
 tools/simg2dontcare.py | 76 ------------------------------------------
 6 files changed, 3 insertions(+), 116 deletions(-)
 delete mode 100755 tools/simg2dontcare.py

diff --git a/Dockerfile.builder b/Dockerfile.builder
index e32c9be..206c2f2 100644
--- a/Dockerfile.builder
+++ b/Dockerfile.builder
@@ -18,7 +18,6 @@ RUN apt-get update && \
     openssl \
     ccache \
     libcap2-bin \
-    android-sdk-libsparse-utils \
     && rm -rf /var/lib/apt/lists/*
 
 RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \
diff --git a/build_system.sh b/build_system.sh
index cde1c45..3426699 100755
--- a/build_system.sh
+++ b/build_system.sh
@@ -17,7 +17,6 @@ OUTPUT_DIR="$DIR/output"
 ROOTFS_DIR="$BUILD_DIR/agnos-rootfs"
 ROOTFS_IMAGE="$BUILD_DIR/system.img"
 OUT_IMAGE="$OUTPUT_DIR/system.img"
-OUT_SKIP_CHUNKS_IMAGE="$OUTPUT_DIR/system-skip-chunks.img"
 
 # the partition is 10G, but openpilot's updater didn't always handle the full size
 # openpilot fix, shipped in 0.9.8 (8/18/24): https://github.com/commaai/openpilot/pull/33320
@@ -143,15 +142,6 @@ exec_as_root bash -c "set -e; export ROOTFS_DIR=$ROOTFS_DIR GIT_HASH=$GIT_HASH;
 echo "Unmount filesystem"
 exec_as_root umount -l $ROOTFS_DIR
 
-# Make system image with skipped chunks
-echo "Sparsifying image $(basename $OUT_SKIP_CHUNKS_IMAGE)"
-exec_as_user bash -c "\
-TMP_SPARSE=\$(mktemp); \
-img2simg $ROOTFS_IMAGE \$TMP_SPARSE; \
-TMP_SKIP=\$(mktemp); \
-$DIR/tools/simg2dontcare.py \$TMP_SPARSE \$TMP_SKIP; \
-mv \$TMP_SKIP $OUT_SKIP_CHUNKS_IMAGE"
-
 # Copy system image to output
 cp $ROOTFS_IMAGE $OUT_IMAGE
 
diff --git a/scripts/ota_push.sh b/scripts/ota_push.sh
index 913301f..aaa6d0d 100755
--- a/scripts/ota_push.sh
+++ b/scripts/ota_push.sh
@@ -41,12 +41,6 @@ process_file() {
   local HASH_RAW=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .hash_raw")
   upload_file "$NAME-$HASH_RAW.img.xz"
 
-  local ALT_URL=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .alt.url")
-  if [ "$ALT_URL" != "null" ]; then
-    local ALT_FILE_NAME=$(basename $ALT_URL)
-    upload_file $ALT_FILE_NAME
-  fi
-
   # if [ "$NAME" == "system" ]; then
   #   local CAIBX_FILE_NAME="system-$HASH_RAW.caibx"
   #   local CHUNKS_FOLDER="system-$HASH_RAW"
diff --git a/scripts/package_ota.py b/scripts/package_ota.py
index 5e3988b..eca967e 100755
--- a/scripts/package_ota.py
+++ b/scripts/package_ota.py
@@ -26,7 +26,7 @@ def compress(fin, fout) -> None:
   subprocess.check_call(f"xz -T4 -vc {fin} > {fout}", shell=True)
 
 
-def process_file(fn, name, full_check=True, has_ab=True, alt=None):
+def process_file(fn, name, full_check=True, has_ab=True):
   print(name)
   hash_raw = hash = checksum(fn)
   size = fn.stat().st_size
@@ -47,22 +47,6 @@ def process_file(fn, name, full_check=True, has_ab=True, alt=None):
     "has_ab": has_ab,
   }
 
-  if alt is not None:
-    print("  calculating alt")
-    alt_hash = checksum(alt)
-    alt_size = alt.stat().st_size
-    print(f"  {alt_size} bytes, hash {alt_hash} (alt)")
-
-    print("  compressing alt")
-    alt_xz_fn = OTA_OUTPUT_DIR / f"{alt.stem}-{hash_raw}.img.xz"
-    compress(alt, alt_xz_fn)
-
-    ret["alt"] = {
-      "hash": alt_hash,
-      "url": "{remote_url}/" + alt_xz_fn.name,
-      "size": alt_size,
-    }
-
   return ret
 
 
@@ -74,7 +58,7 @@ if __name__ == "__main__":
   else:
     files = [
       process_file(OUTPUT_DIR / "boot.img", "boot"),
-      process_file(OUTPUT_DIR / "system.img", "system", full_check=False, alt=OUTPUT_DIR / "system-skip-chunks.img"),
+      process_file(OUTPUT_DIR / "system.img", "system", full_check=False),
     ]
 
     # pull in firmware not built in this repo
@@ -100,8 +84,6 @@ if __name__ == "__main__":
     processed_files = []
     for f in deepcopy(files):
       f["url"] = f["url"].format(remote_url=remote_url)
-      if "alt" in f:
-        f["alt"]["url"] = f["alt"]["url"].format(remote_url=remote_url)
       processed_files.append(f)
 
     with open(OTA_OUTPUT_DIR / output_fn, "w") as out:
diff --git a/scripts/pull_ota.sh b/scripts/pull_ota.sh
index 90448a6..5bd6ed0 100755
--- a/scripts/pull_ota.sh
+++ b/scripts/pull_ota.sh
@@ -20,9 +20,8 @@ cd $OUTPUT_DIR
 
 download_image() {
   local name=$1
-  local alt=${2:-""}
 
-  local url=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$name\") | $alt.url")
+  local url=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$name\") | .url")
   if [ "$url" == "null" ]; then
     return
   fi
@@ -37,7 +36,6 @@ download_image() {
 
 for name in boot system; do
   download_image $name
-  download_image $name ".alt"
 done
 
 echo "Done!"
diff --git a/tools/simg2dontcare.py b/tools/simg2dontcare.py
deleted file mode 100755
index 5a95a0a..0000000
--- a/tools/simg2dontcare.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import sys
-import struct
-from enum import IntEnum
-
-# https://android.googlesource.com/platform/system/core/+/master/libsparse/sparse_format.h
-class ChunkType(IntEnum):
-  RAW = 0xCAC1
-  FILL = 0xCAC2
-  DONT_CARE = 0xCAC3
-  CRC32 = 0xCAC4
-
-ChunkHeader = struct.Struct("<2H2I")
-
-
-def process_image(input_image: str, output_image: str) -> None:
-  with open(input_image, "rb") as inf, open(output_image, "wb") as outf:
-    dat = inf.read(28)
-    outf.write(dat)
-
-    header = struct.unpack("<I4H4I", dat)
-
-    magic = header[0]
-    major_version = header[1]
-    minor_version = header[2]
-    file_hdr_sz = header[3]
-    chunk_hdr_sz = header[4]
-    total_chunks = header[7]
-    image_checksum = header[8]
-
-    assert magic == 0xED26FF3A
-    assert major_version == 1 and minor_version == 0
-    assert file_hdr_sz == 28
-    assert chunk_hdr_sz == 12
-
-    for _ in range(1, total_chunks+1):
-      header_bin = inf.read(12)
-
-      header = ChunkHeader.unpack(header_bin)
-      chunk_type = ChunkType(header[0])
-      chunk_sz = header[2]
-      total_sz = header[3]
-      data_sz = total_sz - 12
-
-      # replace fill 0s with DONT_CARE chunks
-      if chunk_type == ChunkType.FILL:
-        assert data_sz == 4
-        fill_bin = inf.read(4)
-        fill = struct.unpack("<I", fill_bin)[0]
-        if fill == 0:
-          # https://coral.googlesource.com/img2simg/+/refs/heads/master/libsparse/output_file.c#351
-          dat = ChunkHeader.pack(ChunkType.DONT_CARE, 0, chunk_sz, ChunkHeader.size)
-          outf.write(dat)
-          continue
-        else:
-          inf.seek(inf.tell() - data_sz)
-
-      # pass through other chunk types
-      outf.write(header_bin)
-      dat = inf.read(data_sz)
-      outf.write(dat)
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description="Replace FILL 0 chunks in a sparse image with DONT_CARE chunks",
-                                   formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument("input_image", nargs="?", help="Input sparse image")
-  parser.add_argument("output_image", nargs="?", help="Output sparse image")
-  args = parser.parse_args(sys.argv[1:])
-
-  if args.input_image is None or args.output_image is None:
-    parser.print_help()
-    exit(1)
-
-  process_image(args.input_image, args.output_image)