manifest with all partitions (#443)

* all

* ota

* all manif

* ab

* remove these

* hash

* userdata

* ontheflyisthebest

* on the fly

* dont need

* remove

* handle fill

* comment

* all

* all img

* rename

* read

* sparse image

* tool

* no root

* userdata

* userdata

* need this unfort

* fix

* fix

* nop

* remove

* nop

* this too

* ota check

* better for push

* fix

* again

* good now

Dockerfile.builder

@@ -18,6 +18,7 @@ RUN apt-get update && \
     openssl \
     ccache \
     libcap2-bin \
+    android-sdk-libsparse-utils \
     && rm -rf /var/lib/apt/lists/*
 
 RUN if [ ${UID:-0} -ne 0 ] && [ ${GID:-0} -ne 0 ]; then \

build_system.sh

@@ -136,7 +136,7 @@ exec_as_root bash -c "set -e; export ROOTFS_DIR=$ROOTFS_DIR GIT_HASH=$GIT_HASH;
 echo "Unmount filesystem"
 exec_as_root umount -l $ROOTFS_DIR
 
-# Copy system image to output
-cp $ROOTFS_IMAGE $OUT_IMAGE
+# Sparsify system image
+exec_as_user img2simg $ROOTFS_IMAGE $OUT_IMAGE
 
 echo "Done!"

firmware.json

@@ -1,52 +0,0 @@
-[
-  {
-    "name": "xbl",
-    "url": "https://commadist.azureedge.net/agnosupdate/xbl-468f1ad6ab55e198647ff9191f91bd2918db9c0a3e27bae5673b4c5575c1254c.img.xz",
-    "hash": "468f1ad6ab55e198647ff9191f91bd2918db9c0a3e27bae5673b4c5575c1254c",
-    "hash_raw": "468f1ad6ab55e198647ff9191f91bd2918db9c0a3e27bae5673b4c5575c1254c",
-    "size": 3282256,
-    "sparse": false,
-    "full_check": true,
-    "has_ab": true
-  },
-  {
-    "name": "abl",
-    "url": "https://commadist.azureedge.net/agnosupdate/abl-32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6.img.xz",
-    "hash": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6",
-    "hash_raw": "32a2174b5f764e95dfc54cf358ba01752943b1b3b90e626149c3da7d5f1830b6",
-    "size": 274432,
-    "sparse": false,
-    "full_check": true,
-    "has_ab": true
-  },
-  {
-    "name": "xbl_config",
-    "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-92b675dc2862ed15c732d91d9eb307d7e852e349217db8bee8f8829db543686b.img.xz",
-    "hash": "92b675dc2862ed15c732d91d9eb307d7e852e349217db8bee8f8829db543686b",
-    "hash_raw": "92b675dc2862ed15c732d91d9eb307d7e852e349217db8bee8f8829db543686b",
-    "size": 98124,
-    "sparse": false,
-    "full_check": true,
-    "has_ab": true
-  },
-  {
-    "name": "devcfg",
-    "url": "https://commadist.azureedge.net/agnosupdate/devcfg-225b24ea7b1d2fee7f7d2da21386920ddacac2e33e9e938168436292f4eae180.img.xz",
-    "hash": "225b24ea7b1d2fee7f7d2da21386920ddacac2e33e9e938168436292f4eae180",
-    "hash_raw": "225b24ea7b1d2fee7f7d2da21386920ddacac2e33e9e938168436292f4eae180",
-    "size": 40336,
-    "sparse": false,
-    "full_check": true,
-    "has_ab": true
-  },
-  {
-    "name": "aop",
-    "url": "https://commadist.azureedge.net/agnosupdate/aop-f0fcf7611d0890a72984f15a516dd37fa532dfcb70d428a8406838cf74ce23d5.img.xz",
-    "hash": "f0fcf7611d0890a72984f15a516dd37fa532dfcb70d428a8406838cf74ce23d5",
-    "hash_raw": "f0fcf7611d0890a72984f15a516dd37fa532dfcb70d428a8406838cf74ce23d5",
-    "size": 184364,
-    "sparse": false,
-    "full_check": true,
-    "has_ab": true
-  }
-]

firmware/.gitattributes

@@ -1 +1,2 @@
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.img filter=lfs diff=lfs merge=lfs -text

firmware/__init__.py

@@ -1,51 +0,0 @@
-# Structure: (lun, partition_name, start_sector, num_sectors, checksum)
-PARTITION_TABLES = [
-  (0, 'gpt_main_0', 0, 6, '8928A31FD9EE20F8703649F89833EBA9B55E84B6415E67799C777B163C95A0BD'),
-  (1, 'gpt_main_1', 0, 6, 'FE8EF7653DB588D7420A625920CA06927DFCB0ED8AFF3E3A1C74A52A24398BA6'),
-  (2, 'gpt_main_2', 0, 6, '5CCFC7240C8CBFA2F1A018A2E376CF274A6BAF858C9BFE71951D8E28CAB53C21'),
-  (3, 'gpt_main_3', 0, 6, 'C707979FA21E89519328F4F30C2B21C9C453401CA8303F914C1873D410A95159'),
-  (4, 'gpt_main_4', 0, 6, 'E9405DCD785DBE79412184E1894A9C51AB7DEB33BB612166C4C42A3D2BF42A0E'),
-  (5, 'gpt_main_5', 0, 6, '21AE965F05B2FA8D02E04F1EB74718F9779864F6EACDEB859757D6435E8CCCE3'),
-]
-
-# Structure: (lun, partition_name, filename, checksum)
-# All zero-filled partitions are not in this list, since the device gets erased anyways
-QDL_FLASH_ARRAY = [
-  (0, 'persist', 'persist.bin', '9814B07851292F510F3794B767489F38AB379A99F0EA75DC620AD2D3A496D54D'),
-  (0, 'systemrw', 'systemrw.bin', '8CE150CA38EF64A0885FC2FE816E5B63BAE8ADB4DF5D809C5B318E6996366C7E'),
-  (0, 'cache', 'cache.bin', 'EBFBAAA2F96DC4E5FEA4F126364E5BF5B3B44C12CBC753B62FDD8BAAB82F70B4'),
-  (1, 'xbl_a', 'xbl.bin', '7EA3D6A4AB5396D2C47E135B4E027EDDA270064914305CF5B98363779F53A890'),
-  (1, 'xbl_config_a', 'xbl_config.bin', '04E478D8C604CBA3DE571C56A846474D0DEA5270462C18034229123BAE2EA645'),
-  (2, 'xbl_b', 'xbl.bin', '7EA3D6A4AB5396D2C47E135B4E027EDDA270064914305CF5B98363779F53A890'),
-  (2, 'xbl_config_b', 'xbl_config.bin', '04E478D8C604CBA3DE571C56A846474D0DEA5270462C18034229123BAE2EA645'),
-  (4, 'abl_a', 'abl.bin', '262AC633281044C13FA861C244485E940BB92ADECCF14657C9160DF52C360AED'),
-  (4, 'abl_b', 'abl.bin', '262AC633281044C13FA861C244485E940BB92ADECCF14657C9160DF52C360AED'),
-  (4, 'aop_a', 'aop.bin', '29DA7A7F7973001B82DCBCCDA5069F2F8A27BD8850DC25F857BA7919AC06E83E'),
-  (4, 'aop_b', 'aop.bin', '29DA7A7F7973001B82DCBCCDA5069F2F8A27BD8850DC25F857BA7919AC06E83E'),
-  (4, 'bluetooth_a', 'bluetooth.bin', '9BB766D2D2CE0CC4491664B3010FE1EF62F8FFC1E362D55F78E48C4141F75533'),
-  (4, 'bluetooth_b', 'bluetooth.bin', '9BB766D2D2CE0CC4491664B3010FE1EF62F8FFC1E362D55F78E48C4141F75533'),
-  (4, 'cmnlib64_a', 'cmnlib64.bin', '1A876BD151BB9635F18719C4A17F953079DE6E11D3EAEC800968FC75669E0DC3'),
-  (4, 'cmnlib64_b', 'cmnlib64.bin', '1A876BD151BB9635F18719C4A17F953079DE6E11D3EAEC800968FC75669E0DC3'),
-  (4, 'cmnlib_a', 'cmnlib.bin', '63DF823E8A5FAE01D66CB2B8C20F0D2DDB5C5F2425E5D0992A64676273BA1C82'),
-  (4, 'cmnlib_b', 'cmnlib.bin', '63DF823E8A5FAE01D66CB2B8C20F0D2DDB5C5F2425E5D0992A64676273BA1C82'),
-  (4, 'devcfg_a', 'devcfg.bin', '8DE276D6179915B01B9568D18EB69EB18F1E0D06CA99AEC290080B005AA23612'),
-  (4, 'devcfg_b', 'devcfg.bin', '8DE276D6179915B01B9568D18EB69EB18F1E0D06CA99AEC290080B005AA23612'),
-  (4, 'devinfo', 'devinfo.bin', '143869C499A7E878FBEAB756E9C53074195770CC41D6D0D10E45C043141389A3'),
-  (4, 'dsp_a', 'dsp.bin', '4B15FBD2F45581F1553F33F01649E450B24AA19D5DEFF2AC7DCB16A534D9C248'),
-  (4, 'dsp_b', 'dsp.bin', '4B15FBD2F45581F1553F33F01649E450B24AA19D5DEFF2AC7DCB16A534D9C248'),
-  (4, 'hyp_a', 'hyp.bin', 'FF5ECE6A4E3D2B4D898C77FFE193FC8BBC8ACEBE78263996ECF52373D8088927'),
-  (4, 'hyp_b', 'hyp.bin', 'FF5ECE6A4E3D2B4D898C77FFE193FC8BBC8ACEBE78263996ECF52373D8088927'),
-  (4, 'keymaster_a', 'keymaster.bin', '5C968C76F29B9A4D66FBE57E639BAC6B7A2C83B1758E25ABBAF5D276B8A6AF04'),
-  (4, 'keymaster_b', 'keymaster.bin', '5C968C76F29B9A4D66FBE57E639BAC6B7A2C83B1758E25ABBAF5D276B8A6AF04'),
-  (4, 'limits', 'limits.bin', '94951A0F7AA55FB6CB975535CE4EBBFE6D695F04CB5424677B01C10DFA2E94E1'),
-  (4, 'logfs', 'logfs.bin', 'B8B5AC87F3D954404FC7ECBDD9EE3B5B0CF5691E5006E6EC55DB4C899FF61220'),
-  (4, 'modem_a', 'modem.bin', 'A3D014F0896D77A2DF7E5A80A70F43A51A047B9D03CFC675B6F0E31A6ECC4994'),
-  (4, 'modem_b', 'modem.bin', 'A3D014F0896D77A2DF7E5A80A70F43A51A047B9D03CFC675B6F0E31A6ECC4994'),
-  (4, 'qupfw_a', 'qupfw.bin', '64CC7C29D5D69B04267452B8B4DDBA9F4809E68F476FC162CA283F58537AFE4A'),
-  (4, 'qupfw_b', 'qupfw.bin', '64CC7C29D5D69B04267452B8B4DDBA9F4809E68F476FC162CA283F58537AFE4A'),
-  (4, 'splash', 'splash.bin', '5C61260048F22EDE6E6343FABB27F6FF73F9271F4751A01AAF7ABF097AFC1F08'),
-  (4, 'storsec_a', 'storsec.bin', '4494D86F68B125FBF2C004C824B1C6DBE71E61A65D2A1CC7DB13C553EDCB3FCE'),
-  (4, 'storsec_b', 'storsec.bin', '4494D86F68B125FBF2C004C824B1C6DBE71E61A65D2A1CC7DB13C553EDCB3FCE'),
-  (4, 'tz_a', 'tz.bin', 'E9443BF187641661BFA6C96702B9AB0156E72FB7482500F8799BA9EE2503CB16'),
-  (4, 'tz_b', 'tz.bin', 'E9443BF187641661BFA6C96702B9AB0156E72FB7482500F8799BA9EE2503CB16'),
-]

flash_all.sh

@@ -5,8 +5,8 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
 cd $DIR
 
 for part in aop abl xbl xbl_config devcfg; do
-  tools/edl w ${part}_a $DIR/firmware/$part.bin
-  tools/edl w ${part}_b $DIR/firmware/$part.bin
+  tools/edl w ${part}_a $DIR/firmware/$part.img
+  tools/edl w ${part}_b $DIR/firmware/$part.img
 done
 
 ./flash_kernel.sh

scripts/build_and_package.sh

@@ -6,6 +6,7 @@ cd $DIR/..
 
 ./build_kernel.sh
 ./build_system.sh
+scripts/build_userdata.sh
 scripts/package_ota.py
 
 # push to azure on an internal machine

scripts/build_userdata.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+OUTPUT_DIR=$DIR/../output
+GIT_BRANCH=release3-staging
+
+function create_image() {
+  IMAGE_SIZE=$1
+
+  WORKDIR=$(mktemp -d)
+  MNTDIR=$WORKDIR/mnt
+  USERDATA_IMAGE=$WORKDIR/raw.img
+
+  sudo umount $MNTDIR 2> /dev/null || true
+  rm -rf $WORKDIR
+  mkdir $WORKDIR
+  cd $WORKDIR
+
+  fallocate -l $IMAGE_SIZE $USERDATA_IMAGE
+  mkfs.ext4 $USERDATA_IMAGE
+
+  mkdir $MNTDIR
+  sudo mount $USERDATA_IMAGE $MNTDIR
+  sudo git clone --branch=$GIT_BRANCH --depth=1 https://github.com/commaai/openpilot.git $MNTDIR/openpilot.cache
+  echo "clone done for $(sudo cat $MNTDIR/openpilot.cache/common/version.h)"
+  sudo umount $MNTDIR
+
+  echo "Sparsify"
+  img2simg $USERDATA_IMAGE $OUTPUT_DIR/userdata_${sz}.img
+}
+
+for sz in 30 89 90; do
+  echo "Building ${sz}GB userdata image"
+  create_image ${sz}G
+done
+
+echo "Done!"

scripts/ota_push.sh

@@ -12,12 +12,12 @@ DATA_ACCOUNT="commadist"
 # Parse input
 FOUND=0
 if [ "$1" == "production" ]; then
-  OTA_JSON="$OTA_DIR/ota.json"
+  OTA_JSON="$OTA_DIR/all-partitions.json"
   DATA_CONTAINER="agnosupdate"
   FOUND=1
 fi
 if [ "$1" == "staging" ]; then
-  OTA_JSON="$OTA_DIR/ota-staging.json"
+  OTA_JSON="$OTA_DIR/all-partitions-staging.json"
   DATA_CONTAINER="agnosupdate-staging"
   FOUND=1
 fi

scripts/package_ota.py

@@ -3,9 +3,11 @@ import json
 import os
 import hashlib
 import shutil
+import struct
 import subprocess
 from copy import deepcopy
 from pathlib import Path
+from collections import namedtuple
 
 ROOT = Path(__file__).parent.parent
 OUTPUT_DIR = ROOT / "output"
@@ -13,15 +15,106 @@ FIRMWARE_DIR = ROOT / "firmware"
 OTA_OUTPUT_DIR = OUTPUT_DIR / "ota"
 BUILD_DIR = ROOT / "build"
 
+SECTOR_SIZE = 4096
+
 AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate")
 AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging")
 
-def checksum(fn):
+GPT = namedtuple('GPT', ['lun', 'name', 'path', 'start_sector', 'num_sectors', 'has_ab', 'ota', 'full_check', 'sparse'])
+GPTS = [
+  GPT(0, 'gpt_main_0', FIRMWARE_DIR / 'gpt_main_0.img', 0, 6, False, False, True, False),
+  GPT(1, 'gpt_main_1', FIRMWARE_DIR / 'gpt_main_1.img', 0, 6, False, False, True, False),
+  GPT(2, 'gpt_main_2', FIRMWARE_DIR / 'gpt_main_2.img', 0, 6, False, False, True, False),
+  GPT(3, 'gpt_main_3', FIRMWARE_DIR / 'gpt_main_3.img', 0, 6, False, False, True, False),
+  GPT(4, 'gpt_main_4', FIRMWARE_DIR / 'gpt_main_4.img', 0, 6, False, False, True, False),
+  GPT(5, 'gpt_main_5', FIRMWARE_DIR / 'gpt_main_5.img', 0, 6, False, False, True, False),
+]
+
+Partition = namedtuple('Partition', ['name', 'path', 'has_ab', 'ota', 'full_check', 'sparse'])
+PARTITIONS = [
+  Partition('persist', FIRMWARE_DIR / 'persist.img', False, False, True, False),
+  Partition('systemrw', FIRMWARE_DIR / 'systemrw.img', False, False, True, False),
+  Partition('cache', FIRMWARE_DIR / 'cache.img', False, False, True, False),
+  Partition('xbl', FIRMWARE_DIR / 'xbl.img', True, True, True, False),
+  Partition('xbl_config', FIRMWARE_DIR / 'xbl_config.img', True, True, True, False),
+  Partition('abl', FIRMWARE_DIR / 'abl.img', True, True, True, False),
+  Partition('aop', FIRMWARE_DIR / 'aop.img', True, True, True, False),
+  Partition('bluetooth', FIRMWARE_DIR / 'bluetooth.img', True, False, True, False),
+  Partition('cmnlib64', FIRMWARE_DIR / 'cmnlib64.img', True, False, True, False),
+  Partition('cmnlib', FIRMWARE_DIR / 'cmnlib.img', True, False, True, False),
+  Partition('devcfg', FIRMWARE_DIR / 'devcfg.img', True, True, True, False),
+  Partition('devinfo', FIRMWARE_DIR / 'devinfo.img', False, False, True, False),
+  Partition('dsp', FIRMWARE_DIR / 'dsp.img', True, False, True, False),
+  Partition('hyp', FIRMWARE_DIR / 'hyp.img', True, False, True, False),
+  Partition('keymaster', FIRMWARE_DIR / 'keymaster.img', True, False, True, False),
+  Partition('limits', FIRMWARE_DIR / 'limits.img', False, False, True, False),
+  Partition('logfs', FIRMWARE_DIR / 'logfs.img', False, False, True, False),
+  Partition('modem', FIRMWARE_DIR / 'modem.img', True, False, True, False),
+  Partition('qupfw', FIRMWARE_DIR / 'qupfw.img', True, False, True, False),
+  Partition('splash', FIRMWARE_DIR / 'splash.img', False, False, True, False),
+  Partition('storsec', FIRMWARE_DIR / 'storsec.img', True, False, True, False),
+  Partition('tz', FIRMWARE_DIR / 'tz.img', True, False, True, False),
+  Partition('boot', OUTPUT_DIR / 'boot.img', True, True, True, False),
+  Partition('system', OUTPUT_DIR / 'system.img', True, True, False, True),
+  Partition('userdata_90', OUTPUT_DIR / 'userdata_90.img', False, False, True, True),
+  Partition('userdata_89', OUTPUT_DIR / 'userdata_89.img', False, False, True, True),
+  Partition('userdata_30', OUTPUT_DIR / 'userdata_30.img', False, False, True, True),
+]
+
+
+def file_checksum(fn):
   sha256 = hashlib.sha256()
   with open(fn, 'rb') as f:
     for chunk in iter(lambda: f.read(4096), b""):
       sha256.update(chunk)
-  return sha256.hexdigest()
+  return sha256
+
+def ondevice_checksum_sparse(fn, ota):
+  with open(fn, 'rb') as data_source:
+    dat = data_source.read(28)
+    header = struct.unpack("<I4H4I", dat)
+
+    magic = header[0]
+    major_version = header[1]
+    minor_version = header[2]
+    file_hdr_sz = header[3]
+    chunk_hdr_sz = header[4]
+    blk_sz = header[5]
+    total_chunks = header[7]
+
+    assert magic == 0xED26FF3A
+    assert major_version == 1 and minor_version == 0
+    assert file_hdr_sz == 28
+    assert chunk_hdr_sz == 12
+    assert blk_sz == SECTOR_SIZE
+
+    hash_raw = hashlib.sha256()
+    ondevice_hash = hashlib.sha256()
+    total_size = 0
+    for _ in range(total_chunks):
+      header_bin = data_source.read(12)
+
+      header = struct.unpack("<2H2I", header_bin)
+      chunk_type = header[0]
+      chunk_sz = header[2]
+
+      if chunk_type == 0xCAC1: # RAW
+        d = data_source.read(chunk_sz * SECTOR_SIZE)
+        hash_raw.update(d)
+        ondevice_hash.update(d)
+        total_size += chunk_sz * SECTOR_SIZE
+      elif chunk_type == 0xCAC2: # FILL
+        d = data_source.read(4) * (chunk_sz * SECTOR_SIZE // 4)
+        if d[:4] != b'\x00\x00\x00\x00': # For ondevice_hash, treat FILL 0 like DONT_CARE
+          ondevice_hash.update(d)
+        hash_raw.update(d)
+        total_size += chunk_sz * SECTOR_SIZE
+      elif chunk_type == 0xCAC3: # DONT_CARE
+        assert not ota, 'DONT_CARE chunks are currently not supported for OTA'
+      else:
+        raise Exception(f'UNKNOWN SPARSE CHUNK: {hex(chunk_type)}')
+
+    return hash_raw.hexdigest(), ondevice_hash.hexdigest(), total_size
 
 def compress(fin, fout) -> None:
   # since system.img is a squashfs now, we don't rely on this compression.
@@ -30,34 +123,49 @@ def compress(fin, fout) -> None:
   subprocess.check_call(f"xz -0 -T0 -vc {fin} > {fout}", shell=True)
 
 
-def process_file(fn, name, full_check=True, has_ab=True):
-  print(name)
-  hash_raw = hash = checksum(fn)
-  size = fn.stat().st_size
-  print(f"  {size} bytes, hash {hash}")
+def process_file(entry):
+  size = entry.path.stat().st_size
+  print(f"\n{entry.name} {size} bytes")
+
+  sha256 = file_checksum(entry.path)
+  hash = hash_raw = sha256.hexdigest()
+
+  if struct.unpack("<I", open(entry.path, 'rb').read(4))[0] == 0xED26FF3A:
+    hash_raw, ondevice_hash, size = ondevice_checksum_sparse(entry.path, entry.ota)
+  else:
+    sha256.update(b'\x00' * ((SECTOR_SIZE - (size % SECTOR_SIZE)) % SECTOR_SIZE))
+    ondevice_hash = sha256.hexdigest()
 
   print("  compressing")
-  xz_fn = OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img.xz"
-  compress(fn, xz_fn)
+  xz_fn = OTA_OUTPUT_DIR / f"{entry.path.stem}-{hash_raw}.img.xz"
+  compress(entry.path, xz_fn)
 
   ret = {
-    "name": name,
+    "name": entry.name,
     "url": "{remote_url}/" + xz_fn.name,
     "hash": hash,
-    "hash_raw": hash_raw,
-    "size": size,
-    "sparse": False,
-    "full_check": full_check,
-    "has_ab": has_ab,
+    "hash_raw": hash_raw, # for sparse image, this is not the hash of the raw file
+    "size": size, # for sparse image, this is not the size of the raw file
+    "sparse": entry.sparse,
+    "full_check": entry.full_check,
+    "has_ab": entry.has_ab,
+    "ondevice_hash": ondevice_hash,
   }
 
-  if name == "system":
+  if entry.name == "system":
     ret["alt"] = {
       "hash": hash_raw,
       "url": "{remote_url}/" + xz_fn.name.replace(".img.xz", ".img"),
       "size": size,
     }
-    shutil.copy(fn, OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img")
+    shutil.copy(entry.path, OTA_OUTPUT_DIR / f"{entry.path.stem}-{hash_raw}.img")
+
+  if isinstance(entry, GPT):
+    ret["gpt"] = {
+      "lun": entry.lun,
+      "start_sector": entry.start_sector,
+      "num_sectors": entry.num_sectors,
+    }
 
   return ret
 
@@ -65,21 +173,20 @@ def process_file(fn, name, full_check=True, has_ab=True):
 if __name__ == "__main__":
   OTA_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
 
-  files = [
-    process_file(OUTPUT_DIR / "boot.img", "boot"),
-    process_file(OUTPUT_DIR / "system.img", "system", full_check=False),
-  ]
-  for fw in ("xbl", "abl", "xbl_config", "devcfg", "aop"):
-    # firmware not built in this repo
-    files.append(process_file(FIRMWARE_DIR / f"{fw}.bin", fw))
+  files = [(process_file(x), x.ota) for x in GPTS + PARTITIONS]
 
   configs = [
-    (AGNOS_UPDATE_URL, "ota.json"),
-    (AGNOS_STAGING_UPDATE_URL, "ota-staging.json"),
+    # URL, file name, only OTA partitions
+    (AGNOS_UPDATE_URL, "ota.json", True),
+    (AGNOS_STAGING_UPDATE_URL, "ota-staging.json", True),
+    (AGNOS_UPDATE_URL, "all-partitions.json", False),
+    (AGNOS_STAGING_UPDATE_URL, "all-partitions-staging.json", False),
   ]
-  for remote_url, output_fn in configs:
+  for remote_url, output_fn, only_ota in configs:
     processed_files = []
-    for f in deepcopy(files):
+    for f, ota in deepcopy(files):
+      if only_ota and not ota:
+        continue
       f["url"] = f["url"].format(remote_url=remote_url)
       if "alt" in f:
         f["alt"]["url"] = f["alt"]["url"].format(remote_url=remote_url)