From 8632029401c914a5b0196dfd0c45e4c0e44f2cd3 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 12 Oct 2024 14:35:52 -0700 Subject: [PATCH] no external deps --- scripts/download-from-manifest.py | 87 ++++++++++++++++--------------- scripts/ota_push.sh | 15 ------ 2 files changed, 45 insertions(+), 57 deletions(-) diff --git a/scripts/download-from-manifest.py b/scripts/download-from-manifest.py index f6c4fd6..0053679 100755 --- a/scripts/download-from-manifest.py +++ b/scripts/download-from-manifest.py @@ -3,79 +3,82 @@ import json import lzma import hashlib import argparse -import requests - +import http.client from pathlib import Path +from urllib.parse import urlparse MASTER_MANIFEST = "https://raw.githubusercontent.com/commaai/openpilot/master/system/hardware/tici/agnos.json" RELEASE_MANIFEST = "https://raw.githubusercontent.com/commaai/openpilot/release3/system/hardware/tici/agnos.json" ROOT = Path(__file__).parent.parent +def http_get(url): + parsed_url = urlparse(url) + conn = http.client.HTTPSConnection(parsed_url.netloc) + conn.request("GET", parsed_url.path) + response = conn.getresponse() + if response.status != 200: + raise Exception(f"Failed to download {url}: {response.status} {response.reason}") + return response + def download_and_decompress(url, expected_hash, filename): - # Ensure parent directories exist filename.parent.mkdir(parents=True, exist_ok=True) - - # check if already downloaded - if Path(filename).is_file(): + + if filename.is_file(): sha256 = hashlib.sha256() with open(filename, 'rb') as f: - for chunk in iter(lambda: f.read(1024*1024), b''): + for chunk in iter(lambda: f.read(1024 * 1024), b''): sha256.update(chunk) - if sha256.hexdigest().lower() == expected_hash.lower(): - print("already downloaded ", filename) + print(f"Already downloaded: {filename}") return 0 - size_counter = 0 - dot_counter = 0 + response = http_get(url) + size = int(response.getheader("Content-Length", 0)) + decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO) sha256 = hashlib.sha256() - with requests.get(url, stream=True, headers={'Accept-Encoding': None}) as download_stream: - download_stream.raise_for_status() - size = int(download_stream.headers.get("Content-Length")) - with open(filename, 'wb') as f: - for chunk in download_stream.iter_content(chunk_size=1024*1024): - decompressed_chunk = decompressor.decompress(chunk) - sha256.update(decompressed_chunk) - f.write(decompressed_chunk) - size_counter += len(chunk) + size_counter = 0 + dot_counter = 0 + + with open(filename, 'wb') as f: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + decompressed_chunk = decompressor.decompress(chunk) + sha256.update(decompressed_chunk) + f.write(decompressed_chunk) + size_counter += len(chunk) + + if size_counter // (1024 * 1024) > dot_counter: + print(f"Downloading '{filename}': {(size_counter * 100) // size}%", end='\r') + dot_counter += 1 - # Every MB - if size_counter//(1024*1024) > dot_counter: - print(f"Downloading '{filename}': {(size_counter*100)//size}%", end='\r') - dot_counter += 1 print(f"Downloading '{filename}': 100%") - assert(sha256.hexdigest().lower() == expected_hash.lower()) + assert sha256.hexdigest().lower() == expected_hash.lower() def load_manifest(url): if Path(url).is_file(): with open(url) as f: return json.loads(f.read()) - r = requests.get(url) - r.raise_for_status() - return json.loads(r.content.decode()) - + response = http_get(url) + content = response.read().decode() + return json.loads(content) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Download AGNOS') - parser.add_argument('--master', action='store_true', - help='Download AGNOS version used in the master branch') - parser.add_argument('--manifest', nargs='?', - help='Download AGNOS from the manifest at this URL') - parser.add_argument('--partitions', nargs='+', default=None, - help='Whitelist of partitions to download') + parser.add_argument('--master', action='store_true', help='Download AGNOS version used in the master branch') + parser.add_argument('--manifest', nargs='?', help='Download AGNOS from the manifest at this URL') + parser.add_argument('--partitions', nargs='+', default=None, help='Whitelist of partitions to download') args = parser.parse_args() - - manifest = RELEASE_MANIFEST - if args.manifest is not None: - manifest = args.manifest - elif args.master: + manifest = RELEASE_MANIFEST if args.manifest is None else args.manifest + if args.master: manifest = MASTER_MANIFEST update = load_manifest(manifest) for partition in update: - if args.partitions is not None and partition['name'] not in args.partitions: + if args.partitions and partition['name'] not in args.partitions: continue - download_and_decompress(partition['url'], partition['hash'], ROOT / "output"/ f"{partition['name']}.img") + download_and_decompress(partition['url'], partition['hash'], ROOT / "output" / f"{partition['name']}.img") diff --git a/scripts/ota_push.sh b/scripts/ota_push.sh index aaa6d0d..ec30f25 100755 --- a/scripts/ota_push.sh +++ b/scripts/ota_push.sh @@ -40,21 +40,6 @@ process_file() { local NAME=$1 local HASH_RAW=$(cat $OTA_JSON | jq -r ".[] | select(.name == \"$NAME\") | .hash_raw") upload_file "$NAME-$HASH_RAW.img.xz" - - # if [ "$NAME" == "system" ]; then - # local CAIBX_FILE_NAME="system-$HASH_RAW.caibx" - # local CHUNKS_FOLDER="system-$HASH_RAW" - - # echo "Copying system.caibx to the cloud..." - # local SYSTEM_CAIBX_PATH="https://$DATA_ACCOUNT.blob.core.windows.net/$DATA_CONTAINER/$CAIBX_FILE_NAME" - # azcopy cp --overwrite=false $OTA_DIR/$CAIBX_FILE_NAME "$SYSTEM_CAIBX_PATH?$DATA_SAS_TOKEN" - # echo " $SYSTEM_CAIBX_PATH" - - # echo "Copying system chunks to the cloud..." - # local SYSTEM_CHUNKS_PATH="https://$DATA_ACCOUNT.blob.core.windows.net/$DATA_CONTAINER" - # azcopy cp --recursive --overwrite=false $OTA_DIR/$CHUNKS_FOLDER "$SYSTEM_CHUNKS_PATH?$DATA_SAS_TOKEN" - # echo " $SYSTEM_CHUNKS_PATH" - # fi } # Generate token