From b040fcd2a9c00f334b7306230e8dfa2efd4aeb57 Mon Sep 17 00:00:00 2001 From: Andrei Radulescu Date: Thu, 8 Aug 2024 00:24:59 +0300 Subject: [PATCH 1/4] build and push full release in CI to commaai/ci-artifacts (#287) * package_ota.py & upload to commaai/ci-artifacts * fix simg2img incorrectly using shell=True [upload] * fix output path * [upload] * Apply suggestions from code review --------- Co-authored-by: Adeeb Shihadeh --- .github/workflows/build.yaml | 60 +++++++++++++++++++++++++----------- scripts/package_ota.py | 2 +- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 318624b..2c17212 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -40,6 +40,14 @@ jobs: echo EOF } | tee -a $GITHUB_ENV + - name: Checkout ci-artifacts + uses: actions/checkout@v4 + with: + repository: commaai/ci-artifacts + ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} + path: ${{ github.workspace }}/ci-artifacts + ref: master + - name: Get kernel submodule ref id: kernel-submodule run: echo "ref=$(git ls-tree HEAD | awk '$4 == "agnos-kernel-sdm845"' | awk '{print $3}')" | tee -a $GITHUB_OUTPUT @@ -53,26 +61,42 @@ jobs: - name: Build kernel run: ./build_kernel.sh - - name: Upload artifact boot.img - uses: actions/upload-artifact@v4 - if: "contains(env.LAST_COMMIT_MESSAGE, '[upload]')" - with: - name: boot.img - path: output/boot.img - - - name: Upload artifact kernel modules - uses: actions/upload-artifact@v4 - if: "contains(env.LAST_COMMIT_MESSAGE, '[upload]')" - with: - name: kernel-modules - path: output/*.ko - - name: Build system run: ./build_system.sh - - name: Upload artifact system.img - uses: actions/upload-artifact@v4 + - name: Package and OTA push if: "contains(env.LAST_COMMIT_MESSAGE, '[upload]')" + env: + AGNOS_UPDATE_URL: https://raw.githubusercontent.com/commaai/ci-artifacts/agnos-builder/pr-${{ github.event.number }}/ + run: | + sudo apt-get install -y android-sdk-libsparse-utils + scripts/package_ota.py + + - name: Copy and push boot, system and agnos.json + if: false + working-directory: ${{ github.workspace }}/ci-artifacts + run: | + cp ${{ github.workspace }}/output/ota/*.img.xz . + cp ${{ github.workspace }}/output/ota/ota.json agnos.json + git checkout -b agnos-builder/pr-${{ github.event.number }} + git config user.name "GitHub Actions Bot" + git config user.email "<>" + git add ${{ github.workspace }}/ci-artifacts/* + git commit -m "build artifacts for PR #${{ github.event.number }}" + git push origin agnos-builder/pr-${{ github.event.number }} --force + + - name: Comment on PR + if: false + uses: thollander/actions-comment-pull-request@v2 with: - name: system.img - path: output/system.img + message: | + + ## Build agnos.json + Download agnos.json and replace `openpilot/system/hardware/tici/agnos.json` in your openpilot branch. + + If you need to flash locally, you can download the images and unarchive them in `agnos-builder/output` and flash with `./flash_all.sh`. + + --- + comment_tag: run_id + pr_number: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/package_ota.py b/scripts/package_ota.py index b0ed8b8..a03ea21 100755 --- a/scripts/package_ota.py +++ b/scripts/package_ota.py @@ -34,7 +34,7 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) if sparse: with NamedTemporaryFile() as tmp_f: print(" converting sparse image to raw") - subprocess.check_call(["simg2img", fn, tmp_f.name], shell=True) + subprocess.check_call(["simg2img", fn, tmp_f.name]) hash_raw = checksum(tmp_f.name) size = Path(tmp_f.name).stat().st_size print(f" {size} bytes, hash {hash} (raw)") From 2c6b35d6e14a3135724483042b1ce274ebf83949 Mon Sep 17 00:00:00 2001 From: Andrei Radulescu Date: Thu, 8 Aug 2024 02:34:42 +0300 Subject: [PATCH 2/4] package_ota: remove converting sparse to raw (#289) * package_ota: don't convert sparse image to raw if raw images already exists * don't convert anymore --- .github/workflows/build.yaml | 1 - scripts/package_ota.py | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2c17212..28beb76 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -69,7 +69,6 @@ jobs: env: AGNOS_UPDATE_URL: https://raw.githubusercontent.com/commaai/ci-artifacts/agnos-builder/pr-${{ github.event.number }}/ run: | - sudo apt-get install -y android-sdk-libsparse-utils scripts/package_ota.py - name: Copy and push boot, system and agnos.json diff --git a/scripts/package_ota.py b/scripts/package_ota.py index a03ea21..8334bd7 100755 --- a/scripts/package_ota.py +++ b/scripts/package_ota.py @@ -5,7 +5,6 @@ import hashlib import subprocess from copy import deepcopy from pathlib import Path -from tempfile import NamedTemporaryFile ROOT = Path(__file__).parent.parent OUTPUT_DIR = ROOT / "output" @@ -32,12 +31,12 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) print(f" {size} bytes, hash {hash}") if sparse: - with NamedTemporaryFile() as tmp_f: - print(" converting sparse image to raw") - subprocess.check_call(["simg2img", fn, tmp_f.name]) - hash_raw = checksum(tmp_f.name) - size = Path(tmp_f.name).stat().st_size - print(f" {size} bytes, hash {hash} (raw)") + raw_img = OUTPUT_DIR / "system.raw.img" + if raw_img.exists(): + print(" using existing raw image") + hash_raw = checksum(raw_img) + size = raw_img.stat().st_size + print(f" {size} bytes, hash {hash_raw} (raw)") print(" compressing") xz_fn = OTA_OUTPUT_DIR / f"{fn.stem}-{hash_raw}.img.xz" From b33ebce21db8e7b07899cef32c54ece9a05fbd06 Mon Sep 17 00:00:00 2001 From: Andrei Radulescu Date: Thu, 8 Aug 2024 20:28:11 +0300 Subject: [PATCH 3/4] package_ota: fix path for raw image (#292) --- scripts/package_ota.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/package_ota.py b/scripts/package_ota.py index 8334bd7..d10a3d7 100755 --- a/scripts/package_ota.py +++ b/scripts/package_ota.py @@ -9,6 +9,7 @@ from pathlib import Path ROOT = Path(__file__).parent.parent OUTPUT_DIR = ROOT / "output" OTA_OUTPUT_DIR = OUTPUT_DIR / "ota" +BUILD_DIR = ROOT / "build" AGNOS_UPDATE_URL = os.getenv("AGNOS_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate") AGNOS_STAGING_UPDATE_URL = os.getenv("AGNOS_STAGING_UPDATE_URL", "https://commadist.azureedge.net/agnosupdate-staging") @@ -31,11 +32,14 @@ def process_file(fn, name, sparse=False, full_check=True, has_ab=True, alt=None) print(f" {size} bytes, hash {hash}") if sparse: - raw_img = OUTPUT_DIR / "system.raw.img" + raw_img = BUILD_DIR / "system.img.raw" if raw_img.exists(): print(" using existing raw image") hash_raw = checksum(raw_img) size = raw_img.stat().st_size + else: + print("Error: existing raw image not found") + exit(1) print(f" {size} bytes, hash {hash_raw} (raw)") print(" compressing") From ba34182d5d109defafdc210726ac178502d54dd6 Mon Sep 17 00:00:00 2001 From: Robin Reckmann Date: Fri, 9 Aug 2024 09:09:47 +0900 Subject: [PATCH 4/4] Point pip3 to shimd pip (#294) --- userspace/usr/comma/shims/pip3 | 1 + 1 file changed, 1 insertion(+) create mode 120000 userspace/usr/comma/shims/pip3 diff --git a/userspace/usr/comma/shims/pip3 b/userspace/usr/comma/shims/pip3 new file mode 120000 index 0000000..4d6aa0d --- /dev/null +++ b/userspace/usr/comma/shims/pip3 @@ -0,0 +1 @@ +pip \ No newline at end of file