diff --git a/.github/workflows/build-default-big-model.yaml b/.github/workflows/build-default-big-model.yaml new file mode 100644 index 0000000000..8cda5f7aa1 --- /dev/null +++ b/.github/workflows/build-default-big-model.yaml @@ -0,0 +1,64 @@ +name: Build default big model + +on: + push: + branches: [ master, master-dev ] + paths: + - 'openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx' + workflow_dispatch: + +jobs: + build_model: + uses: ./.github/workflows/sunnypilot-build-model.yaml + with: + upstream_branch: ${{ github.sha }} + custom_name: default-big-model + target_hardware: usbgpu + secrets: inherit + + upload_defaults: + needs: build_model + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" + + - name: Download artifact name + uses: actions/download-artifact@v4 + with: + name: artifact-name-default-big-model + path: artifact_name + + - name: Download model artifact + id: artifact + run: | + ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) + echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT + + - uses: actions/download-artifact@v4 + with: + name: ${{ steps.artifact.outputs.artifact_name }} + path: model_output + + - name: Get tinygrad ref + id: tinygrad + run: | + source /etc/profile 2>/dev/null || true + export PYTHONPATH=$(pwd) + REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py) + echo "ref=$REF" >> $GITHUB_OUTPUT + + - name: Install huggingface_hub + run: pip install --upgrade "huggingface_hub>=0.22.0" + + - name: Upload to HF defaults + env: + HF_OIDC_RESOURCE: datasets/sunnypilot/sunnypilot_models_v1 + run: | + python3 release/ci/upload_default_model.py \ + --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \ + --model-dir model_output \ + --tinygrad-ref "${{ steps.tinygrad.outputs.ref }}" \ + --hf-repo sunnypilot/sunnypilot_models_v1 diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml index db0d0f55b4..4efce4e832 100644 --- a/.github/workflows/sunnypilot-build-prebuilt.yaml +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -36,6 +36,7 @@ jobs: publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }} is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }} build: ${{ steps.strategy.outputs.build }} + include_big_model: ${{ steps.strategy.outputs.include_big_model }} steps: - uses: actions/checkout@v4 - name: Extract deploy strategy @@ -78,6 +79,9 @@ jobs: stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g'); echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT + + include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')"; + echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT fi echo "build=$BUILD" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT @@ -203,6 +207,84 @@ jobs: source ${UV_PROJECT_ENVIRONMENT}/bin/activate PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable + prepare_chestnut: + needs: [ build, prepare_strategy ] + runs-on: ubuntu-24.04 + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.outputs.include_big_model == 'true' + }} + env: + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref || github.ref_name }} + + - name: Wait for default big model build + uses: ./.github/workflows/wait-for-action + with: + workflow: build-default-big-model.yaml + github-token: ${{ secrets.GITHUB_TOKEN }} + should-wait-for-start: 'true' + wait-time: '45' + + - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" + + - name: Verify and download big model from HF + run: | + MANIFEST_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/manifest.json" + MANIFEST=$(curl -fsSL "$MANIFEST_URL") + + EXPECTED_ONNX_HASH=$(echo "$MANIFEST" | jq -r '.big.onnx_sha256') + ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1) + + echo "Repo ONNX hash: $ACTUAL_ONNX_HASH" + echo "Manifest ONNX hash: $EXPECTED_ONNX_HASH" + if [ "$ACTUAL_ONNX_HASH" != "$EXPECTED_ONNX_HASH" ]; then + echo "::error::ONNX hash mismatch — run build-default-big-model workflow first" + exit 1 + fi + + NUM_CHUNKS=$(echo "$MANIFEST" | jq -r '.big.artifact.num_chunks') + PKL_NAME=$(echo "$MANIFEST" | jq -r '.big.artifact.file_name') + + - name: Download prebuilt artifact + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Inject big model into prebuilt + run: | + mkdir -p chestnut_output + tar xzf prebuilt.tar.gz -C chestnut_output + + MODELS_DEST="chestnut_output/openpilot/selfdrive/modeld/models" + MANIFEST_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/manifest.json" + MANIFEST=$(curl -fsSL "$MANIFEST_URL") + NUM_CHUNKS=$(echo "$MANIFEST" | jq -r '.big.artifact.num_chunks') + PKL_NAME=$(echo "$MANIFEST" | jq -r '.big.artifact.file_name') + + CHUNK_BASE_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/big" + for i in $(seq 1 $NUM_CHUNKS); do + PADDED=$(printf "%02dof%02d" "$i" "$NUM_CHUNKS") + CHUNK_FILE="${PKL_NAME}.chunk${PADDED}" + echo "Downloading $CHUNK_FILE" + curl -fsSL -o "${MODELS_DEST}/${CHUNK_FILE}" "${CHUNK_BASE_URL}/${CHUNK_FILE}" + done + + echo "$NUM_CHUNKS" > "${MODELS_DEST}/${PKL_NAME}.chunkmanifest" + + tar czf prebuilt-chestnut.tar.gz -C chestnut_output . + + - name: Upload chestnut artifact + uses: actions/upload-artifact@v4 + with: + name: prebuilt-chestnut + path: prebuilt-chestnut.tar.gz + compression-level: 0 publish: concurrency: @@ -211,14 +293,20 @@ jobs: # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time. group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }} cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} - if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }} - needs: [ build, prepare_strategy ] + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.result == 'success' && + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) && + (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success') + }} + needs: [ build, prepare_strategy, prepare_chestnut ] runs-on: ubuntu-24.04 environment: ${{ needs.prepare_strategy.outputs.environment }} steps: - uses: actions/checkout@v4 - - name: Download build artifacts + - name: Download prebuilt artifact uses: actions/download-artifact@v4 with: name: prebuilt @@ -228,6 +316,18 @@ jobs: mkdir -p ${{ env.OUTPUT_DIR }} tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + - name: Download chestnut artifact + if: ${{ needs.prepare_chestnut.result == 'success' }} + uses: actions/download-artifact@v4 + with: + name: prebuilt-chestnut + + - name: Prepare chestnut output + if: ${{ needs.prepare_chestnut.result == 'success' }} + run: | + mkdir -p "${{ github.workspace }}/chestnut_output" + tar xzf prebuilt-chestnut.tar.gz -C "${{ github.workspace }}/chestnut_output" + - name: Configure Git run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" @@ -248,6 +348,22 @@ jobs: "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" + - name: Publish chestnut branch + if: ${{ needs.prepare_chestnut.result == 'success' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" + CHESTNUT_DIR="${{ github.workspace }}/chestnut_output" + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "$CHESTNUT_DIR" \ + "$CHESTNUT_BRANCH" \ + "${{ needs.prepare_strategy.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" + - name: Tag ${{ needs.prepare_strategy.outputs.environment }} if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }} run: | @@ -260,6 +376,7 @@ jobs: - prepare_strategy - build - publish + - prepare_chestnut runs-on: ubuntu-24.04 if: ${{ (always() && !cancelled() && !failure()) && needs.publish.result == 'success' @@ -279,6 +396,7 @@ jobs: export commit_short_sha="${commit_short_sha:0:7}" export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}" export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}" + export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}" MESSAGE=$(cat << 'EOF' | envsubst ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }} diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index 80b102a6f1..ff9be64783 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -90,6 +90,18 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path: return old_pkl.rename(new_pkl) +def _hash_onnx_files(model_dir: Path) -> str | None: + onnx_files = sorted(model_dir.glob("*.onnx")) + if not onnx_files: + return None + digest = hashlib.sha256() + for f in onnx_files: + with f.open('rb') as fh: + while block := fh.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + def generate_chunked_model(driving_pkl: Path) -> dict: tinygrad_hash = _hash_pkl(driving_pkl) @@ -123,7 +135,8 @@ def generate_chunked_model(driving_pkl: Path) -> dict: } -def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown") -> None: +def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown", + onnx_sha256=None) -> None: bundle_json = { "short_name": short_name, "display_name": custom_name or upstream_branch, @@ -139,6 +152,9 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short "models": models, } + if onnx_sha256: + bundle_json["onnx_sha256"] = onnx_sha256 + # Write metadata to output_dir metadata_json = { "bundles": [bundle_json] @@ -178,4 +194,6 @@ if __name__ == "__main__": _driving_pkl = new_pkl _model_metadata = generate_chunked_model(_driving_pkl) - create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch) + _onnx_sha256 = _hash_onnx_files(Path(args.model_dir)) + create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch, + onnx_sha256=_onnx_sha256) diff --git a/release/ci/upload_default_model.py b/release/ci/upload_default_model.py new file mode 100644 index 0000000000..bea0a23066 --- /dev/null +++ b/release/ci/upload_default_model.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" + +import argparse +import hashlib +import json +import os +import re +import sys +import tempfile +import urllib.request +from datetime import datetime, UTC +from pathlib import Path + + +def hash_file(path: str) -> str: + digest = hashlib.sha256() + with open(path, 'rb') as f: + while block := f.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def fetch_catalog(catalog_url: str) -> dict: + with urllib.request.urlopen(catalog_url) as resp: + return json.loads(resp.read()) + + +def find_default_bundle(catalog: dict, default_name: str) -> dict | None: + for bundle in catalog.get('bundles', []): + if bundle.get('display_name', '').startswith(default_name): + return bundle + return None + + +def _collect_local_chunks(model_dir: Path) -> tuple[list[dict], str, int]: + canonical_name = "big_driving_tinygrad.pkl" + chunk_files = sorted(model_dir.glob("*.chunk*")) + if not chunk_files: + chunk_files = sorted(model_dir.glob("driving_*_tinygrad.pkl.chunk*")) + + chunks = [] + for f in chunk_files: + if f.suffix == '.chunkmanifest': + continue + name = f.name + canonical = re.sub(r'driving_.*_tinygrad', 'big_driving_tinygrad', name) + chunks.append({"file_name": canonical, "sha256": hash_file(str(f)), "src": str(f)}) + + num_chunks = len(chunks) + + # hash all chunks together for the full pkl hash + digest = hashlib.sha256() + for c in chunks: + with open(c['src'], 'rb') as fh: + while block := fh.read(1024 * 1024): + digest.update(block) + pkl_sha256 = digest.hexdigest() + + return chunks, pkl_sha256, num_chunks + + +def _prepare_from_catalog(args, onnx_hash: str) -> tuple[str, dict]: + catalog = fetch_catalog(args.catalog_url) + tinygrad_ref = catalog.get('tinygrad_ref', '') + print(f"Catalog tinygrad_ref: {tinygrad_ref}") + + bundle = find_default_bundle(catalog, args.default_name) + if not bundle: + print(f"No bundle found starting with '{args.default_name}'", file=sys.stderr) + sys.exit(1) + + print(f"Found bundle: {bundle['display_name']} (ref={bundle.get('ref', '?')})") + + artifact = bundle['models'][0]['artifact'] + source_chunks = artifact.get('chunks', []) + source_url = artifact['download_uri']['url'] + pkl_sha256 = artifact['download_uri']['sha256'] + + canonical_name = "big_driving_tinygrad.pkl" + num_chunks = len(source_chunks) + + tmpdir = tempfile.mkdtemp() + big_dir = os.path.join(tmpdir, "big") + os.makedirs(big_dir) + + chunk_hashes = [] + for i, chunk in enumerate(source_chunks): + src_chunk_name = chunk['file_name'] + canonical_chunk = f"{canonical_name}.chunk{i+1:02d}of{num_chunks:02d}" + chunk_url = source_url.rsplit('/', 1)[0] + '/' + src_chunk_name + dest_path = os.path.join(big_dir, canonical_chunk) + + print(f" {src_chunk_name} -> {canonical_chunk}") + if not args.dry_run: + urllib.request.urlretrieve(chunk_url, dest_path) + actual_hash = hash_file(dest_path) + if actual_hash != chunk['sha256']: + print(f" Hash mismatch for {canonical_chunk}", file=sys.stderr) + sys.exit(1) + chunk_hashes.append({"file_name": canonical_chunk, "sha256": actual_hash}) + else: + chunk_hashes.append({"file_name": canonical_chunk, "sha256": chunk['sha256']}) + + manifest = { + "big": { + "name": args.default_name, + "onnx_sha256": onnx_hash, + "tinygrad_ref": tinygrad_ref, + "source_ref": bundle.get('ref', ''), + "artifact": { + "file_name": canonical_name, + "sha256": pkl_sha256, + "num_chunks": num_chunks, + "chunks": chunk_hashes + }, + "updated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + } + } + + return big_dir, manifest + + +def _prepare_from_local(args, onnx_hash: str) -> tuple[str, dict]: + model_dir = Path(args.model_dir) + chunks, pkl_sha256, num_chunks = _collect_local_chunks(model_dir) + + if not chunks: + print(f"No chunk files found in {model_dir}", file=sys.stderr) + sys.exit(1) + + canonical_name = "big_driving_tinygrad.pkl" + tmpdir = tempfile.mkdtemp() + big_dir = os.path.join(tmpdir, "big") + os.makedirs(big_dir) + + chunk_hashes = [] + for c in chunks: + dest = os.path.join(big_dir, c['file_name']) + os.link(c['src'], dest) if not args.dry_run else None + chunk_hashes.append({"file_name": c['file_name'], "sha256": c['sha256']}) + + tinygrad_ref = args.tinygrad_ref or '' + + manifest = { + "big": { + "name": args.default_name, + "onnx_sha256": onnx_hash, + "tinygrad_ref": tinygrad_ref, + "source_ref": "", + "artifact": { + "file_name": canonical_name, + "sha256": pkl_sha256, + "num_chunks": num_chunks, + "chunks": chunk_hashes + }, + "updated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + } + } + + return big_dir, manifest + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--onnx-path", required=True) + parser.add_argument("--model-dir", help="Local directory with compiled model chunks (skips catalog)") + parser.add_argument("--tinygrad-ref", help="Tinygrad ref (used with --model-dir)") + parser.add_argument("--catalog-url", default="https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json") + parser.add_argument("--default-name", default="Lebowski") + parser.add_argument("--hf-repo", default="sunnypilot/sunnypilot_models_v1") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + onnx_hash = hash_file(args.onnx_path) + print(f"ONNX hash: {onnx_hash}") + + if args.model_dir: + big_dir, manifest = _prepare_from_local(args, onnx_hash) + else: + big_dir, manifest = _prepare_from_catalog(args, onnx_hash) + + manifest_path = os.path.join(os.path.dirname(big_dir), "manifest.json") + with open(manifest_path, 'w') as f: + json.dump(manifest, f, indent=2) + + print(json.dumps(manifest, indent=2)) + + if args.dry_run: + print("[DRY RUN]") + return + + try: + from huggingface_hub import HfApi + except ImportError: + print("pip install huggingface_hub", file=sys.stderr) + sys.exit(1) + + api = HfApi() + + api.upload_file( + path_or_fileobj=manifest_path, + path_in_repo="models/defaults/manifest.json", + repo_id=args.hf_repo, + repo_type="dataset", + ) + + api.upload_folder( + folder_path=big_dir, + path_in_repo="models/defaults/big", + repo_id=args.hf_repo, + repo_type="dataset", + ) + + print("Done.") + + +if __name__ == "__main__": + main()