diff --git a/.github/workflows/build-default-big-model.yaml b/.github/workflows/build-default-big-model.yaml new file mode 100644 index 0000000000..1a734a3af8 --- /dev/null +++ b/.github/workflows/build-default-big-model.yaml @@ -0,0 +1,82 @@ +name: Build default big model + +on: + push: + branches: [ master, master-dev ] + paths: + - 'openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx' + workflow_dispatch: + +env: + HF_REPO: sunnypilot/sunnypilot_models_v1 + HF_DEFAULTS_PATH: models/defaults/big + +jobs: + build_model: + uses: ./.github/workflows/sunnypilot-build-model.yaml + with: + upstream_branch: ${{ github.sha }} + custom_name: default-big-model + target_hardware: usbgpu + secrets: inherit + + upload_defaults: + needs: build_model + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" + + - name: Install huggingface_hub + run: pip install --upgrade "huggingface_hub>=0.22.0" + + - name: Download artifact name + uses: actions/download-artifact@v4 + with: + name: artifact-name-default-big-model + path: artifact_name + + - name: Read artifact name + id: artifact + run: | + ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt) + echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT + + - name: Download model artifact + uses: actions/download-artifact@v4 + with: + name: ${{ steps.artifact.outputs.artifact_name }} + path: output + + - name: Upload model to HF defaults + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }} + run: | + rm -f output/artifact_name.txt + hf upload ${{ env.HF_REPO }} \ + output/ \ + "${HF_DEFAULTS_PATH}/${ARTIFACT_NAME}/" \ + --repo-type=dataset + + - name: Get tinygrad ref and ONNX hash + id: meta + run: | + export PYTHONPATH=$(pwd) + echo "tinygrad_ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" >> $GITHUB_OUTPUT + echo "onnx_sha256=$(sha256sum openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx | cut -d' ' -f1)" >> $GITHUB_OUTPUT + + - name: Update default_models.json on HF + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }} + run: | + python3 release/ci/upload_default_model.py \ + --hf-repo "${{ env.HF_REPO }}" \ + --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \ + --artifact-name "$ARTIFACT_NAME" \ + --metadata-path "output/metadata.json" \ + --onnx-sha256 "${{ steps.meta.outputs.onnx_sha256 }}" \ + --tinygrad-ref "${{ steps.meta.outputs.tinygrad_ref }}" diff --git a/release/ci/upload_default_model.py b/release/ci/upload_default_model.py new file mode 100644 index 0000000000..ab31dcabce --- /dev/null +++ b/release/ci/upload_default_model.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" + +import argparse +import json +import sys +import tempfile + +from huggingface_hub import HfApi, hf_hub_download + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--hf-repo", required=True) + parser.add_argument("--hf-defaults-path", required=True) + parser.add_argument("--artifact-name", required=True) + parser.add_argument("--metadata-path", required=True) + parser.add_argument("--onnx-sha256", required=True) + parser.add_argument("--tinygrad-ref", required=True) + args = parser.parse_args() + + with open(args.metadata_path) as f: + metadata = json.load(f) + + bundle = metadata['bundles'][0] + bundle['onnx_sha256'] = args.onnx_sha256 + + artifact = bundle['models'][0]['artifact'] + hf_base = f"https://huggingface.co/datasets/{args.hf_repo}/resolve/main/{args.hf_defaults_path}/{args.artifact_name}" + artifact['download_uri']['url'] = f"{hf_base}/{artifact['file_name']}" + for chunk in artifact.get('chunks', []): + chunk['url'] = f"{hf_base}/{chunk['file_name']}" + + json_filename = f"{args.hf_defaults_path}/default_models.json" + try: + local_path = hf_hub_download(repo_id=args.hf_repo, repo_type='dataset', filename=json_filename) + with open(local_path) as f: + defaults_json = json.load(f) + except Exception: + defaults_json = {"tinygrad_ref": args.tinygrad_ref, "bundles": []} + + defaults_json['tinygrad_ref'] = args.tinygrad_ref + + existing_idx = next((i for i, b in enumerate(defaults_json['bundles']) + if b.get('display_name') == bundle.get('display_name')), None) + if existing_idx is not None: + defaults_json['bundles'][existing_idx] = bundle + else: + defaults_json['bundles'].append(bundle) + + print(json.dumps(defaults_json, indent=2)) + + api = HfApi() + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(defaults_json, f, indent=2) + tmp_path = f.name + + api.upload_file( + path_or_fileobj=tmp_path, + path_in_repo=json_filename, + repo_id=args.hf_repo, + repo_type="dataset", + ) + + print(f"Updated {json_filename}") + + +if __name__ == "__main__": + main()