diff --git a/.github/workflows/build-all-tinygrad-models.yaml b/.github/workflows/build-all-tinygrad-models.yaml index 30ab1b4b4a..d4deac1fd8 100644 --- a/.github/workflows/build-all-tinygrad-models.yaml +++ b/.github/workflows/build-all-tinygrad-models.yaml @@ -15,6 +15,11 @@ on: options: - qcom - usbgpu + hf_repo: + description: 'Hugging Face dataset repository' + required: false + type: string + default: 'sunnypilot/sunnypilot_models_v1' jobs: setup: @@ -70,45 +75,23 @@ jobs: jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT - - name: Set up SSH - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - - run: | - mkdir -p ~/.ssh - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - - - name: Clone GitLab docs repo and create new recompiled dir + - name: Get next recompiled dir number id: create-recompiled-dir env: - GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts' + HF_REPO: ${{ github.event.inputs.hf_repo }} run: | - git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs - cd gitlab_docs - git checkout main - git sparse-checkout set --no-cone models/ - cd models - latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1) - if [[ -z "$latest_dir" ]]; then - next_dir=1 - else - next_dir=$((latest_dir+1)) - fi - recompiled_dir="${next_dir}" - mkdir -p "recompiled${recompiled_dir}" - touch "recompiled${recompiled_dir}/.gitkeep" - cd ../.. + pip install huggingface_hub + recompiled_dir=$(python3 -c " + from huggingface_hub import HfApi + import re, sys + api = HfApi() + files = api.list_repo_files(repo_id=sys.argv[1], repo_type='dataset') + dirs = [re.search(r'models/recompiled([0-9]+)', f) for f in files] + nums = [int(m.group(1)) for m in dirs if m] + print(max(nums) + 1) + " "$HF_REPO") echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT - - name: Push empty recompiled dir to GitLab - run: | - cd gitlab_docs - git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} - git config --global user.name "GitHub Action" - git config --global user.email "action@github.com" - git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit" - git push origin main - - name: Push new JSON to GitHub docs repo run: | cd docs @@ -133,6 +116,7 @@ jobs: recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }} json_version: ${{ needs.setup.outputs.json_version }} target_hardware: ${{ github.event.inputs.target_hardware }} + hf_repo: ${{ github.event.inputs.hf_repo }} secrets: inherit retry_failed_models: @@ -174,6 +158,7 @@ jobs: json_version: ${{ needs.setup.outputs.json_version }} target_hardware: ${{ github.event.inputs.target_hardware }} artifact_suffix: -retry + hf_repo: ${{ github.event.inputs.hf_repo }} secrets: inherit publish_models: @@ -181,6 +166,9 @@ jobs: needs: [setup, get_and_build, retry_failed_models, retry_get_and_build] if: ${{ !cancelled() && needs.setup.result == 'success' && (needs.get_and_build.result == 'success' || needs.get_and_build.result == 'failure') && (needs.retry_failed_models.result == 'success' || needs.retry_failed_models.result == 'failure' || needs.retry_failed_models.result == 'skipped') && (needs.retry_get_and_build.result == 'success' || needs.retry_get_and_build.result == 'failure' || needs.retry_get_and_build.result == 'skipped') }} runs-on: ubuntu-latest + permissions: + id-token: write + contents: write strategy: fail-fast: false max-parallel: 1 @@ -190,29 +178,8 @@ jobs: RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }} JSON_FILE: ${{ needs.setup.outputs.json_file }} ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }} + HF_REPO: ${{ github.event.inputs.hf_repo }} steps: - - name: Set up SSH - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} - - - name: Add GitLab.com SSH key to known_hosts - run: | - mkdir -p ~/.ssh - ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts - - - name: Clone GitLab docs repo - env: - GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts' - run: | - echo "Cloning GitLab" - git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs - cd gitlab_docs - echo "checkout models/${RECOMPILED_DIR}" - git sparse-checkout set --no-cone models/${RECOMPILED_DIR} - git checkout main - cd .. - - name: Checkout docs repo uses: actions/checkout@v4 with: @@ -221,16 +188,31 @@ jobs: path: docs ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }} + - name: Install huggingface_hub + run: pip install --upgrade "huggingface_hub>=0.22.0" + - name: Validate recompiled dir and JSON version + env: + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} run: | - if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then - echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo" - exit 1 - fi if [ ! -f "$JSON_FILE" ]; then echo "JSON file $JSON_FILE does not exist!" exit 1 fi + python3 -c " + import sys + from huggingface_hub import HfApi + try: + api = HfApi() + files = api.list_repo_files(repo_id=sys.argv[2], repo_type='dataset') + if not any(f.startswith('models/' + sys.argv[1] + '/') for f in files): + print(f'Error: Recompiled dir models/{sys.argv[1]} does not exist in Hugging Face dataset {sys.argv[2]}!') + sys.exit(1) + print(f'Success: Validated that models/{sys.argv[1]} exists on Hugging Face dataset {sys.argv[2]}!') + except Exception as e: + print('HF validation failed:', e) + sys.exit(1) + " "$RECOMPILED_DIR" "$HF_REPO" - name: Download artifact name file id: download-name @@ -254,44 +236,26 @@ jobs: name: ${{ steps.read-artifact-name.outputs.artifact_name }} path: output - - name: Copy model artifacts to gitlab + - name: Create models folder if: steps.download-name.outcome == 'success' env: ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }} run: | - ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}" - mkdir -p "$ARTIFACT_DIR" - for path in output/*; do - if [ "$(basename "$path")" = "artifact_name.txt" ]; then - continue - fi - name="$(basename "$path")" - if [ -d "$path" ]; then - mkdir -p "$ARTIFACT_DIR/$name" - cp -r "$path"/* "$ARTIFACT_DIR/$name/" - echo "Copied dir $name -> $ARTIFACT_DIR/$name" - else - cp "$path" "$ARTIFACT_DIR/" - echo "Copied file $name -> $ARTIFACT_DIR/" - fi - done + mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}" + cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" + rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt" - - name: Push recompiled dir to GitLab + + - name: Upload to Hugging Face if: steps.download-name.outcome == 'success' env: - GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} + HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }} + ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }} run: | - cd gitlab_docs - git checkout main - git pull origin main - for d in models/"$RECOMPILED_DIR"/*/; do - git sparse-checkout add "$d" - done - git add models/"$RECOMPILED_DIR" - git config --global user.name "GitHub Action" - git config --global user.email "action@github.com" - git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit" - git push origin main + hf upload $HF_REPO \ + output/ \ + "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \ + --repo-type=dataset - name: Pull gh-pages if: steps.download-name.outcome == 'success' run: | @@ -307,7 +271,7 @@ jobs: ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\"" eval python3 docs/json_parser.py \ --json-path "$JSON_FILE" \ - --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \ + --recompiled-dir "local_models/$RECOMPILED_DIR" \ $ARGS - name: Push updated json to GitHub diff --git a/.github/workflows/build-single-tinygrad-model.yaml b/.github/workflows/build-single-tinygrad-model.yaml index d0ff34a5c7..ea17c95950 100644 --- a/.github/workflows/build-single-tinygrad-model.yaml +++ b/.github/workflows/build-single-tinygrad-model.yaml @@ -182,7 +182,7 @@ jobs: name: ${{ steps.read-artifact-name.outputs.artifact_name }} path: output - - name: make models folder + - name: Create models folder env: ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }} run: |