diff --git a/.github/workflows/download-hf-model-chunks/action.yml b/.github/workflows/download-hf-model-chunks/action.yml new file mode 100644 index 0000000000..01ab5385da --- /dev/null +++ b/.github/workflows/download-hf-model-chunks/action.yml @@ -0,0 +1,66 @@ +name: Download HF model chunks +description: Resolve and download model chunks from HuggingFace in parallel + +inputs: + hf_repo: + description: HuggingFace dataset repo + required: true + models: + description: 'JSON array of {hf_path, onnx_hash, canonical} objects' + required: true + dest_dir: + description: Destination directory for downloaded chunks + required: true + +runs: + using: composite + steps: + - name: Download model chunks + shell: bash + env: + HF_REPO: ${{ inputs.hf_repo }} + MODELS_JSON: ${{ inputs.models }} + DEST_DIR: ${{ inputs.dest_dir }} + run: | + set -eo pipefail + DOWNLOAD_LIST=$(mktemp) + + resolve_chunks() { + local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4" + local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json" + local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS + DEFAULTS=$(curl -fsSL "$JSON_URL") + BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') + ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') + BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') + NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') + + mkdir -p "$DEST_DIR" + while IFS= read -r CHUNK_NAME; do + CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true) + if [ -z "$CHUNK_IDX" ]; then + echo "::error::Failed to parse chunk index from: $CHUNK_NAME" + return 1 + fi + ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") + printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST" + done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name') + echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest" + } + + echo "$MODELS_JSON" | jq -c '.[]' | while IFS= read -r model; do + HF_PATH=$(echo "$model" | jq -r '.hf_path') + ONNX_HASH=$(echo "$model" | jq -r '.onnx_hash') + CANONICAL=$(echo "$model" | jq -r '.canonical') + resolve_chunks "$HF_PATH" "$ONNX_HASH" "$CANONICAL" "$DEST_DIR" + done + + TOTAL=$(wc -l < "$DOWNLOAD_LIST") + echo "Downloading $TOTAL chunks with 8 parallel connections..." + xargs -P8 -d'\n' -I{} bash -c ' + URL="${1%% *}" + DEST="${1#* }" + echo "Downloading $(basename "$DEST")" + curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL" + ' _ {} < "$DOWNLOAD_LIST" + rm -f "$DOWNLOAD_LIST" diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml index 4081a7507d..75e32ed156 100644 --- a/.github/workflows/sunnypilot-build-prebuilt.yaml +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -405,10 +405,9 @@ jobs: needs.prepare_strategy.result == 'success' && needs.prepare_small_model.result == 'success' && needs.prepare_dm_model.result == 'success' && - (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) && - (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success') + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }} - needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] + needs: [ build, prepare_strategy, prepare_small_model, prepare_dm_model ] runs-on: ubuntu-24.04 environment: ${{ needs.prepare_strategy.outputs.environment }} steps: @@ -424,85 +423,23 @@ jobs: mkdir -p ${{ env.OUTPUT_DIR }} tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} - - name: Prepare chestnut output - if: ${{ needs.prepare_chestnut.result == 'success' }} - run: | - mkdir -p "${{ github.workspace }}/chestnut_output" - tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output" - - - name: Download all model chunks from HF - env: - HF_REPO: sunnypilot/sunnypilot_models_v1 - INCLUDE_BIG: ${{ needs.prepare_chestnut.result == 'success' && 'true' || 'false' }} - run: | - set -eo pipefail - MODELS_DIR="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models" - DOWNLOAD_LIST=$(mktemp) - - resolve_chunks() { - local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4" - local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json" - local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS - DEFAULTS=$(curl -fsSL "$JSON_URL") - BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)') - ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact') - BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||') - NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length') - - mkdir -p "$DEST_DIR" - while IFS= read -r CHUNK_NAME; do - CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true) - if [ -z "$CHUNK_IDX" ]; then - echo "::error::Failed to parse chunk index from: $CHUNK_NAME" - return 1 - fi - ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))") - printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST" - done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name') - echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest" - } - - resolve_chunks "models/defaults/small" \ - "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}" \ - "driving_tinygrad.pkl" "$MODELS_DIR" - resolve_chunks "models/defaults/dm" \ - "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}" \ - "dmonitoring_model_tinygrad.pkl" "$MODELS_DIR" - - if [ "$INCLUDE_BIG" = "true" ]; then - mkdir -p big_model_chunks - resolve_chunks "models/defaults/big" \ - "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}" \ - "big_driving_tinygrad.pkl" "big_model_chunks" - fi - - TOTAL=$(wc -l < "$DOWNLOAD_LIST") - echo "Downloading $TOTAL chunks with 8 parallel connections..." - xargs -P8 -d'\n' -I{} bash -c ' - URL="${1%% *}" - DEST="${1#* }" - echo "Downloading $(basename "$DEST")" - curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL" - ' _ {} < "$DOWNLOAD_LIST" - rm -f "$DOWNLOAD_LIST" - - - name: Inject models into chestnut - if: ${{ needs.prepare_chestnut.result == 'success' }} - run: | - CHESTNUT_MODELS="${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models" - MAIN_MODELS="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models" - cp big_model_chunks/* "$CHESTNUT_MODELS/" - cp "$MAIN_MODELS"/driving_tinygrad.pkl.chunk* "$CHESTNUT_MODELS/" - cp "$MAIN_MODELS"/driving_tinygrad.pkl.chunkmanifest "$CHESTNUT_MODELS/" - cp "$MAIN_MODELS"/dmonitoring_model_tinygrad.pkl.chunk* "$CHESTNUT_MODELS/" - cp "$MAIN_MODELS"/dmonitoring_model_tinygrad.pkl.chunkmanifest "$CHESTNUT_MODELS/" + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"} + ] - name: Configure Git run: | git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" - - name: Publish branches + - name: Publish to Public Repository env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -515,36 +452,7 @@ jobs: "${{ needs.prepare_strategy.outputs.new_branch }}" \ "${{ needs.prepare_strategy.outputs.version }}" \ "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ - "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" & - MAIN_PID=$! - - CHESTNUT_PID="" - if [ "${{ needs.prepare_chestnut.result }}" = "success" ]; then - CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" - CHESTNUT_DIR="${{ github.workspace }}/chestnut_output" - - ${{ env.CI_DIR }}/publish.sh \ - "${{ github.workspace }}" \ - "$CHESTNUT_DIR" \ - "$CHESTNUT_BRANCH" \ - "${{ needs.prepare_strategy.outputs.version }}" \ - "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ - "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" & - CHESTNUT_PID=$! - fi - - wait $MAIN_PID - MAIN_RC=$? - CHESTNUT_RC=0 - if [ -n "$CHESTNUT_PID" ]; then - wait $CHESTNUT_PID - CHESTNUT_RC=$? - fi - - if [ $MAIN_RC -ne 0 ] || [ $CHESTNUT_RC -ne 0 ]; then - echo "::error::Publish failed (main=$MAIN_RC, chestnut=$CHESTNUT_RC)" - exit 1 - fi + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" - name: Tag ${{ needs.prepare_strategy.outputs.environment }} if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }} @@ -553,11 +461,71 @@ jobs: git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}." git push -f origin ${TAG} + publish_chestnut: + concurrency: + group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}-chestnut + cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }} + if: ${{ + always() && !cancelled() && + needs.build.result == 'success' && + needs.prepare_strategy.result == 'success' && + needs.prepare_small_model.result == 'success' && + needs.prepare_dm_model.result == 'success' && + needs.prepare_chestnut.result == 'success' && + (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) + }} + needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Download prebuilt artifact + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Untar prebuilt + run: | + mkdir -p ${{ env.OUTPUT_DIR }} + tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + + - name: Download model chunks from HF + uses: ./.github/workflows/download-hf-model-chunks + with: + hf_repo: sunnypilot/sunnypilot_models_v1 + dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models + models: | + [ + {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"}, + {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"}, + {"hf_path": "models/defaults/big", "onnx_hash": "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}", "canonical": "big_driving_tinygrad.pkl"} + ] + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + - name: Publish chestnut branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut" + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "${{ env.OUTPUT_DIR }}" \ + "$CHESTNUT_BRANCH" \ + "${{ needs.prepare_strategy.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" + notify: needs: - prepare_strategy - build - publish + - publish_chestnut - prepare_chestnut - prepare_small_model - prepare_dm_model