split split

2026-08-31 13:03:45 +08:00 · 2026-08-24 21:25:33 -04:00
parent b5813ffc9b
commit e79c052d5f
2 changed files with 140 additions and 106 deletions
@@ -0,0 +1,66 @@
+name: Download HF model chunks
+description: Resolve and download model chunks from HuggingFace in parallel
+
+inputs:
+  hf_repo:
+    description: HuggingFace dataset repo
+    required: true
+  models:
+    description: 'JSON array of {hf_path, onnx_hash, canonical} objects'
+    required: true
+  dest_dir:
+    description: Destination directory for downloaded chunks
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Download model chunks
+      shell: bash
+      env:
+        HF_REPO: ${{ inputs.hf_repo }}
+        MODELS_JSON: ${{ inputs.models }}
+        DEST_DIR: ${{ inputs.dest_dir }}
+      run: |
+        set -eo pipefail
+        DOWNLOAD_LIST=$(mktemp)
+
+        resolve_chunks() {
+          local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4"
+          local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json"
+          local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS
+          DEFAULTS=$(curl -fsSL "$JSON_URL")
+          BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+          ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
+          BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
+          NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+
+          mkdir -p "$DEST_DIR"
+          while IFS= read -r CHUNK_NAME; do
+            CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
+            if [ -z "$CHUNK_IDX" ]; then
+              echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
+              return 1
+            fi
+            ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
+            printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST"
+          done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name')
+          echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest"
+        }
+
+        echo "$MODELS_JSON" | jq -c '.[]' | while IFS= read -r model; do
+          HF_PATH=$(echo "$model" | jq -r '.hf_path')
+          ONNX_HASH=$(echo "$model" | jq -r '.onnx_hash')
+          CANONICAL=$(echo "$model" | jq -r '.canonical')
+          resolve_chunks "$HF_PATH" "$ONNX_HASH" "$CANONICAL" "$DEST_DIR"
+        done
+
+        TOTAL=$(wc -l < "$DOWNLOAD_LIST")
+        echo "Downloading $TOTAL chunks with 8 parallel connections..."
+        xargs -P8 -d'\n' -I{} bash -c '
+          URL="${1%%	*}"
+          DEST="${1#*	}"
+          echo "Downloading $(basename "$DEST")"
+          curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL"
+        ' _ {} < "$DOWNLOAD_LIST"
+        rm -f "$DOWNLOAD_LIST"
@@ -405,10 +405,9 @@ jobs:
      needs.prepare_strategy.result == 'success' &&
      needs.prepare_small_model.result == 'success' &&
      needs.prepare_dm_model.result == 'success' &&
-      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
-      (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
+      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
      }}
-    needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ]
+    needs: [ build, prepare_strategy, prepare_small_model, prepare_dm_model ]
    runs-on: ubuntu-24.04
    environment: ${{ needs.prepare_strategy.outputs.environment }}
    steps:
@@ -424,85 +423,23 @@ jobs:
          mkdir -p ${{ env.OUTPUT_DIR }}
          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}

-      - name: Prepare chestnut output
-        if: ${{ needs.prepare_chestnut.result == 'success' }}
-        run: |
-          mkdir -p "${{ github.workspace }}/chestnut_output"
-          tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
-
-      - name: Download all model chunks from HF
-        env:
-          HF_REPO: sunnypilot/sunnypilot_models_v1
-          INCLUDE_BIG: ${{ needs.prepare_chestnut.result == 'success' && 'true' || 'false' }}
-        run: |
-          set -eo pipefail
-          MODELS_DIR="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models"
-          DOWNLOAD_LIST=$(mktemp)
-
-          resolve_chunks() {
-            local HF_PATH="$1" ONNX_HASH="$2" CANONICAL="$3" DEST_DIR="$4"
-            local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_PATH}/default_models.json"
-            local DEFAULTS BUNDLE ARTIFACT BASE_URL NUM_CHUNKS
-            DEFAULTS=$(curl -fsSL "$JSON_URL")
-            BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
-            ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
-            BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
-            NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
-
-            mkdir -p "$DEST_DIR"
-            while IFS= read -r CHUNK_NAME; do
-              CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
-              if [ -z "$CHUNK_IDX" ]; then
-                echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
-                return 1
-              fi
-              ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
-              printf '%s\t%s\n' "$ENCODED_URL" "${DEST_DIR}/${CANONICAL}.chunk${CHUNK_IDX}" >> "$DOWNLOAD_LIST"
-            done < <(echo "$ARTIFACT" | jq -r '.chunks[].file_name')
-            echo "$NUM_CHUNKS" > "${DEST_DIR}/${CANONICAL}.chunkmanifest"
-          }
-
-          resolve_chunks "models/defaults/small" \
-            "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}" \
-            "driving_tinygrad.pkl" "$MODELS_DIR"
-          resolve_chunks "models/defaults/dm" \
-            "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}" \
-            "dmonitoring_model_tinygrad.pkl" "$MODELS_DIR"
-
-          if [ "$INCLUDE_BIG" = "true" ]; then
-            mkdir -p big_model_chunks
-            resolve_chunks "models/defaults/big" \
-              "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}" \
-              "big_driving_tinygrad.pkl" "big_model_chunks"
-          fi
-
-          TOTAL=$(wc -l < "$DOWNLOAD_LIST")
-          echo "Downloading $TOTAL chunks with 8 parallel connections..."
-          xargs -P8 -d'\n' -I{} bash -c '
-            URL="${1%%	*}"
-            DEST="${1#*	}"
-            echo "Downloading $(basename "$DEST")"
-            curl -fsSL --retry 3 --retry-delay 5 -o "$DEST" "$URL"
-          ' _ {} < "$DOWNLOAD_LIST"
-          rm -f "$DOWNLOAD_LIST"
-
-      - name: Inject models into chestnut
-        if: ${{ needs.prepare_chestnut.result == 'success' }}
-        run: |
-          CHESTNUT_MODELS="${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models"
-          MAIN_MODELS="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models"
-          cp big_model_chunks/* "$CHESTNUT_MODELS/"
-          cp "$MAIN_MODELS"/driving_tinygrad.pkl.chunk* "$CHESTNUT_MODELS/"
-          cp "$MAIN_MODELS"/driving_tinygrad.pkl.chunkmanifest "$CHESTNUT_MODELS/"
-          cp "$MAIN_MODELS"/dmonitoring_model_tinygrad.pkl.chunk* "$CHESTNUT_MODELS/"
-          cp "$MAIN_MODELS"/dmonitoring_model_tinygrad.pkl.chunkmanifest "$CHESTNUT_MODELS/"
+      - name: Download model chunks from HF
+        uses: ./.github/workflows/download-hf-model-chunks
+        with:
+          hf_repo: sunnypilot/sunnypilot_models_v1
+          dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models
+          models: |
+            [
+              {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"},
+              {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"}
+            ]

      - name: Configure Git
        run: |
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "github-actions[bot]"

-      - name: Publish branches
+      - name: Publish to Public Repository
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
@@ -515,36 +452,7 @@ jobs:
            "${{ needs.prepare_strategy.outputs.new_branch }}" \
            "${{ needs.prepare_strategy.outputs.version }}" \
            "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" &
-          MAIN_PID=$!
-
-          CHESTNUT_PID=""
-          if [ "${{ needs.prepare_chestnut.result }}" = "success" ]; then
-            CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
-            CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
-
-            ${{ env.CI_DIR }}/publish.sh \
-              "${{ github.workspace }}" \
-              "$CHESTNUT_DIR" \
-              "$CHESTNUT_BRANCH" \
-              "${{ needs.prepare_strategy.outputs.version }}" \
-              "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-              "${{ needs.prepare_strategy.outputs.extra_version_identifier }}" &
-            CHESTNUT_PID=$!
-          fi
-
-          wait $MAIN_PID
-          MAIN_RC=$?
-          CHESTNUT_RC=0
-          if [ -n "$CHESTNUT_PID" ]; then
-            wait $CHESTNUT_PID
-            CHESTNUT_RC=$?
-          fi
-
-          if [ $MAIN_RC -ne 0 ] || [ $CHESTNUT_RC -ne 0 ]; then
-            echo "::error::Publish failed (main=$MAIN_RC, chestnut=$CHESTNUT_RC)"
-            exit 1
-          fi
+            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"

      - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
        if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
@@ -553,11 +461,71 @@ jobs:
            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
            git push -f origin ${TAG}

+  publish_chestnut:
+    concurrency:
+      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}-chestnut
+      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
+    if: ${{
+      always() && !cancelled() &&
+      needs.build.result == 'success' &&
+      needs.prepare_strategy.result == 'success' &&
+      needs.prepare_small_model.result == 'success' &&
+      needs.prepare_dm_model.result == 'success' &&
+      needs.prepare_chestnut.result == 'success' &&
+      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
+      }}
+    needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ]
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download prebuilt artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: prebuilt
+
+      - name: Untar prebuilt
+        run: |
+          mkdir -p ${{ env.OUTPUT_DIR }}
+          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
+
+      - name: Download model chunks from HF
+        uses: ./.github/workflows/download-hf-model-chunks
+        with:
+          hf_repo: sunnypilot/sunnypilot_models_v1
+          dest_dir: ${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models
+          models: |
+            [
+              {"hf_path": "models/defaults/small", "onnx_hash": "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}", "canonical": "driving_tinygrad.pkl"},
+              {"hf_path": "models/defaults/dm", "onnx_hash": "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}", "canonical": "dmonitoring_model_tinygrad.pkl"},
+              {"hf_path": "models/defaults/big", "onnx_hash": "${{ needs.prepare_chestnut.outputs.onnx_sha256 }}", "canonical": "big_driving_tinygrad.pkl"}
+            ]
+
+      - name: Configure Git
+        run: |
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+
+      - name: Publish chestnut branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
+
+          ${{ env.CI_DIR }}/publish.sh \
+            "${{ github.workspace }}" \
+            "${{ env.OUTPUT_DIR }}" \
+            "$CHESTNUT_BRANCH" \
+            "${{ needs.prepare_strategy.outputs.version }}" \
+            "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
+            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
+
  notify:
    needs:
    - prepare_strategy
    - build
    - publish
+    - publish_chestnut
    - prepare_chestnut
    - prepare_small_model
    - prepare_dm_model