Compare commits

...

2 Commits

Author SHA1 Message Date
discountchubbs 5b7ddce980 replace oauth with token 2026-09-02 22:45:27 -07:00
Jason Wen e87dbbaba7 models: sanitize default model name for HF (#1984) 2026-09-02 14:53:31 -04:00
4 changed files with 19 additions and 19 deletions
@@ -78,6 +78,7 @@ jobs:
- name: Get next recompiled dir number
id: create-recompiled-dir
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
HF_REPO: ${{ github.event.inputs.hf_repo }}
run: |
pip install huggingface_hub
+12 -9
View File
@@ -30,6 +30,7 @@ jobs:
runs-on: ubuntu-24.04
outputs:
model_name: ${{ steps.resolve.outputs.model_name }}
safe_model_name: ${{ steps.resolve.outputs.safe_model_name }}
onnx_ref: ${{ steps.resolve.outputs.onnx_ref }}
onnx_path: ${{ steps.resolve.outputs.onnx_path }}
hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }}
@@ -64,7 +65,9 @@ jobs:
exit 1
fi
SAFE_NAME="${NAME// /-}"
echo "model_name=${NAME}" >> $GITHUB_OUTPUT
echo "safe_model_name=${SAFE_NAME}" >> $GITHUB_OUTPUT
echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT
echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT
echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT
@@ -135,7 +138,7 @@ jobs:
- name: Prepare output
env:
MODEL_NAME: ${{ needs.resolve.outputs.model_name }}
MODEL_NAME: ${{ needs.resolve.outputs.safe_model_name }}
run: |
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
export PYTHONPATH=${{ github.workspace }}
@@ -158,13 +161,13 @@ jobs:
- name: Upload small model artifact
uses: actions/upload-artifact@v4
with:
name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }}
name: model-${{ needs.resolve.outputs.safe_model_name }}-${{ github.run_number }}
path: ${{ github.workspace }}/small_output/
- name: Upload artifact name file
uses: actions/upload-artifact@v4
with:
name: artifact-name-${{ needs.resolve.outputs.model_name }}
name: artifact-name-${{ needs.resolve.outputs.safe_model_name }}
path: ${{ github.workspace }}/small_output/artifact_name.txt
- name: Re-enable powersave
@@ -254,7 +257,7 @@ jobs:
- name: Prepare output
env:
MODEL_NAME: ${{ needs.resolve.outputs.model_name }}
MODEL_NAME: ${{ needs.resolve.outputs.safe_model_name }}
run: |
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
export PYTHONPATH=${{ github.workspace }}
@@ -277,13 +280,13 @@ jobs:
- name: Upload big model artifact
uses: actions/upload-artifact@v4
with:
name: model-${{ needs.resolve.outputs.model_name }}-${{ github.run_number }}
name: model-${{ needs.resolve.outputs.safe_model_name }}-${{ github.run_number }}
path: ${{ github.workspace }}/big_output/
- name: Upload artifact name file
uses: actions/upload-artifact@v4
with:
name: artifact-name-${{ needs.resolve.outputs.model_name }}
name: artifact-name-${{ needs.resolve.outputs.safe_model_name }}
path: ${{ github.workspace }}/big_output/artifact_name.txt
- name: Re-enable powersave
@@ -318,7 +321,7 @@ jobs:
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
uses: actions/download-artifact@v4
with:
name: artifact-name-${{ needs.resolve.outputs.model_name }}
name: artifact-name-${{ needs.resolve.outputs.safe_model_name }}
path: artifact_name
- name: Read artifact name
@@ -338,7 +341,7 @@ jobs:
- name: Upload model to HF
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
env:
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
run: |
rm -f output/artifact_name.txt
@@ -364,7 +367,7 @@ jobs:
- name: Generate DM metadata and upload to HF
if: ${{ inputs.target == 'dm' }}
env:
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
export PYTHONPATH=$(pwd)
python3 -c "
@@ -146,7 +146,7 @@ jobs:
- name: Validate hf_repo and JSON version
env:
HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
if [ ! -f "$JSON_FILE" ]; then
echo "JSON file $JSON_FILE does not exist!"
@@ -155,13 +155,8 @@ jobs:
python3 -c "
import sys
from huggingface_hub import HfApi
try:
api = HfApi()
api.repo_info(repo_id=sys.argv[1], repo_type='dataset')
print(f'Success: Repo {sys.argv[1]} exists.')
except Exception as e:
print('HF validation failed:', e)
sys.exit(1)
HfApi().repo_info(repo_id=sys.argv[1], repo_type='dataset')
print(f'Success: Repo {sys.argv[1]} exists.')
" "${{ inputs.hf_repo }}"
- name: Download artifact name file
@@ -192,7 +187,7 @@ jobs:
- name: Upload to Hugging Face
env:
HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
run: |
hf upload ${{ inputs.hf_repo }} \
+2 -1
View File
@@ -38,7 +38,8 @@ def main():
api = HfApi()
onnx_sha256 = hash_file(args.onnx_path)
short_ref = args.onnx_ref[:8]
folder_name = f"model-{args.model_name}-{short_ref}-{args.run_number}"
safe_name = args.model_name.replace(" ", "-")
folder_name = f"model-{safe_name}-{short_ref}-{args.run_number}"
print(f"ONNX hash: {onnx_sha256}")
print(f"ONNX ref: {args.onnx_ref} (short: {short_ref})")