auth

.github/workflows/build-default-big-model.yaml

@@ -19,6 +19,9 @@ jobs:
   upload_defaults:
     needs: build_model
     runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+      contents: write
     steps:
       - uses: actions/checkout@v4
         with:
@@ -46,33 +49,17 @@ jobs:
           name: ${{ steps.artifact.outputs.artifact_name }}
           path: output
 
-      - name: Upload model to HF defaults
+      - name: Upload to HF and update default_models.json
         env:
           HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
           ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
         run: |
           rm -f output/artifact_name.txt
-          hf upload ${{ env.HF_REPO }} \
-            output/ \
-            "${HF_DEFAULTS_PATH}/${ARTIFACT_NAME}/" \
-            --repo-type=dataset
-
-      - name: Get tinygrad ref and ONNX hash
-        id: meta
-        run: |
           export PYTHONPATH=$(pwd)
-          echo "tinygrad_ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" >> $GITHUB_OUTPUT
-          echo "onnx_sha256=$(sha256sum openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx | cut -d' ' -f1)" >> $GITHUB_OUTPUT
-
-      - name: Update default_models.json on HF
-        env:
-          HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
-          ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
-        run: |
           python3 release/ci/upload_default_model.py \
             --hf-repo "${{ env.HF_REPO }}" \
             --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \
             --artifact-name "$ARTIFACT_NAME" \
-            --metadata-path "output/metadata.json" \
-            --onnx-sha256 "${{ steps.meta.outputs.onnx_sha256 }}" \
-            --tinygrad-ref "${{ steps.meta.outputs.tinygrad_ref }}"
+            --model-dir output \
+            --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \
+            --tinygrad-ref "$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)"

release/ci/upload_default_model.py

@@ -7,28 +7,41 @@ See the LICENSE.md file in the root directory for more details.
 """
 
 import argparse
+import hashlib
 import json
-import sys
 import tempfile
 
 from huggingface_hub import HfApi, hf_hub_download
 
 
+def hash_file(path: str) -> str:
+  digest = hashlib.sha256()
+  with open(path, 'rb') as f:
+    while block := f.read(1024 * 1024):
+      digest.update(block)
+  return digest.hexdigest()
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument("--hf-repo", required=True)
   parser.add_argument("--hf-defaults-path", required=True)
   parser.add_argument("--artifact-name", required=True)
-  parser.add_argument("--metadata-path", required=True)
-  parser.add_argument("--onnx-sha256", required=True)
+  parser.add_argument("--model-dir", required=True)
+  parser.add_argument("--onnx-path", required=True)
   parser.add_argument("--tinygrad-ref", required=True)
   args = parser.parse_args()
 
-  with open(args.metadata_path) as f:
+  api = HfApi()
+  onnx_sha256 = hash_file(args.onnx_path)
+  print(f"ONNX hash: {onnx_sha256}")
+
+  metadata_path = f"{args.model_dir}/metadata.json"
+  with open(metadata_path) as f:
     metadata = json.load(f)
 
   bundle = metadata['bundles'][0]
-  bundle['onnx_sha256'] = args.onnx_sha256
+  bundle['onnx_sha256'] = onnx_sha256
 
   artifact = bundle['models'][0]['artifact']
   hf_base = f"https://huggingface.co/datasets/{args.hf_repo}/resolve/main/{args.hf_defaults_path}/{args.artifact_name}"
@@ -36,6 +49,14 @@ def main():
   for chunk in artifact.get('chunks', []):
     chunk['url'] = f"{hf_base}/{chunk['file_name']}"
 
+  print(f"Uploading model to {args.hf_defaults_path}/{args.artifact_name}/")
+  api.upload_folder(
+    folder_path=args.model_dir,
+    path_in_repo=f"{args.hf_defaults_path}/{args.artifact_name}",
+    repo_id=args.hf_repo,
+    repo_type="dataset",
+  )
+
   json_filename = f"{args.hf_defaults_path}/default_models.json"
   try:
     local_path = hf_hub_download(repo_id=args.hf_repo, repo_type='dataset', filename=json_filename)
@@ -47,7 +68,7 @@ def main():
   defaults_json['tinygrad_ref'] = args.tinygrad_ref
 
   existing_idx = next((i for i, b in enumerate(defaults_json['bundles'])
-                       if b.get('display_name') == bundle.get('display_name')), None)
+                       if b.get('onnx_sha256') == onnx_sha256), None)
   if existing_idx is not None:
     defaults_json['bundles'][existing_idx] = bundle
   else:
@@ -55,7 +76,6 @@ def main():
 
   print(json.dumps(defaults_json, indent=2))
 
-  api = HfApi()
   with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
     json.dump(defaults_json, f, indent=2)
     tmp_path = f.name