ci: chestnut prebuilt branches

.github/workflows/build-default-big-model.yaml

@@ -0,0 +1,64 @@
+name: Build default big model
+
+on:
+  push:
+    branches: [ master, master-dev ]
+    paths:
+      - 'openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx'
+  workflow_dispatch:
+
+jobs:
+  build_model:
+    uses: ./.github/workflows/sunnypilot-build-model.yaml
+    with:
+      upstream_branch: ${{ github.sha }}
+      custom_name: default-big-model
+      target_hardware: usbgpu
+    secrets: inherit
+
+  upload_defaults:
+    needs: build_model
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+
+      - name: Download artifact name
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-name-default-big-model
+          path: artifact_name
+
+      - name: Download model artifact
+        id: artifact
+        run: |
+          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.artifact.outputs.artifact_name }}
+          path: model_output
+
+      - name: Get tinygrad ref
+        id: tinygrad
+        run: |
+          source /etc/profile 2>/dev/null || true
+          export PYTHONPATH=$(pwd)
+          REF=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
+          echo "ref=$REF" >> $GITHUB_OUTPUT
+
+      - name: Install huggingface_hub
+        run: pip install --upgrade "huggingface_hub>=0.22.0"
+
+      - name: Upload to HF defaults
+        env:
+          HF_OIDC_RESOURCE: datasets/sunnypilot/sunnypilot_models_v1
+        run: |
+          python3 release/ci/upload_default_model.py \
+            --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \
+            --model-dir model_output \
+            --tinygrad-ref "${{ steps.tinygrad.outputs.ref }}" \
+            --hf-repo sunnypilot/sunnypilot_models_v1

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -36,6 +36,7 @@ jobs:
       publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
       is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
       build: ${{ steps.strategy.outputs.build }}
+      include_big_model: ${{ steps.strategy.outputs.include_big_model }}
     steps:
       - uses: actions/checkout@v4
       - name: Extract deploy strategy
@@ -78,6 +79,9 @@ jobs:
             stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
             echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
             echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
+
+            include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')";
+            echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT
           fi
           echo "build=$BUILD" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
@@ -203,6 +207,84 @@ jobs:
           source ${UV_PROJECT_ENVIRONMENT}/bin/activate
           PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
 
+  prepare_chestnut:
+    needs: [ build, prepare_strategy ]
+    runs-on: ubuntu-24.04
+    if: ${{
+      always() && !cancelled() &&
+      needs.build.result == 'success' &&
+      needs.prepare_strategy.outputs.include_big_model == 'true'
+      }}
+    env:
+      HF_REPO: sunnypilot/sunnypilot_models_v1
+      HF_DEFAULTS_PATH: models/defaults
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+
+      - name: Wait for default big model build
+        uses: ./.github/workflows/wait-for-action
+        with:
+          workflow: build-default-big-model.yaml
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          should-wait-for-start: 'true'
+          wait-time: '45'
+
+      - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+
+      - name: Verify and download big model from HF
+        run: |
+          MANIFEST_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/manifest.json"
+          MANIFEST=$(curl -fsSL "$MANIFEST_URL")
+
+          EXPECTED_ONNX_HASH=$(echo "$MANIFEST" | jq -r '.big.onnx_sha256')
+          ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
+
+          echo "Repo ONNX hash:     $ACTUAL_ONNX_HASH"
+          echo "Manifest ONNX hash: $EXPECTED_ONNX_HASH"
+          if [ "$ACTUAL_ONNX_HASH" != "$EXPECTED_ONNX_HASH" ]; then
+            echo "::error::ONNX hash mismatch — run build-default-big-model workflow first"
+            exit 1
+          fi
+
+          NUM_CHUNKS=$(echo "$MANIFEST" | jq -r '.big.artifact.num_chunks')
+          PKL_NAME=$(echo "$MANIFEST" | jq -r '.big.artifact.file_name')
+
+      - name: Download prebuilt artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: prebuilt
+
+      - name: Inject big model into prebuilt
+        run: |
+          mkdir -p chestnut_output
+          tar xzf prebuilt.tar.gz -C chestnut_output
+
+          MODELS_DEST="chestnut_output/openpilot/selfdrive/modeld/models"
+          MANIFEST_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/manifest.json"
+          MANIFEST=$(curl -fsSL "$MANIFEST_URL")
+          NUM_CHUNKS=$(echo "$MANIFEST" | jq -r '.big.artifact.num_chunks')
+          PKL_NAME=$(echo "$MANIFEST" | jq -r '.big.artifact.file_name')
+
+          CHUNK_BASE_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/big"
+          for i in $(seq 1 $NUM_CHUNKS); do
+            PADDED=$(printf "%02dof%02d" "$i" "$NUM_CHUNKS")
+            CHUNK_FILE="${PKL_NAME}.chunk${PADDED}"
+            echo "Downloading $CHUNK_FILE"
+            curl -fsSL -o "${MODELS_DEST}/${CHUNK_FILE}" "${CHUNK_BASE_URL}/${CHUNK_FILE}"
+          done
+
+          echo "$NUM_CHUNKS" > "${MODELS_DEST}/${PKL_NAME}.chunkmanifest"
+
+          tar czf prebuilt-chestnut.tar.gz -C chestnut_output .
+
+      - name: Upload chestnut artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: prebuilt-chestnut
+          path: prebuilt-chestnut.tar.gz
+          compression-level: 0
 
   publish:
     concurrency:
@@ -211,14 +293,20 @@ jobs:
       # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
       group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
       cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
-    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build, prepare_strategy ]
+    if: ${{
+      always() && !cancelled() &&
+      needs.build.result == 'success' &&
+      needs.prepare_strategy.result == 'success' &&
+      (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
+      (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
+      }}
+    needs: [ build, prepare_strategy, prepare_chestnut ]
     runs-on: ubuntu-24.04
     environment: ${{ needs.prepare_strategy.outputs.environment }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download build artifacts
+      - name: Download prebuilt artifact
         uses: actions/download-artifact@v4
         with:
           name: prebuilt
@@ -228,6 +316,18 @@ jobs:
           mkdir -p ${{ env.OUTPUT_DIR }}
           tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
 
+      - name: Download chestnut artifact
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: prebuilt-chestnut
+
+      - name: Prepare chestnut output
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        run: |
+          mkdir -p "${{ github.workspace }}/chestnut_output"
+          tar xzf prebuilt-chestnut.tar.gz -C "${{ github.workspace }}/chestnut_output"
+
       - name: Configure Git
         run: |
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
@@ -248,6 +348,22 @@ jobs:
             "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
             "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
 
+      - name: Publish chestnut branch
+        if: ${{ needs.prepare_chestnut.result == 'success' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
+          CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
+
+          ${{ env.CI_DIR }}/publish.sh \
+            "${{ github.workspace }}" \
+            "$CHESTNUT_DIR" \
+            "$CHESTNUT_BRANCH" \
+            "${{ needs.prepare_strategy.outputs.version }}" \
+            "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
+            "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
+
       - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
         if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
         run: |
@@ -260,6 +376,7 @@ jobs:
     - prepare_strategy
     - build
     - publish
+    - prepare_chestnut
     runs-on: ubuntu-24.04
     if: ${{ (always() && !cancelled() && !failure())
       && needs.publish.result == 'success'
@@ -279,6 +396,7 @@ jobs:
           export commit_short_sha="${commit_short_sha:0:7}"
           export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
           export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
+          export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}"
 
           MESSAGE=$(cat << 'EOF' | envsubst
           ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}

release/ci/model_generator.py

@@ -90,6 +90,18 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path:
   return old_pkl.rename(new_pkl)
 
 
+def _hash_onnx_files(model_dir: Path) -> str | None:
+  onnx_files = sorted(model_dir.glob("*.onnx"))
+  if not onnx_files:
+    return None
+  digest = hashlib.sha256()
+  for f in onnx_files:
+    with f.open('rb') as fh:
+      while block := fh.read(1024 * 1024):
+        digest.update(block)
+  return digest.hexdigest()
+
+
 def generate_chunked_model(driving_pkl: Path) -> dict:
   tinygrad_hash = _hash_pkl(driving_pkl)
 
@@ -123,7 +135,8 @@ def generate_chunked_model(driving_pkl: Path) -> dict:
   }
 
 
-def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown") -> None:
+def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown",
+                         onnx_sha256=None) -> None:
   bundle_json = {
     "short_name": short_name,
     "display_name": custom_name or upstream_branch,
@@ -139,6 +152,9 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short
     "models": models,
   }
 
+  if onnx_sha256:
+    bundle_json["onnx_sha256"] = onnx_sha256
+
   # Write metadata to output_dir
   metadata_json = {
     "bundles": [bundle_json]
@@ -178,4 +194,6 @@ if __name__ == "__main__":
       _driving_pkl = new_pkl
 
   _model_metadata = generate_chunked_model(_driving_pkl)
-  create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch)
+  _onnx_sha256 = _hash_onnx_files(Path(args.model_dir))
+  create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch,
+                       onnx_sha256=_onnx_sha256)

release/ci/upload_default_model.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""
+Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+
+This file is part of sunnypilot and is licensed under the MIT License.
+See the LICENSE.md file in the root directory for more details.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+import tempfile
+import urllib.request
+from datetime import datetime, UTC
+from pathlib import Path
+
+
+def hash_file(path: str) -> str:
+  digest = hashlib.sha256()
+  with open(path, 'rb') as f:
+    while block := f.read(1024 * 1024):
+      digest.update(block)
+  return digest.hexdigest()
+
+
+def fetch_catalog(catalog_url: str) -> dict:
+  with urllib.request.urlopen(catalog_url) as resp:
+    return json.loads(resp.read())
+
+
+def find_default_bundle(catalog: dict, default_name: str) -> dict | None:
+  for bundle in catalog.get('bundles', []):
+    if bundle.get('display_name', '').startswith(default_name):
+      return bundle
+  return None
+
+
+def _collect_local_chunks(model_dir: Path) -> tuple[list[dict], str, int]:
+  canonical_name = "big_driving_tinygrad.pkl"
+  chunk_files = sorted(model_dir.glob("*.chunk*"))
+  if not chunk_files:
+    chunk_files = sorted(model_dir.glob("driving_*_tinygrad.pkl.chunk*"))
+
+  chunks = []
+  for f in chunk_files:
+    if f.suffix == '.chunkmanifest':
+      continue
+    name = f.name
+    canonical = re.sub(r'driving_.*_tinygrad', 'big_driving_tinygrad', name)
+    chunks.append({"file_name": canonical, "sha256": hash_file(str(f)), "src": str(f)})
+
+  num_chunks = len(chunks)
+
+  # hash all chunks together for the full pkl hash
+  digest = hashlib.sha256()
+  for c in chunks:
+    with open(c['src'], 'rb') as fh:
+      while block := fh.read(1024 * 1024):
+        digest.update(block)
+  pkl_sha256 = digest.hexdigest()
+
+  return chunks, pkl_sha256, num_chunks
+
+
+def _prepare_from_catalog(args, onnx_hash: str) -> tuple[str, dict]:
+  catalog = fetch_catalog(args.catalog_url)
+  tinygrad_ref = catalog.get('tinygrad_ref', '')
+  print(f"Catalog tinygrad_ref: {tinygrad_ref}")
+
+  bundle = find_default_bundle(catalog, args.default_name)
+  if not bundle:
+    print(f"No bundle found starting with '{args.default_name}'", file=sys.stderr)
+    sys.exit(1)
+
+  print(f"Found bundle: {bundle['display_name']} (ref={bundle.get('ref', '?')})")
+
+  artifact = bundle['models'][0]['artifact']
+  source_chunks = artifact.get('chunks', [])
+  source_url = artifact['download_uri']['url']
+  pkl_sha256 = artifact['download_uri']['sha256']
+
+  canonical_name = "big_driving_tinygrad.pkl"
+  num_chunks = len(source_chunks)
+
+  tmpdir = tempfile.mkdtemp()
+  big_dir = os.path.join(tmpdir, "big")
+  os.makedirs(big_dir)
+
+  chunk_hashes = []
+  for i, chunk in enumerate(source_chunks):
+    src_chunk_name = chunk['file_name']
+    canonical_chunk = f"{canonical_name}.chunk{i+1:02d}of{num_chunks:02d}"
+    chunk_url = source_url.rsplit('/', 1)[0] + '/' + src_chunk_name
+    dest_path = os.path.join(big_dir, canonical_chunk)
+
+    print(f"  {src_chunk_name} -> {canonical_chunk}")
+    if not args.dry_run:
+      urllib.request.urlretrieve(chunk_url, dest_path)
+      actual_hash = hash_file(dest_path)
+      if actual_hash != chunk['sha256']:
+        print(f"  Hash mismatch for {canonical_chunk}", file=sys.stderr)
+        sys.exit(1)
+      chunk_hashes.append({"file_name": canonical_chunk, "sha256": actual_hash})
+    else:
+      chunk_hashes.append({"file_name": canonical_chunk, "sha256": chunk['sha256']})
+
+  manifest = {
+    "big": {
+      "name": args.default_name,
+      "onnx_sha256": onnx_hash,
+      "tinygrad_ref": tinygrad_ref,
+      "source_ref": bundle.get('ref', ''),
+      "artifact": {
+        "file_name": canonical_name,
+        "sha256": pkl_sha256,
+        "num_chunks": num_chunks,
+        "chunks": chunk_hashes
+      },
+      "updated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+    }
+  }
+
+  return big_dir, manifest
+
+
+def _prepare_from_local(args, onnx_hash: str) -> tuple[str, dict]:
+  model_dir = Path(args.model_dir)
+  chunks, pkl_sha256, num_chunks = _collect_local_chunks(model_dir)
+
+  if not chunks:
+    print(f"No chunk files found in {model_dir}", file=sys.stderr)
+    sys.exit(1)
+
+  canonical_name = "big_driving_tinygrad.pkl"
+  tmpdir = tempfile.mkdtemp()
+  big_dir = os.path.join(tmpdir, "big")
+  os.makedirs(big_dir)
+
+  chunk_hashes = []
+  for c in chunks:
+    dest = os.path.join(big_dir, c['file_name'])
+    os.link(c['src'], dest) if not args.dry_run else None
+    chunk_hashes.append({"file_name": c['file_name'], "sha256": c['sha256']})
+
+  tinygrad_ref = args.tinygrad_ref or ''
+
+  manifest = {
+    "big": {
+      "name": args.default_name,
+      "onnx_sha256": onnx_hash,
+      "tinygrad_ref": tinygrad_ref,
+      "source_ref": "",
+      "artifact": {
+        "file_name": canonical_name,
+        "sha256": pkl_sha256,
+        "num_chunks": num_chunks,
+        "chunks": chunk_hashes
+      },
+      "updated_at": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+    }
+  }
+
+  return big_dir, manifest
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--onnx-path", required=True)
+  parser.add_argument("--model-dir", help="Local directory with compiled model chunks (skips catalog)")
+  parser.add_argument("--tinygrad-ref", help="Tinygrad ref (used with --model-dir)")
+  parser.add_argument("--catalog-url", default="https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v21.json")
+  parser.add_argument("--default-name", default="Lebowski")
+  parser.add_argument("--hf-repo", default="sunnypilot/sunnypilot_models_v1")
+  parser.add_argument("--dry-run", action="store_true")
+  args = parser.parse_args()
+
+  onnx_hash = hash_file(args.onnx_path)
+  print(f"ONNX hash: {onnx_hash}")
+
+  if args.model_dir:
+    big_dir, manifest = _prepare_from_local(args, onnx_hash)
+  else:
+    big_dir, manifest = _prepare_from_catalog(args, onnx_hash)
+
+  manifest_path = os.path.join(os.path.dirname(big_dir), "manifest.json")
+  with open(manifest_path, 'w') as f:
+    json.dump(manifest, f, indent=2)
+
+  print(json.dumps(manifest, indent=2))
+
+  if args.dry_run:
+    print("[DRY RUN]")
+    return
+
+  try:
+    from huggingface_hub import HfApi
+  except ImportError:
+    print("pip install huggingface_hub", file=sys.stderr)
+    sys.exit(1)
+
+  api = HfApi()
+
+  api.upload_file(
+    path_or_fileobj=manifest_path,
+    path_in_repo="models/defaults/manifest.json",
+    repo_id=args.hf_repo,
+    repo_type="dataset",
+  )
+
+  api.upload_folder(
+    folder_path=big_dir,
+    path_in_repo="models/defaults/big",
+    repo_id=args.hf_repo,
+    repo_type="dataset",
+  )
+
+  print("Done.")
+
+
+if __name__ == "__main__":
+  main()