dunno dunno...

That both build all and sunnypilot-build-model reference

.github/workflows/build-all-tinygrad-models.yaml

@@ -14,36 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       json_file: ${{ steps.get-json.outputs.json_file }}
-    steps:
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-docs
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Get next JSON version to use
-        id: get-json
-        run: |
-          cd docs/docs
-          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
-          next=$((latest+1))
-          json_file="driving_models_v${next}.json"
-          cp "driving_models_v${latest}.json" "$json_file"
-          echo "json_file=$json_file" >> $GITHUB_OUTPUT
-
-      - name: Upload context for next jobs
-        uses: actions/upload-artifact@v4
-        with:
-          name: context
-          path: docs
-
-  build-all:
-    runs-on: ubuntu-latest
-    needs: setup
-    env:
-      JSON_FILE: docs/docs/${{ needs.setup.outputs.json_file }}
+      model_matrix: ${{ steps.set-matrix.outputs.model_matrix }}
     steps:
       - name: Set up SSH
         uses: webfactory/ssh-agent@v0.9.0
@@ -55,6 +26,14 @@ jobs:
           mkdir -p ~/.ssh
           ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
 
+      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/sunnypilot-docs
+          ref: gh-pages
+          path: docs
+          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
+
       - name: Clone GitLab docs repo
         env:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
@@ -68,7 +47,11 @@ jobs:
       - name: Set next recompiled dir
         id: set-recompiled
         run: |
-          cd gitlab_docs/models
+          cd gitlab_docs
+          echo "checkout models/"
+          git sparse-checkout set --no-cone models/
+          git checkout main
+          cd models
           latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
           if [[ -z "$latest_dir" ]]; then
             next_dir=1
@@ -76,124 +59,57 @@ jobs:
             next_dir=$((latest_dir+1))
           fi
           recompiled_dir="recompiled${next_dir}"
-          mkdir -p "$recompiled_dir"
-          echo "RECOMPILED_DIR=$recompiled_dir" >> $GITHUB_ENV
-
-      - name: Download context
-        uses: actions/download-artifact@v4
-        with:
-          name: context
-          path: .
-
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y jq gh
-
-      - name: Build all tinygrad models
-        id: trigger-builds
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          set -e
-          > triggered_run_ids.txt
-          BRANCH="${{ github.event.inputs.branch }}"
-          jq -c '.bundles[] | select(.runner=="tinygrad")' "$JSON_FILE" | while read -r bundle; do
-            ref=$(echo "$bundle" | jq -r '.ref')
-            display_name=$(echo "$bundle" | jq -r '.display_name' | sed 's/ ([^)]*)//g')
-            is_20hz=$(echo "$bundle" | jq -r '.is_20hz')
-            echo "Triggering build for: $display_name ($ref) [20Hz: $is_20hz]"
-            START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-            gh workflow run sunnypilot-build-model.yaml \
-              --repo sunnypilot/sunnypilot \
-              --ref "$BRANCH" \
-              -f upstream_branch="$ref" \
-              -f custom_name="$display_name" \
-              -f is_20hz="$is_20hz"
-            for i in {1..24}; do
-              RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="$BRANCH" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
-              if [ -n "$RUN_ID" ]; then
-                break
-              fi
-              sleep 5
-            done
-            if [ -z "$RUN_ID" ]; then
-              echo "ould not find the triggered workflow run for $display_name ($ref)"
-              exit 1
-            fi
-            echo "$RUN_ID" >> triggered_run_ids.txt
-          done
-
-      - name: Wait for all model builds to finish
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          set -e
-          SUCCESS_RUNS=()
-          FAILED_RUNS=()
-          declare -A RUN_ID_TO_NAME
-
-          while read -r RUN_ID; do
-            ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name' || echo "unknown")
-            RUN_ID_TO_NAME["$RUN_ID"]="$ARTIFACT_NAME"
-          done < triggered_run_ids.txt
-
-          while read -r RUN_ID; do
-            echo "Watching run ID: $RUN_ID"
-            gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
-            CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
-            ARTIFACT_NAME="${RUN_ID_TO_NAME[$RUN_ID]}"
-            echo "Run $RUN_ID ($ARTIFACT_NAME) concluded with: $CONCLUSION"
-            if [[ "$CONCLUSION" == "success" ]]; then
-              SUCCESS_RUNS+=("$RUN_ID")
-            else
-              FAILED_RUNS+=("$RUN_ID")
-            fi
-          done < triggered_run_ids.txt
-
-          if [[ ${#SUCCESS_RUNS[@]} -eq 0 ]]; then
-            echo "All model builds failed. Aborting."
+          if [ -d "$recompiled_dir" ]; then
+            echo "ERROR: $recompiled_dir already exists in GitLab repo"
             exit 1
           fi
+          mkdir -p "$recompiled_dir"
+          touch "$recompiled_dir/.gitkeep"
+          echo "RECOMPILED_DIR=$recompiled_dir" >> $GITHUB_ENV
+          echo "Created new recompiled dir: $recompiled_dir"
+          cd ../..
 
-          if [[ ${#FAILED_RUNS[@]} -gt 0 ]]; then
-            echo "WARNING: The following model builds failed:"
-            for RUN_ID in "${FAILED_RUNS[@]}"; do
-              echo "- $RUN_ID (${RUN_ID_TO_NAME[$RUN_ID]})"
-            done
-            echo "You may want to rerun these models manually."
-          fi
+      - name: Get next JSON version to use (from GitHub docs repo)
+        id: get-json
+        run: |
+          cd docs/docs
+          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
+          next=$((latest+1))
+          json_file="driving_models_v${next}.json"
+          cp "driving_models_v${latest}.json" "$json_file"
+          echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
+          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
+      - name: Extract tinygrad models
+        id: set-matrix
+        working-directory: docs/docs
+        run: |
+          jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name, is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
+          echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
-      - name: Download and extract all model artifacts
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  get_and_build:
+    needs: [setup]
+    uses: ./.github/workflows/sunnypilot-build-model.yaml
+    strategy:
+      matrix:
+        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
+    name: Model [${{ matrix.model.display_name }}] from ref [${{ matrix.model.ref }}]
+    with:
+      upstream_branch: ${{ matrix.model.ref }}
+      custom_name: ${{ matrix.model.display_name }}
+      is_20hz: ${{ matrix.model.is_20hz || true }}
+    secrets: inherit
+
+  push_results:
+    needs: [setup, get_and_build ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all model artifacts
         run: |
           ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
-          SUCCESS_RUNS=()
-          while read -r RUN_ID; do
-            CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
-            if [[ "$CONCLUSION" == "success" ]]; then
-              SUCCESS_RUNS+=("$RUN_ID")
-            fi
-          done < triggered_run_ids.txt
-
-          for RUN_ID in "${SUCCESS_RUNS[@]}"; do
-            ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
-            echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
-            mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
-            echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
-            gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
-            echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
-            ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
-            if [ -n "$ZIP_PATH" ]; then
-              echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
-              unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
-              rm -f "$ZIP_PATH"
-              echo "Unzipped and removed $ZIP_PATH"
-            else
-              echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME (This is NOT an error)."
-            fi
-            echo "Done processing $ARTIFACT_NAME"
+          mkdir -p "$ARTIFACT_DIR"
+          for name in $(ls ${{ github.workspace }}/output); do
+            mkdir -p "$ARTIFACT_DIR/$name"
+            cp -r ${{ github.workspace }}/output/$name/* "$ARTIFACT_DIR/$name/"
           done
 
       - name: Push recompiled dir to GitLab
@@ -212,7 +128,7 @@ jobs:
       - name: Run json_parser.py to update JSON
         run: |
           python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
+            --json-path "${{ needs.setup.outputs.json_file }}" \
             --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR"
 
       - name: Push updated JSON to GitHub docs repo
@@ -221,6 +137,6 @@ jobs:
           git config --global user.name "GitHub Action"
           git config --global user.email "action@github.com"
           git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling models" || echo "No changes to commit"
+          git add docs/"$(basename ${{ needs.setup.outputs.json_file }})"
+          git commit -m "Create $(basename ${{ needs.setup.outputs.json_file }}) after recompiling models" || echo "No changes to commit"
           git push origin gh-pages

.github/workflows/build-single-tinygrad-model.yaml

@@ -3,6 +3,14 @@ name: Build Single Tinygrad Model and Push
 on:
   workflow_dispatch:
     inputs:
+      runner_type:
+        description: 'Runner type'
+        required: false
+        default: '[self-hosted, james-mac]'
+        type: choice
+        options:
+          - ubuntu-latest
+          - '[self-hosted, james-mac]'
       build_model_ref:
         description: 'Branch to use for build-model workflow'
         required: false
@@ -50,7 +58,7 @@ on:
 
 jobs:
   build-single:
-    runs-on: ubuntu-latest
+    runs-on: ${{ github.event.inputs.runner_type }}
     env:
       RECOMPILED_DIR: recompiled${{ github.event.inputs.recompiled_dir }}
       JSON_FILE: docs/docs/driving_models_v${{ github.event.inputs.json_version }}.json

.github/workflows/sunnypilot-build-model.yaml

@@ -9,6 +9,22 @@ env:
   MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
 
 on:
+  workflow_call:
+    inputs:
+      upstream_branch:
+        description: 'Upstream branch to build from'
+        required: true
+        default: 'master'
+        type: string
+      custom_name:
+        description: 'Custom name for the model (no date, only name)'
+        required: false
+        type: string
+      is_20hz:
+        description: 'Is this a 20Hz model'
+        required: false
+        type: boolean
+        default: true
   workflow_dispatch:
     inputs:
       upstream_branch:
@@ -32,34 +48,53 @@ run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from
 jobs:
   get_model:
     runs-on: ubuntu-latest
+    env:
+      REF: ${{ inputs.upstream_branch }}
     outputs:
       model_date: ${{ steps.commit-date.outputs.model_date }}
     steps:
-      - uses: actions/checkout@v4
+      # Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
+      # and fallback to sunnypilot if the ref checkout fails.
+      - name: Checkout commaai/openpilot
+        id: checkout_upstream
+        continue-on-error: true
+        uses: actions/checkout@v4
         with:
-          repository: ${{ env.UPSTREAM_REPO }}
-          ref: ${{ github.event.inputs.upstream_branch }}
+          repository: commaai/openpilot
+          ref: ${{ inputs.upstream_branch }}
           submodules: recursive
+          path: openpilot
+
+      - name: Fallback to sunnypilot/sunnypilot
+        if: steps.checkout_upstream.outcome == 'failure'
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/sunnypilot
+          ref: ${{ inputs.upstream_branch }}
+          submodules: recursive
+          path: openpilot
       - name: Get commit date
         id: commit-date
         run: |
-          # Get the commit date in YYYY-MM-DD format
+          cd ${{ github.workspace }}/openpilot
           commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
           echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
-      - run: git lfs pull
+      - run: |
+          cd ${{ github.workspace }}/openpilot
+          git lfs pull
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v4
         with:
-          name: models
-          path: ${{ github.workspace }}/selfdrive/modeld/models/*.onnx
+          name: models-${{ env.REF }}
+          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
 
   build_model:
     runs-on: self-hosted
     needs: get_model
     env:
       MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
-
+      REF: ${{ inputs.upstream_branch }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -71,7 +106,7 @@ jobs:
         with:
           path: ${{env.SCONS_CACHE_DIR}}
           key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.) 
+          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
           #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
           restore-keys: |
             scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
@@ -114,7 +149,7 @@ jobs:
       - name: Download model artifacts
         uses: actions/download-artifact@v4
         with:
-          name: models
+          name: models-${{ env.REF }}
           path: ${{ github.workspace }}/selfdrive/modeld/models
 
       - name: Build Model

release/ci/model_generator.py

@@ -13,17 +13,32 @@ def create_short_name(full_name):
   words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)]
 
   if len(words) == 1:
-    # If there's only one word, return it as is, lowercased, truncated to 8 characters
-    truncated = words[0][:8]
-    return truncated.lower() if truncated.isupper() else truncated
+    return words[0][:8].upper()
 
   # Handle special case: Name + Version (e.g., "Word A1" -> "WordA1")
   if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]):
-    first_word = words[0].lower() if words[0].isupper() else words[0]
-    return (first_word + words[1])[:8]
+    return (words[0] + words[1])[:8].upper()
 
-  # Normal case: first letter and trailing numbers from each word
-  result = ''.join(word if word.isdigit() else word[0] + (re.search(r'\d+$', word) or [''])[0] for word in words)
+  result = ""
+  for word in words:
+    # Version or number patterns
+    if (re.match(r'^\d+[a-zA-Z]+$', word) or
+        re.match(r'^\d+[vVbB]\d+$', word) or
+        re.match(r'^[vVbB]\d+$', word) or
+        re.match(r'^\d{4}$', word)):
+      result += word.upper()
+    # All uppercase abbreviations (2-3 letters)
+    elif re.match(r'^[A-Z]{2,3}$', word):
+      result += word
+    # Letters+digits (for example tr15 rev2)
+    elif re.match(r'^[a-zA-Z]+[0-9]+$', word):
+      result += word[0].upper() + ''.join(re.findall(r'\d+', word))
+    elif word.isalpha():
+      result += word[0].upper()
+    elif word.isdigit():
+      result += word
+    else:
+      result += word[0].upper()
   return result[:8]
 
 
@@ -58,14 +73,14 @@ def generate_metadata(model_path: Path, output_dir: Path, short_name: str):
     "artifact": {
       "file_name": tinygrad_file.name,
       "download_uri": {
-        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/<FILLME>",
+        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/",
         "sha256": tinygrad_hash
       }
     },
     "metadata": {
       "file_name": metadata_file.name,
       "download_uri": {
-        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/<FILLME>",
+        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/",
         "sha256": metadata_hash
       }
     }
@@ -83,12 +98,12 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short
     "ref": upstream_branch,
     "environment": "development",
     "runner": "tinygrad",
-    "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
-    "models": models,
-    "overrides": {},
     "index": -1,
     "minimum_selector_version": "-1",
     "generation": "-1",
+    "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    "overrides": {},
+    "models": models,
   }
 
   # Write metadata to output_dir

sunnypilot/models/fetcher.py

@@ -115,7 +115,7 @@ class ModelCache:
 
 class ModelFetcher:
   """Handles fetching and caching of model data from remote source"""
-  MODEL_URL = "https://docs.sunnypilot.ai/driving_models_v4.json"
+  MODEL_URL = "https://docs.sunnypilot.ai/driving_models_v5.json"
 
   def __init__(self, params: Params):
     self.params = params

sunnypilot/models/helpers.py

@@ -19,8 +19,8 @@ from openpilot.system.hardware import PC
 from openpilot.system.hardware.hw import Paths
 from pathlib import Path
 
-CURRENT_SELECTOR_VERSION = 7
-REQUIRED_MIN_SELECTOR_VERSION = 5
+CURRENT_SELECTOR_VERSION = 1
+REQUIRED_MIN_SELECTOR_VERSION = 1
 
 USE_ONNX = os.getenv('USE_ONNX', PC)