Merge branch 'master-new' of https://github.com/sunnypilot/sunnypilot into master-new

2026-08-06 08:46:08 +08:00 · 2025-07-31 19:26:00 +02:00
parent 1cc9b4c770 713f8bfc21
commit e3699385dd
7 changed files with 439 additions and 237 deletions
@@ -1,21 +1,23 @@
-name: Build All Tinygrad Models and Push to GitLab
+name: Build and push all tinygrad models

 on:
  workflow_dispatch:
    inputs:
-      branch:
-        description: 'Branch to run workflow from'
-        required: false
-        default: 'master-new'
+      set_min_version:
+        description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
+        required: true
        type: string

 jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
+      json_version: ${{ steps.get-json.outputs.json_version }}
+      recompiled_dir: ${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
      json_file: ${{ steps.get-json.outputs.json_file }}
+      model_matrix: ${{ steps.set-matrix.outputs.model_matrix }}
    steps:
-      - name: Checkout docs repo
+      - name: Checkout docs repo (sunnypilot-docs, gh-pages)
        uses: actions/checkout@v4
        with:
          repository: sunnypilot/sunnypilot-docs
@@ -23,7 +25,7 @@ jobs:
          path: docs
          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

-      - name: Get next JSON version to use
+      - name: Get next JSON version to use (from GitHub docs repo)
        id: get-json
        run: |
          cd docs/docs
@@ -31,19 +33,131 @@ jobs:
          next=$((latest+1))
          json_file="driving_models_v${next}.json"
          cp "driving_models_v${latest}.json" "$json_file"
-          echo "json_file=$json_file" >> $GITHUB_OUTPUT
+          echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
+          echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
+          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV

-      - name: Upload context for next jobs
-        uses: actions/upload-artifact@v4
+      - name: Extract tinygrad models
+        id: set-matrix
+        working-directory: docs/docs
+        run: |
+          jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
+          echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+
+      - name: Set up SSH
+        uses: webfactory/ssh-agent@v0.9.0
        with:
-          name: context
-          path: docs
+          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+      - run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts

-  build-all:
+      - name: Clone GitLab docs repo and create new recompiled dir
+        id: create-recompiled-dir
+        env:
+          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+        run: |
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          cd gitlab_docs
+          git checkout main
+          git sparse-checkout set --no-cone models/
+          cd models
+          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
+          if [[ -z "$latest_dir" ]]; then
+            next_dir=1
+          else
+            next_dir=$((latest_dir+1))
+          fi
+          recompiled_dir="${next_dir}"
+          mkdir -p "recompiled${recompiled_dir}"
+          touch "recompiled${recompiled_dir}/.gitkeep"
+          cd ../..
+          echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
+
+      - name: Push empty recompiled dir to GitLab
+        run: |
+          cd gitlab_docs
+          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
+          git push origin main
+
+      - name: Push new JSON to GitHub docs repo
+        run: |
+          cd docs
+          git pull origin gh-pages
+          git add docs/"$(basename ${{ steps.get-json.outputs.json_file }})"
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git commit -m "Add new ${{ steps.get-json.outputs.json_file }} for build-all" || echo "No changes to commit"
+          git push origin gh-pages
+
+  get_and_build:
+    needs: [setup]
+    strategy:
+      matrix:
+        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
+      fail-fast: false
+    uses: ./.github/workflows/build-single-tinygrad-model.yaml
+    with:
+      upstream_branch: ${{ matrix.model.ref }}
+      custom_name: ${{ matrix.model.display_name }}
+      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
+      json_version: ${{ needs.setup.outputs.json_version }}
+    secrets: inherit
+
+  retry_failed_models:
+    needs: [setup, get_and_build]
    runs-on: ubuntu-latest
-    needs: setup
+    if: ${{ needs.setup.result != 'failure' && (failure() && !cancelled()) }}
+    outputs:
+      retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: model-*
+          path: output
+
+      - id: set-retry-matrix
+        run: |
+          echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
+          built=(); while IFS= read -r line; do built+=("$line"); done < <(
+            ls output | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
+          )
+          jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
+            'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
+          echo "retry_matrix=$(cat retry_matrix.json)" >> $GITHUB_OUTPUT
+
+  retry_get_and_build:
+    needs: [setup, get_and_build, retry_failed_models]
+    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
+    strategy:
+      matrix:
+        model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
+      fail-fast: false
+    uses: ./.github/workflows/build-single-tinygrad-model.yaml
+    with:
+      upstream_branch: ${{ matrix.model.ref }}
+      custom_name: ${{ matrix.model.display_name }}
+      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
+      json_version: ${{ needs.setup.outputs.json_version }}
+      artifact_suffix: -retry
+    secrets: inherit
+
+  publish_models:
+    name: Publish models sequentially
+    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
+    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      matrix:
+        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
    env:
-      JSON_FILE: docs/docs/${{ needs.setup.outputs.json_file }}
+      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
+      JSON_FILE: ${{ needs.setup.outputs.json_file }}
+      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
    steps:
      - name: Set up SSH
        uses: webfactory/ssh-agent@v0.9.0
@@ -62,138 +176,73 @@ jobs:
          echo "Cloning GitLab"
          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
          cd gitlab_docs
+          echo "checkout models/${RECOMPILED_DIR}"
+          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
          git checkout main
          cd ..

-      - name: Set next recompiled dir
-        id: set-recompiled
-        run: |
-          cd gitlab_docs/models
-          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          if [[ -z "$latest_dir" ]]; then
-            next_dir=1
-          else
-            next_dir=$((latest_dir+1))
-          fi
-          recompiled_dir="recompiled${next_dir}"
-          mkdir -p "$recompiled_dir"
-          echo "RECOMPILED_DIR=$recompiled_dir" >> $GITHUB_ENV
-
-      - name: Download context
-        uses: actions/download-artifact@v4
+      - name: Checkout docs repo
+        uses: actions/checkout@v4
        with:
-          name: context
-          path: .
+          repository: sunnypilot/sunnypilot-docs
+          ref: gh-pages
+          path: docs
+          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}

-      - name: Install dependencies
+      - name: Validate recompiled dir and JSON version
        run: |
-          sudo apt-get update
-          sudo apt-get install -y jq gh
-
-      - name: Build all tinygrad models
-        id: trigger-builds
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          set -e
-          > triggered_run_ids.txt
-          BRANCH="${{ github.event.inputs.branch }}"
-          jq -c '.bundles[] | select(.runner=="tinygrad")' "$JSON_FILE" | while read -r bundle; do
-            ref=$(echo "$bundle" | jq -r '.ref')
-            display_name=$(echo "$bundle" | jq -r '.display_name' | sed 's/ ([^)]*)//g')
-            is_20hz=$(echo "$bundle" | jq -r '.is_20hz')
-            echo "Triggering build for: $display_name ($ref) [20Hz: $is_20hz]"
-            START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-            gh workflow run sunnypilot-build-model.yaml \
-              --repo sunnypilot/sunnypilot \
-              --ref "$BRANCH" \
-              -f upstream_branch="$ref" \
-              -f custom_name="$display_name" \
-              -f is_20hz="$is_20hz"
-            for i in {1..24}; do
-              RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="$BRANCH" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
-              if [ -n "$RUN_ID" ]; then
-                break
-              fi
-              sleep 5
-            done
-            if [ -z "$RUN_ID" ]; then
-              echo "ould not find the triggered workflow run for $display_name ($ref)"
-              exit 1
-            fi
-            echo "$RUN_ID" >> triggered_run_ids.txt
-          done
-
-      - name: Wait for all model builds to finish
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          set -e
-          SUCCESS_RUNS=()
-          FAILED_RUNS=()
-          declare -A RUN_ID_TO_NAME
-
-          while read -r RUN_ID; do
-            ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name' || echo "unknown")
-            RUN_ID_TO_NAME["$RUN_ID"]="$ARTIFACT_NAME"
-          done < triggered_run_ids.txt
-
-          while read -r RUN_ID; do
-            echo "Watching run ID: $RUN_ID"
-            gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
-            CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
-            ARTIFACT_NAME="${RUN_ID_TO_NAME[$RUN_ID]}"
-            echo "Run $RUN_ID ($ARTIFACT_NAME) concluded with: $CONCLUSION"
-            if [[ "$CONCLUSION" == "success" ]]; then
-              SUCCESS_RUNS+=("$RUN_ID")
-            else
-              FAILED_RUNS+=("$RUN_ID")
-            fi
-          done < triggered_run_ids.txt
-
-          if [[ ${#SUCCESS_RUNS[@]} -eq 0 ]]; then
-            echo "All model builds failed. Aborting."
+          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
+            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
+            exit 1
+          fi
+          if [ ! -f "$JSON_FILE" ]; then
+            echo "JSON file $JSON_FILE does not exist!"
            exit 1
          fi

-          if [[ ${#FAILED_RUNS[@]} -gt 0 ]]; then
-            echo "WARNING: The following model builds failed:"
-            for RUN_ID in "${FAILED_RUNS[@]}"; do
-              echo "- $RUN_ID (${RUN_ID_TO_NAME[$RUN_ID]})"
-            done
-            echo "You may want to rerun these models manually."
-          fi
+      - name: Download artifact name file
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
+          path: artifact_name

-      - name: Download and extract all model artifacts
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Read artifact name
+        id: read-artifact-name
        run: |
-          ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
-          SUCCESS_RUNS=()
-          while read -r RUN_ID; do
-            CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
-            if [[ "$CONCLUSION" == "success" ]]; then
-              SUCCESS_RUNS+=("$RUN_ID")
-            fi
-          done < triggered_run_ids.txt
+          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT

-          for RUN_ID in "${SUCCESS_RUNS[@]}"; do
-            ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
-            echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
-            mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
-            echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
-            gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
-            echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
-            ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
-            if [ -n "$ZIP_PATH" ]; then
-              echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
-              unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
-              rm -f "$ZIP_PATH"
-              echo "Unzipped and removed $ZIP_PATH"
+      - name: Download model artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
+          path: output
+
+      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
+        run: |
+          find output -type f -name '*.onnx' -delete
+          find output -type f -name 'big_*.pkl' -delete
+          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
+
+      - name: Copy model artifacts to gitlab
+        env:
+          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+        run: |
+          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          mkdir -p "$ARTIFACT_DIR"
+          for path in output/*; do
+            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
+              continue
+            fi
+            name="$(basename "$path")"
+            if [ -d "$path" ]; then
+              mkdir -p "$ARTIFACT_DIR/$name"
+              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
+              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
            else
-              echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME (This is NOT an error)."
+              cp "$path" "$ARTIFACT_DIR/"
+              echo "Copied file $name -> $ARTIFACT_DIR/"
            fi
-            echo "Done processing $ARTIFACT_NAME"
          done

      - name: Push recompiled dir to GitLab
@@ -202,25 +251,35 @@ jobs:
        run: |
          cd gitlab_docs
          git checkout main
-          mkdir -p models/"$(basename $RECOMPILED_DIR)"
-          git add models/"$(basename $RECOMPILED_DIR)"
+          git pull origin main
+          for d in models/"$RECOMPILED_DIR"/*/; do
+            git sparse-checkout add "$d"
+          done
+          git add models/"$RECOMPILED_DIR"
          git config --global user.name "GitHub Action"
          git config --global user.email "action@github.com"
-          git commit -m "Add $(basename $RECOMPILED_DIR) from build-all-tinygrad-models"
+          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
          git push origin main
+      - run: |
+          cd docs
+          git pull origin gh-pages

-      - name: Run json_parser.py to update JSON
+      - name: update json
        run: |
-          python3 docs/json_parser.py \
+          ARGS=""
+          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
+          ARGS="$ARGS --sort-by-date"
+          eval python3 docs/json_parser.py \
            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR"
+            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
+            $ARGS

-      - name: Push updated JSON to GitHub docs repo
+      - name: Push updated json to GitHub
        run: |
          cd docs
          git config --global user.name "GitHub Action"
          git config --global user.email "action@github.com"
          git checkout gh-pages
          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling models" || echo "No changes to commit"
+          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
          git push origin gh-pages
@@ -1,13 +1,36 @@
 name: Build Single Tinygrad Model and Push

 on:
+  workflow_call:
+    inputs:
+      upstream_branch:
+        description: 'Upstream commit to build from'
+        required: true
+        type: string
+      custom_name:
+        description: 'Custom name for the model (no date, only name)'
+        required: false
+        type: string
+      recompiled_dir:
+        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
+        required: true
+        type: string
+      json_version:
+        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
+        required: true
+        type: string
+      artifact_suffix:
+        description: 'Suffix for artifact name'
+        required: false
+        type: string
+        default: ''
+      bypass_push:
+        description: 'Bypass pushing to GitLab for build-all'
+        required: false
+        default: true
+        type: boolean
  workflow_dispatch:
    inputs:
-      build_model_ref:
-        description: 'Branch to use for build-model workflow'
-        required: false
-        default: 'master-new'
-        type: string
      upstream_branch:
        description: 'Upstream commit to build from'
        required: true
@@ -26,10 +49,12 @@ on:
        type: number
      model_folder:
        description: 'Model folder'
-        required: true
        type: choice
+        default: 'None'
        options:
+          - None
          - Simple Plan Models
+          - Space Lab Models
          - TR Models
          - DTR Models
          - Custom Merge Models
@@ -47,13 +72,27 @@ on:
        description: 'Minimum selector version'
        required: false
        type: number
+env:
+  RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
+  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json

 jobs:
-  build-single:
+  build_model:
+    uses: ./.github/workflows/sunnypilot-build-model.yaml
+    with:
+      upstream_branch: ${{ inputs.upstream_branch }}
+      custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
+      is_20hz: true
+      artifact_suffix: ${{ inputs.artifact_suffix }}
+    secrets: inherit
+
+  publish_model:
+    if: ${{ !inputs.bypass_push && !cancelled() }}
+    concurrency:
+      group: gitlab-push-${{ inputs.recompiled_dir }}
+      cancel-in-progress: false
+    needs: build_model
    runs-on: ubuntu-latest
-    env:
-      RECOMPILED_DIR: recompiled${{ github.event.inputs.recompiled_dir }}
-      JSON_FILE: docs/docs/driving_models_v${{ github.event.inputs.json_version }}.json
    steps:
      - name: Set up SSH
        uses: webfactory/ssh-agent@v0.9.0
@@ -96,66 +135,49 @@ jobs:
            exit 1
          fi

-      - name: Install dependencies
+      - name: Download artifact name file
+        uses: actions/download-artifact@v4
+        with:
+          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
+          path: artifact_name
+
+      - name: Read artifact name
+        id: read-artifact-name
        run: |
-          sudo apt-get update
-          sudo apt-get install -y jq gh
-
-      - name: Build model
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-          gh workflow run sunnypilot-build-model.yaml \
-            --repo sunnypilot/sunnypilot \
-            --ref "${{ github.event.inputs.build_model_ref }}" \
-            -f upstream_branch="${{ github.event.inputs.upstream_branch }}" \
-            -f custom_name="${{ github.event.inputs.custom_name }}"
-
-          for i in {1..24}; do
-            RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="${{ github.event.inputs.build_model_ref }}" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
-            if [ -n "$RUN_ID" ]; then
-              break
-            fi
-            sleep 5
-          done
-
-          if [ -z "$RUN_ID" ]; then
-            echo "Could not find the triggered workflow run."
-            exit 1
-          fi
-
-          echo "Watching run ID: $RUN_ID"
-          gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
-          CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
-          echo "Run concluded with: $CONCLUSION"
-          if [[ "$CONCLUSION" != "success" ]]; then
-            echo "Workflow run failed with conclusion: $CONCLUSION"
-            exit 1
-          fi
+          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT

      - name: Download and extract model artifact
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
+          path: output
+
+      - name: Remove unwanted files
        run: |
-          ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
-          RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="${{ github.event.inputs.build_model_ref }}" --limit=1 --json databaseId --jq '.[0].databaseId')
-          ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
-          echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
-          mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
-          echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
-          gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
-          echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
-          ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
-          if [ -n "$ZIP_PATH" ]; then
-            echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
-            unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
-            rm -f "$ZIP_PATH"
-            echo "Unzipped and removed $ZIP_PATH"
-          else
-            echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME"
-          fi
-          echo "Done processing $ARTIFACT_NAME"
+          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
+          find output -type f -name 'dmonitoring_model.onnx' -delete
+
+      - name: Copy model artifact(s) to GitLab recompiled dir
+        env:
+          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
+        run: |
+          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+          mkdir -p "$ARTIFACT_DIR"
+          for path in output/*; do
+            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
+              continue
+            fi
+            name="$(basename "$path")"
+            if [ -d "$path" ]; then
+              mkdir -p "$ARTIFACT_DIR/$name"
+              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
+              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
+            else
+              cp "$path" "$ARTIFACT_DIR/"
+              echo "Copied file $name -> $ARTIFACT_DIR/"
+            fi
+          done

      - name: Push recompiled dir to GitLab
        env:
@@ -163,28 +185,36 @@ jobs:
        run: |
          cd gitlab_docs
          git checkout main
+          git pull origin main
          for d in models/"$RECOMPILED_DIR"/*/; do
            git sparse-checkout add "$d"
          done
          git add models/"$RECOMPILED_DIR"
          git config --global user.name "GitHub Action"
          git config --global user.email "action@github.com"
-          git commit -m "Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
+          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
          git push origin main

+      - run: |
+          cd docs
+          git pull origin gh-pages
+
      - name: Run json_parser.py to update JSON
        run: |
-          FOLDER="${{ github.event.inputs.model_folder }}"
+          FOLDER="${{ inputs.model_folder }}"
          if [ "$FOLDER" = "Other" ]; then
-            FOLDER="${{ github.event.inputs.custom_model_folder }}"
+            FOLDER="${{ inputs.custom_model_folder }}"
          fi
          ARGS=""
-          [ -n "$FOLDER" ] && ARGS="$ARGS --model-folder \"$FOLDER\""
-          [ -n "${{ github.event.inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ github.event.inputs.generation }}\""
-          [ -n "${{ github.event.inputs.version }}" ] && ARGS="$ARGS --version \"${{ github.event.inputs.version }}\""
+          if [ "$FOLDER" != "None" ] && [ -n "$FOLDER" ]; then
+            ARGS="$ARGS --model-folder \"$FOLDER\""
+          fi
+          [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
+          [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
          eval python3 docs/json_parser.py \
            --json-path "$JSON_FILE" \
            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
+            --sort-by-date \
            $ARGS

      - name: Push updated JSON to GitHub docs repo
@@ -9,6 +9,27 @@ env:
  MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models

 on:
+  workflow_call:
+    inputs:
+      upstream_branch:
+        description: 'Upstream branch to build from'
+        required: true
+        default: 'master'
+        type: string
+      custom_name:
+        description: 'Custom name for the model (no date, only name)'
+        required: false
+        type: string
+      is_20hz:
+        description: 'Is this a 20Hz model'
+        required: false
+        type: boolean
+        default: true
+      artifact_suffix:
+        description: 'Suffix for artifact name'
+        required: false
+        type: string
+        default: ''
  workflow_dispatch:
    inputs:
      upstream_branch:
@@ -32,34 +53,53 @@ run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from
 jobs:
  get_model:
    runs-on: ubuntu-latest
+    env:
+      REF: ${{ inputs.upstream_branch }}
    outputs:
      model_date: ${{ steps.commit-date.outputs.model_date }}
    steps:
-      - uses: actions/checkout@v4
+      # Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
+      # and fallback to sunnypilot if the ref checkout fails.
+      - name: Checkout commaai/openpilot
+        id: checkout_upstream
+        continue-on-error: true
+        uses: actions/checkout@v4
        with:
-          repository: ${{ env.UPSTREAM_REPO }}
-          ref: ${{ github.event.inputs.upstream_branch }}
+          repository: commaai/openpilot
+          ref: ${{ inputs.upstream_branch }}
          submodules: recursive
+          path: openpilot
+
+      - name: Fallback to sunnypilot/sunnypilot
+        if: steps.checkout_upstream.outcome == 'failure'
+        uses: actions/checkout@v4
+        with:
+          repository: sunnypilot/sunnypilot
+          ref: ${{ inputs.upstream_branch }}
+          submodules: recursive
+          path: openpilot
      - name: Get commit date
        id: commit-date
        run: |
-          # Get the commit date in YYYY-MM-DD format
+          cd ${{ github.workspace }}/openpilot
          commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
          echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
          cat $GITHUB_OUTPUT
-      - run: git lfs pull
+      - run: |
+          cd ${{ github.workspace }}/openpilot
+          git lfs pull
      - name: 'Upload Artifact'
        uses: actions/upload-artifact@v4
        with:
-          name: models
-          path: ${{ github.workspace }}/selfdrive/modeld/models/*.onnx
+          name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
+          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx

  build_model:
    runs-on: [self-hosted, tici]
    needs: get_model
    env:
      MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
-
+      REF: ${{ inputs.upstream_branch }}
    steps:
      - uses: actions/checkout@v4
        with:
@@ -114,7 +154,7 @@ jobs:
      - name: Download model artifacts
        uses: actions/download-artifact@v4
        with:
-          name: models
+          name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
          path: ${{ github.workspace }}/selfdrive/modeld/models

      - name: Build Model
@@ -157,12 +197,22 @@ jobs:
            --upstream-branch "${{ inputs.upstream_branch }}" \
            ${{ inputs.is_20hz && '--is-20hz' || '' }}

+      - name: Write artifact name to file
+        run: echo "model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}" > ${{ env.OUTPUT_DIR }}/artifact_name.txt
+
      - name: Upload Build Artifacts
+        id: upload-artifact
        uses: actions/upload-artifact@v4
        with:
-          name: model-${{ env.MODEL_NAME }}-${{ github.run_number }}
+          name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
          path: ${{ env.OUTPUT_DIR }}

+      - name: Upload artifact name file
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
+          path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
+
      - name: Re-enable powersave
        if: always()
        run: |
@@ -25,6 +25,27 @@ struct ModularAssistiveDrivingSystem {
  }
 }

+# Same struct as Log.RadarState.LeadData
+struct LeadData {
+  dRel @0 :Float32;
+  yRel @1 :Float32;
+  vRel @2 :Float32;
+  aRel @3 :Float32;
+  vLead @4 :Float32;
+  dPath @6 :Float32;
+  vLat @7 :Float32;
+  vLeadK @8 :Float32;
+  aLeadK @9 :Float32;
+  fcw @10 :Bool;
+  status @11 :Bool;
+  aLeadTau @12 :Float32;
+  modelProb @13 :Float32;
+  radar @14 :Bool;
+  radarTrackId @15 :Int32 = -1;
+
+  aLeadDEPRECATED @5 :Float32;
+}
+
 struct SelfdriveStateSP @0x81c2f05a394cf4af {
  mads @0 :ModularAssistiveDrivingSystem;
 }
@@ -174,6 +195,8 @@ struct CarParamsSP @0x80ae746ee2596b11 {
 struct CarControlSP @0xa5cd762cd951a455 {
  mads @0 :ModularAssistiveDrivingSystem;
  params @1 :List(Param);
+  leadOne @2 :LeadData;
+  leadTwo @3 :LeadData;

  struct Param {
    key @0 :Text;
@@ -13,17 +13,32 @@ def create_short_name(full_name):
  words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)]

  if len(words) == 1:
-    # If there's only one word, return it as is, lowercased, truncated to 8 characters
-    truncated = words[0][:8]
-    return truncated.lower() if truncated.isupper() else truncated
+    return words[0][:8].upper()

  # Handle special case: Name + Version (e.g., "Word A1" -> "WordA1")
  if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]):
-    first_word = words[0].lower() if words[0].isupper() else words[0]
-    return (first_word + words[1])[:8]
+    return (words[0] + words[1])[:8].upper()

-  # Normal case: first letter and trailing numbers from each word
-  result = ''.join(word if word.isdigit() else word[0] + (re.search(r'\d+$', word) or [''])[0] for word in words)
+  result = ""
+  for word in words:
+    # Version or number patterns
+    if (re.match(r'^\d+[a-zA-Z]+$', word) or
+        re.match(r'^\d+[vVbB]\d+$', word) or
+        re.match(r'^[vVbB]\d+$', word) or
+        re.match(r'^\d{4}$', word)):
+      result += word.upper()
+    # All uppercase abbreviations (2-3 letters)
+    elif re.match(r'^[A-Z]{2,3}$', word):
+      result += word
+    # Letters+digits (for example tr15 rev2)
+    elif re.match(r'^[a-zA-Z]+[0-9]+$', word):
+      result += word[0].upper() + ''.join(re.findall(r'\d+', word))
+    elif word.isalpha():
+      result += word[0].upper()
+    elif word.isdigit():
+      result += word
+    else:
+      result += word[0].upper()
  return result[:8]


@@ -58,14 +73,14 @@ def generate_metadata(model_path: Path, output_dir: Path, short_name: str):
    "artifact": {
      "file_name": tinygrad_file.name,
      "download_uri": {
-        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/<FILLME>",
+        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/",
        "sha256": tinygrad_hash
      }
    },
    "metadata": {
      "file_name": metadata_file.name,
      "download_uri": {
-        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/<FILLME>",
+        "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/",
        "sha256": metadata_hash
      }
    }
@@ -83,12 +98,12 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short
    "ref": upstream_branch,
    "environment": "development",
    "runner": "tinygrad",
-    "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
-    "models": models,
-    "overrides": {},
    "index": -1,
    "minimum_selector_version": "-1",
    "generation": "-1",
+    "build_time": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    "overrides": {},
+    "models": models,
  }

  # Write metadata to output_dir
@@ -58,5 +58,7 @@ def convert_carControlSP(struct: capnp.lib.capnp._DynamicStructReader) -> struct

  struct_dataclass.mads = structs.ModularAssistiveDrivingSystem(**remove_deprecated(struct_dict.get('mads', {})))
  struct_dataclass.params = [structs.CarControlSP.Param(**remove_deprecated(p)) for p in struct_dict.get('params', [])]
+  struct_dataclass.leadOne = structs.LeadData(**remove_deprecated(struct_dict.get('leadOne', {})))
+  struct_dataclass.leadTwo = structs.LeadData(**remove_deprecated(struct_dict.get('leadTwo', {})))

  return struct_dataclass
@@ -5,7 +5,7 @@ This file is part of sunnypilot and is licensed under the MIT License.
 See the LICENSE.md file in the root directory for more details.
 """
 import cereal.messaging as messaging
-from cereal import custom
+from cereal import log, custom

 from opendbc.car import structs
 from openpilot.common.params import Params
@@ -26,7 +26,7 @@ class ControlsExt:
    self.CP_SP = messaging.log_from_bytes(params.get("CarParamsSP", block=True), custom.CarParamsSP)
    cloudlog.info("controlsd_ext got CarParamsSP")

-    self.sm_services_ext = ['selfdriveStateSP']
+    self.sm_services_ext = ['radarState', 'selfdriveStateSP']
    self.pm_services_ext = ['carControlSP']

  def get_params_sp(self) -> None:
@@ -44,9 +44,32 @@ class ControlsExt:
    # MADS not available, use stock state to engage
    return bool(sm['selfdriveState'].active)

+  @staticmethod
+  def get_lead_data(ld: log.RadarState.LeadData) -> dict:
+    return {
+      "dRel": ld.dRel,
+      "yRel": ld.yRel,
+      "vRel": ld.vRel,
+      "aRel": ld.aRel,
+      "vLead": ld.vLead,
+      "dPath": ld.dPath,
+      "vLat": ld.vLat,
+      "vLeadK": ld.vLeadK,
+      "aLeadK": ld.aLeadK,
+      "fcw": ld.fcw,
+      "status": ld.status,
+      "aLeadTau": ld.aLeadTau,
+      "modelProb": ld.modelProb,
+      "radar": ld.radar,
+      "radarTrackId": ld.radarTrackId,
+    }
+
  def state_control_ext(self, sm: messaging.SubMaster) -> custom.CarControlSP:
    CC_SP = custom.CarControlSP.new_message()

+    CC_SP.leadOne = self.get_lead_data(sm['radarState'].leadOne)
+    CC_SP.leadTwo = self.get_lead_data(sm['radarState'].leadTwo)
+
    # MADS state
    CC_SP.mads = sm['selfdriveStateSP'].mads