mawei/sunnypilot
1
0
Fork 0
mirror of https://github.com/sunnypilot/sunnypilot.git synced 2026-08-31 21:43:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

ci: fix DM model build, separate HF defaults paths, nuke build races (#1956)

Browse Source 
* ci: fix DM model build, separate HF defaults paths, nuke build races

* more split!

* name

* ci: download driving and DM model chunks into chestnut prebuilt output
This commit is contained in:
Jason Wen
2026-08-24 20:02:48 -04:00
committed by GitHub
parent 8e16c9babb
commit 6cc5f3aad8
2 changed files with 177 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-default-models.yaml
+32 -33
View File
@@ -10,13 +10,18 @@ on:
options:
- small
- big
- dm
workflow_call:
inputs:
target:
description: 'Model target to build (small or big)'
description: 'Model target to build (small, big, or dm)'
required: true
type: string
concurrency:
group: build-default-models-${{ inputs.target }}
cancel-in-progress: false
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
@@ -28,10 +33,7 @@ jobs:
onnx_ref: ${{ steps.resolve.outputs.onnx_ref }}
onnx_path: ${{ steps.resolve.outputs.onnx_path }}
hf_defaults_path: ${{ steps.resolve.outputs.hf_defaults_path }}
target_hardware: ${{ steps.resolve.outputs.target_hardware }}
tinygrad_ref: ${{ steps.resolve.outputs.tinygrad_ref }}
dm_onnx_ref: ${{ steps.resolve.outputs.dm_onnx_ref }}
dm_onnx_date: ${{ steps.resolve.outputs.dm_onnx_date }}
steps:
- uses: actions/checkout@v4
with:
@@ -45,12 +47,14 @@ jobs:
NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
ONNX_PATH="openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
HF_DEFAULTS_PATH="models/defaults/big"
TARGET_HW="usbgpu"
elif [ "${{ inputs.target }}" = "dm" ]; then
ONNX_PATH="openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
HF_DEFAULTS_PATH="models/defaults/dm"
NAME="dmonitoring_model ($(git log -1 --format=%cd --date=format:'%B %d, %Y' -- "$ONNX_PATH"))"
else
NAME=$(python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_MODEL; print(DEFAULT_MODEL)")
ONNX_PATH="openpilot/selfdrive/modeld/models/driving_supercombo.onnx"
HF_DEFAULTS_PATH="models/defaults/small"
TARGET_HW="qcom"
fi
ONNX_REF=$(git log -1 --format='%H' -- "$ONNX_PATH")
@@ -60,21 +64,11 @@ jobs:
exit 1
fi
DM_ONNX_REF=""
DM_ONNX_DATE=""
if [ "${{ inputs.target }}" = "small" ]; then
DM_ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
DM_ONNX_DATE=$(git log -1 --format=%cd --date=format:'%B %d, %Y' -- openpilot/selfdrive/modeld/models/dmonitoring_model.onnx)
fi
echo "model_name=${NAME}" >> $GITHUB_OUTPUT
echo "onnx_ref=${ONNX_REF}" >> $GITHUB_OUTPUT
echo "onnx_path=${ONNX_PATH}" >> $GITHUB_OUTPUT
echo "hf_defaults_path=${HF_DEFAULTS_PATH}" >> $GITHUB_OUTPUT
echo "target_hardware=${TARGET_HW}" >> $GITHUB_OUTPUT
echo "tinygrad_ref=${TINYGRAD_REF}" >> $GITHUB_OUTPUT
echo "dm_onnx_ref=${DM_ONNX_REF}" >> $GITHUB_OUTPUT
echo "dm_onnx_date=${DM_ONNX_DATE}" >> $GITHUB_OUTPUT
build_small_model:
needs: resolve
@@ -304,43 +298,45 @@ jobs:
${{
!cancelled() &&
(inputs.target == 'big' && needs.build_big_model.result == 'success' ||
inputs.target == 'small' && needs.build_small_model.result == 'success') &&
(inputs.target != 'small' || needs.build_dm_model.result == 'success')
inputs.target == 'small' && needs.build_small_model.result == 'success' ||
inputs.target == 'dm' && needs.build_dm_model.result == 'success')
}}
runs-on: ubuntu-24.04
permissions:
id-token: write
contents: write
env:
DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
steps:
- uses: actions/checkout@v4
- name: Pull ONNX via LFS
run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}${{ inputs.target == 'small' && ',openpilot/selfdrive/modeld/models/dmonitoring_model.onnx' || '' }}"
run: git lfs pull -I "${{ needs.resolve.outputs.onnx_path }}"
- name: Install huggingface_hub
run: pip install --upgrade "huggingface_hub>=0.22.0"
- name: Download driving artifact name
- name: Download artifact name
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
uses: actions/download-artifact@v4
with:
name: artifact-name-${{ needs.resolve.outputs.model_name }}
path: artifact_name
- name: Read driving artifact name
- name: Read artifact name
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
id: artifact
run: |
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
- name: Download driving model artifact
- name: Download model artifact
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
uses: actions/download-artifact@v4
with:
name: ${{ steps.artifact.outputs.artifact_name }}
path: output
- name: Upload driving model to HF
- name: Upload model to HF
if: ${{ inputs.target == 'small' || inputs.target == 'big' }}
env:
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
@@ -359,14 +355,14 @@ jobs:
--run-number "${{ github.run_number }}"
- name: Download DM artifact
if: ${{ inputs.target == 'small' }}
if: ${{ inputs.target == 'dm' }}
uses: actions/download-artifact@v4
with:
name: dm-model-${{ github.run_number }}
path: dm_output
- name: Generate DM metadata and upload to HF
if: ${{ inputs.target == 'small' }}
if: ${{ inputs.target == 'dm' }}
env:
HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
run: |
@@ -399,8 +395,8 @@ jobs:
metadata = {
'bundles': [{
'short_name': 'DMMODEL',
'display_name': 'dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})',
'ref': '${{ needs.resolve.outputs.dm_onnx_ref }}',
'display_name': '${{ needs.resolve.outputs.model_name }}',
'ref': '${{ needs.resolve.outputs.onnx_ref }}',
'runner': 'tinygrad',
'build_time': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
'models': [{
@@ -423,15 +419,15 @@ jobs:
--hf-defaults-path "${{ needs.resolve.outputs.hf_defaults_path }}" \
--artifact-name "dm-model-${{ github.run_number }}" \
--model-dir dm_output \
--onnx-path "${{ env.DM_ONNX }}" \
--onnx-ref "${{ needs.resolve.outputs.dm_onnx_ref }}" \
--model-name "dmonitoring_model (${{ needs.resolve.outputs.dm_onnx_date }})" \
--onnx-path "${{ needs.resolve.outputs.onnx_path }}" \
--onnx-ref "${{ needs.resolve.outputs.onnx_ref }}" \
--model-name "${{ needs.resolve.outputs.model_name }}" \
--tinygrad-ref "${{ needs.resolve.outputs.tinygrad_ref }}" \
--run-number "${{ github.run_number }}"
build_dm_model:
needs: resolve
if: ${{ inputs.target == 'small' }}
if: ${{ inputs.target == 'dm' }}
runs-on: [self-hosted, tici]
env:
DM_ONNX: openpilot/selfdrive/modeld/models/dmonitoring_model.onnx
@@ -441,6 +437,9 @@ jobs:
with:
submodules: recursive
- name: Pull DM ONNX via LFS
run: git lfs pull -I "${{ env.DM_ONNX }}"
- name: Set environment variables
run: |
source /etc/profile
.github/workflows/sunnypilot-build-prebuilt.yaml
+145 -74
View File
@@ -240,34 +240,24 @@ jobs:
if check_hash; then
echo "HF defaults match repo ONNX"
else
echo "No matching model on HF — triggering build"
TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big
echo "Waiting for build to start..."
sleep 120
RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
echo "::error::Failed to find build-default-models run"
exit 1
fi
echo "Waiting for run $RUN_ID..."
gh run watch "$RUN_ID"
CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
if [ "$CONCLUSION" != "success" ]; then
echo "::error::build-default-models failed: $CONCLUSION"
exit 1
fi
if ! check_hash; then
echo "::error::HF defaults still don't match after build"
exit 1
fi
exit 0
fi
echo "No matching model on HF — dispatching build"
gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=big
echo "Polling HF for big model availability..."
for i in $(seq 1 90); do
sleep 30
if check_hash; then
echo "Big model available on HF after $((i * 30))s"
exit 0
fi
echo "Poll $i/90: not yet available"
done
echo "::error::Big model not available on HF after 45 minutes"
exit 1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -277,12 +267,11 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
prepare_small_models:
prepare_small_model:
needs: [ prepare_strategy ]
runs-on: ubuntu-24.04
outputs:
driving_onnx_sha256: ${{ steps.resolve.outputs.driving_onnx_sha256 }}
dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }}
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
HF_DEFAULTS_PATH: models/defaults/small
@@ -292,18 +281,15 @@ jobs:
ref: ${{ github.head_ref || github.ref_name }}
submodules: recursive
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/driving_supercombo.onnx,openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/driving_supercombo.onnx"
- name: Check HF defaults and build if needed
id: resolve
run: |
DRIVING_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/driving_supercombo.onnx" | cut -d' ' -f1)
DM_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" | cut -d' ' -f1)
TINYGRAD_REF=$(PYTHONPATH=${{ github.workspace }} python3 openpilot/sunnypilot/models/tinygrad_ref.py)
echo "driving_onnx_sha256=$DRIVING_HASH" >> $GITHUB_OUTPUT
echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT
echo "Driving ONNX hash: $DRIVING_HASH"
echo "DM ONNX hash: $DM_HASH"
echo "tinygrad ref: $TINYGRAD_REF"
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
@@ -314,40 +300,92 @@ jobs:
[ "$TINYGRAD_MATCH" = "true" ] || return 1
DRIVING=$(echo "$DEFAULTS" | jq --arg hash "$DRIVING_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
[ -n "$DRIVING" ] && [ "$DRIVING" != "null" ] || return 1
}
if check_defaults; then
echo "HF defaults match repo ONNX hash and tinygrad ref"
exit 0
fi
echo "No matching model on HF — dispatching build"
gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=small
echo "Polling HF for model availability..."
for i in $(seq 1 60); do
sleep 30
if check_defaults; then
echo "Model available on HF after $((i * 30))s"
exit 0
fi
echo "Poll $i/60: not yet available"
done
echo "::error::Small driving model not available on HF after 30 minutes"
exit 1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Cancel run on failure
if: failure()
run: gh run cancel ${{ github.run_id }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
prepare_dm_model:
needs: [ prepare_strategy ]
runs-on: ubuntu-24.04
outputs:
dm_onnx_sha256: ${{ steps.resolve.outputs.dm_onnx_sha256 }}
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
HF_DEFAULTS_PATH: models/defaults/dm
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref || github.ref_name }}
submodules: recursive
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx"
- name: Check HF defaults and build if needed
id: resolve
run: |
DM_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx" | cut -d' ' -f1)
TINYGRAD_REF=$(PYTHONPATH=${{ github.workspace }} python3 openpilot/sunnypilot/models/tinygrad_ref.py)
echo "dm_onnx_sha256=$DM_HASH" >> $GITHUB_OUTPUT
echo "DM ONNX hash: $DM_HASH"
echo "tinygrad ref: $TINYGRAD_REF"
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
check_defaults() {
DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
TINYGRAD_MATCH=$(echo "$DEFAULTS" | jq -r --arg ref "$TINYGRAD_REF" '.tinygrad_ref == $ref' 2>/dev/null)
[ "$TINYGRAD_MATCH" = "true" ] || return 1
DM=$(echo "$DEFAULTS" | jq --arg hash "$DM_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
[ -n "$DM" ] && [ "$DM" != "null" ] || return 1
}
if check_defaults; then
echo "HF defaults match repo ONNX hashes and tinygrad ref"
else
echo "No matching models on HF — triggering build"
TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=small
echo "Waiting for build to start..."
sleep 120
RUN_ID=$(gh run list --workflow=build-default-models.yaml --branch="${{ github.head_ref || github.ref_name }}" --created=">$TRIGGER_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
echo "::error::Failed to find build-default-models run"
exit 1
fi
echo "Waiting for run $RUN_ID..."
gh run watch "$RUN_ID"
CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
if [ "$CONCLUSION" != "success" ]; then
echo "::error::build-default-models failed: $CONCLUSION"
exit 1
fi
if ! check_defaults; then
echo "::error::HF defaults still don't match after build"
exit 1
fi
echo "HF defaults match DM ONNX hash and tinygrad ref"
exit 0
fi
echo "No matching DM model on HF — dispatching build"
gh workflow run build-default-models.yaml --ref "${{ github.head_ref || github.ref_name }}" -f target=dm
echo "Polling HF for DM model availability..."
for i in $(seq 1 60); do
sleep 30
if check_defaults; then
echo "DM model available on HF after $((i * 30))s"
exit 0
fi
echo "Poll $i/60: not yet available"
done
echo "::error::DM model not available on HF after 30 minutes"
exit 1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -365,11 +403,12 @@ jobs:
always() && !cancelled() &&
needs.build.result == 'success' &&
needs.prepare_strategy.result == 'success' &&
needs.prepare_small_models.result == 'success' &&
needs.prepare_small_model.result == 'success' &&
needs.prepare_dm_model.result == 'success' &&
(!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
(needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
}}
needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_models ]
needs: [ build, prepare_strategy, prepare_chestnut, prepare_small_model, prepare_dm_model ]
runs-on: ubuntu-24.04
environment: ${{ needs.prepare_strategy.outputs.environment }}
steps:
@@ -385,19 +424,19 @@ jobs:
mkdir -p ${{ env.OUTPUT_DIR }}
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
- name: Download small model chunks from HF
- name: Download default model chunks from HF
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
HF_DEFAULTS_PATH: models/defaults/small
run: |
set -o pipefail
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
DEFAULTS=$(curl -fsSL "$JSON_URL")
MODELS_DIR="${{ env.OUTPUT_DIR }}/openpilot/selfdrive/modeld/models"
download_model_chunks() {
local ONNX_HASH="$1"
local CANONICAL="$2"
local DEFAULTS_PATH="$1"
local ONNX_HASH="$2"
local CANONICAL="$3"
local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${DEFAULTS_PATH}/default_models.json"
local DEFAULTS=$(curl -fsSL "$JSON_URL")
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
@@ -417,8 +456,8 @@ jobs:
echo "$NUM_CHUNKS" > "${MODELS_DIR}/${CANONICAL}.chunkmanifest"
}
download_model_chunks "${{ needs.prepare_small_models.outputs.driving_onnx_sha256 }}" "driving_tinygrad.pkl"
download_model_chunks "${{ needs.prepare_small_models.outputs.dm_onnx_sha256 }}" "dmonitoring_model_tinygrad.pkl"
download_model_chunks "models/defaults/small" "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}" "driving_tinygrad.pkl"
download_model_chunks "models/defaults/dm" "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}" "dmonitoring_model_tinygrad.pkl"
- name: Prepare chestnut output
if: ${{ needs.prepare_chestnut.result == 'success' }}
@@ -453,10 +492,41 @@ jobs:
echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
- name: Inject big model into chestnut
- name: Inject models into chestnut
if: ${{ needs.prepare_chestnut.result == 'success' }}
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
run: |
cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/"
CHESTNUT_MODELS="${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models"
cp big_model_chunks/* "$CHESTNUT_MODELS/"
download_model_chunks() {
local DEFAULTS_PATH="$1"
local ONNX_HASH="$2"
local CANONICAL="$3"
local JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${DEFAULTS_PATH}/default_models.json"
local DEFAULTS=$(curl -fsSL "$JSON_URL")
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+' || true)
if [ -z "$CHUNK_IDX" ]; then
echo "::error::Failed to parse chunk index from: $CHUNK_NAME"
exit 1
fi
CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
curl -fsSL -o "${CHESTNUT_MODELS}/${CANONICAL_CHUNK}" "$ENCODED_URL"
done
echo "$NUM_CHUNKS" > "${CHESTNUT_MODELS}/${CANONICAL}.chunkmanifest"
}
download_model_chunks "models/defaults/small" "${{ needs.prepare_small_model.outputs.driving_onnx_sha256 }}" "driving_tinygrad.pkl"
download_model_chunks "models/defaults/dm" "${{ needs.prepare_dm_model.outputs.dm_onnx_sha256 }}" "dmonitoring_model_tinygrad.pkl"
- name: Configure Git
run: |
@@ -507,7 +577,8 @@ jobs:
- build
- publish
- prepare_chestnut
- prepare_small_models
- prepare_small_model
- prepare_dm_model
runs-on: ubuntu-24.04
if: ${{ (always() && !cancelled() && !failure())
&& needs.publish.result == 'success'