diff --git a/.github/workflows/build-all-tinygrad-models.yaml b/.github/workflows/build-all-tinygrad-models.yaml
index 0eedb04703..1ba407b3cf 100644
--- a/.github/workflows/build-all-tinygrad-models.yaml
+++ b/.github/workflows/build-all-tinygrad-models.yaml
@@ -7,6 +7,19 @@ on:
description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
required: true
type: string
+ target_hardware:
+ description: 'Hardware target to compile for (qcom or usbgpu)'
+ required: true
+ type: choice
+ default: 'qcom'
+ options:
+ - qcom
+ - usbgpu
+ hf_repo:
+ description: 'Hugging Face dataset repository'
+ required: false
+ type: string
+ default: 'sunnypilot/sunnypilot_models_v1'
jobs:
setup:
@@ -46,13 +59,14 @@ jobs:
id: get-json
run: |
cd docs/docs
- latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
+ PREFIX="driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_' || '' }}v"
+ latest=$(ls ${PREFIX}*.json | sed -E "s/${PREFIX}([0-9]+)\.json/\1/" | sort -n | tail -1)
next=$((latest+1))
- json_file="driving_models_v${next}.json"
- cp "driving_models_v${latest}.json" "$json_file"
+ json_file="${PREFIX}${next}.json"
+ cp "${PREFIX}${latest}.json" "$json_file"
echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
- echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
+ echo "SRC_JSON_FILE=docs/docs/${PREFIX}${latest}.json" >> $GITHUB_ENV
- name: Extract tinygrad models
id: set-matrix
@@ -61,45 +75,23 @@ jobs:
jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
- - name: Set up SSH
- uses: webfactory/ssh-agent@v0.9.0
- with:
- ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
- - run: |
- mkdir -p ~/.ssh
- ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
- - name: Clone GitLab docs repo and create new recompiled dir
+ - name: Get next recompiled dir number
id: create-recompiled-dir
env:
- GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
+ HF_REPO: ${{ github.event.inputs.hf_repo }}
run: |
- git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
- cd gitlab_docs
- git checkout main
- git sparse-checkout set --no-cone models/
- cd models
- latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
- if [[ -z "$latest_dir" ]]; then
- next_dir=1
- else
- next_dir=$((latest_dir+1))
- fi
- recompiled_dir="${next_dir}"
- mkdir -p "recompiled${recompiled_dir}"
- touch "recompiled${recompiled_dir}/.gitkeep"
- cd ../..
+ pip install huggingface_hub
+ recompiled_dir=$(python3 -c "
+ from huggingface_hub import HfApi
+ import re, sys
+ api = HfApi()
+ files = api.list_repo_files(repo_id=sys.argv[1], repo_type='dataset')
+ dirs = [re.search(r'models/recompiled([0-9]+)', f) for f in files]
+ nums = [int(m.group(1)) for m in dirs if m]
+ print(max(nums) + 1)
+ " "$HF_REPO")
echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
- - name: Push empty recompiled dir to GitLab
- run: |
- cd gitlab_docs
- git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
- git config --global user.name "GitHub Action"
- git config --global user.email "action@github.com"
- git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
- git push origin main
-
- name: Push new JSON to GitHub docs repo
run: |
cd docs
@@ -123,25 +115,30 @@ jobs:
is_20hz: ${{ matrix.model.is_20hz }}
recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
json_version: ${{ needs.setup.outputs.json_version }}
+ target_hardware: ${{ github.event.inputs.target_hardware }}
+ hf_repo: ${{ github.event.inputs.hf_repo }}
+ set_min_version: ${{ github.event.inputs.set_min_version }}
+ tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
secrets: inherit
retry_failed_models:
needs: [setup, get_and_build]
runs-on: ubuntu-latest
- if: ${{ needs.setup.result != 'failure' && !cancelled() }}
+ if: ${{ !cancelled() && needs.setup.result == 'success' && (needs.get_and_build.result == 'success' || needs.get_and_build.result == 'failure') }}
outputs:
retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
steps:
- uses: actions/download-artifact@v4
with:
- pattern: model-*
+ pattern: artifact-name-*
path: output
+ continue-on-error: true
- id: set-retry-matrix
run: |
echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
- built=(); while IFS= read -r line; do built+=("$line"); done < <(
- find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
+ built=(); while IFS= read -r line; do [ -n "$line" ] && built+=("$line"); done < <(
+ find output -maxdepth 1 -name 'artifact-name-*' -printf "%f\n" 2>/dev/null | sed -E 's/^artifact-name-//' | awk '{gsub(/^ +| +$/, ""); print}'
)
jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
@@ -149,7 +146,7 @@ jobs:
retry_get_and_build:
needs: [setup, get_and_build, retry_failed_models]
- if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
+ if: ${{ !cancelled() && needs.retry_failed_models.result == 'success' && needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '' }}
strategy:
matrix:
model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
@@ -161,146 +158,9 @@ jobs:
is_20hz: ${{ matrix.model.is_20hz }}
recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
json_version: ${{ needs.setup.outputs.json_version }}
+ target_hardware: ${{ github.event.inputs.target_hardware }}
artifact_suffix: -retry
+ hf_repo: ${{ github.event.inputs.hf_repo }}
+ set_min_version: ${{ github.event.inputs.set_min_version }}
+ tinygrad_ref: ${{ needs.setup.outputs.tinygrad_ref }}
secrets: inherit
-
- publish_models:
- name: Publish models sequentially
- needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
- if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- max-parallel: 1
- matrix:
- model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
- env:
- RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
- JSON_FILE: ${{ needs.setup.outputs.json_file }}
- ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
- steps:
- - name: Set up SSH
- uses: webfactory/ssh-agent@v0.9.0
- with:
- ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
- - name: Add GitLab.com SSH key to known_hosts
- run: |
- mkdir -p ~/.ssh
- ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
- - name: Clone GitLab docs repo
- env:
- GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
- run: |
- echo "Cloning GitLab"
- git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
- cd gitlab_docs
- echo "checkout models/${RECOMPILED_DIR}"
- git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
- git checkout main
- cd ..
-
- - name: Checkout docs repo
- uses: actions/checkout@v4
- with:
- repository: sunnypilot/sunnypilot-models
- ref: gh-pages
- path: docs
- ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
- - name: Validate recompiled dir and JSON version
- run: |
- if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
- echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
- exit 1
- fi
- if [ ! -f "$JSON_FILE" ]; then
- echo "JSON file $JSON_FILE does not exist!"
- exit 1
- fi
-
- - name: Download artifact name file
- uses: actions/download-artifact@v4
- with:
- name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
- path: artifact_name
-
- - name: Read artifact name
- id: read-artifact-name
- run: |
- ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
- echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
- - name: Download model artifact
- uses: actions/download-artifact@v4
- with:
- name: ${{ steps.read-artifact-name.outputs.artifact_name }}
- path: output
-
- - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
- run: |
- find output -type f -name '*.onnx' -delete
- find output -type f -name 'big_*.pkl' -delete
- find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-
- - name: Copy model artifacts to gitlab
- env:
- ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
- run: |
- ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
- mkdir -p "$ARTIFACT_DIR"
- for path in output/*; do
- if [ "$(basename "$path")" = "artifact_name.txt" ]; then
- continue
- fi
- name="$(basename "$path")"
- if [ -d "$path" ]; then
- mkdir -p "$ARTIFACT_DIR/$name"
- cp -r "$path"/* "$ARTIFACT_DIR/$name/"
- echo "Copied dir $name -> $ARTIFACT_DIR/$name"
- else
- cp "$path" "$ARTIFACT_DIR/"
- echo "Copied file $name -> $ARTIFACT_DIR/"
- fi
- done
-
- - name: Push recompiled dir to GitLab
- env:
- GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
- run: |
- cd gitlab_docs
- git checkout main
- git pull origin main
- for d in models/"$RECOMPILED_DIR"/*/; do
- git sparse-checkout add "$d"
- done
- git add models/"$RECOMPILED_DIR"
- git config --global user.name "GitHub Action"
- git config --global user.email "action@github.com"
- git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
- git push origin main
- - run: |
- cd docs
- git pull origin gh-pages
-
- - name: update json
- run: |
- ARGS=""
- [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
- ARGS="$ARGS --sort-by-date"
- ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
- eval python3 docs/json_parser.py \
- --json-path "$JSON_FILE" \
- --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
- $ARGS
-
- - name: Push updated json to GitHub
- run: |
- cd docs
- git config --global user.name "GitHub Action"
- git config --global user.email "action@github.com"
- git checkout gh-pages
- git add docs/"$(basename $JSON_FILE)"
- git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
- git push origin gh-pages
diff --git a/.github/workflows/build-default-big-model.yaml b/.github/workflows/build-default-big-model.yaml
new file mode 100644
index 0000000000..4b05a7977d
--- /dev/null
+++ b/.github/workflows/build-default-big-model.yaml
@@ -0,0 +1,83 @@
+name: Build default big model
+
+on:
+ workflow_dispatch:
+
+env:
+ HF_REPO: sunnypilot/sunnypilot_models_v1
+ HF_DEFAULTS_PATH: models/defaults/big
+
+jobs:
+ resolve_name:
+ runs-on: ubuntu-24.04
+ outputs:
+ model_name: ${{ steps.name.outputs.model_name }}
+ onnx_ref: ${{ steps.name.outputs.onnx_ref }}
+ steps:
+ - uses: actions/checkout@v4
+ - id: name
+ run: |
+ NAME=$(PYTHONPATH=${{ github.workspace }} python3 -c "from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL; print(DEFAULT_BIG_MODEL)")
+ ONNX_REF=$(git log -1 --format='%H' -- openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx)
+ echo "model_name=${NAME}" >> $GITHUB_OUTPUT
+ echo "onnx_ref=$ONNX_REF" >> $GITHUB_OUTPUT
+
+ build_model:
+ needs: resolve_name
+ uses: ./.github/workflows/sunnypilot-build-model.yaml
+ with:
+ upstream_branch: ${{ needs.resolve_name.outputs.onnx_ref }}
+ custom_name: ${{ needs.resolve_name.outputs.model_name }}
+ target_hardware: usbgpu
+ secrets: inherit
+
+ upload_defaults:
+ needs: [ resolve_name, build_model ]
+ runs-on: ubuntu-24.04
+ permissions:
+ id-token: write
+ contents: write
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+
+ - name: Install huggingface_hub
+ run: pip install --upgrade "huggingface_hub>=0.22.0"
+
+ - name: Download artifact name
+ uses: actions/download-artifact@v4
+ with:
+ name: artifact-name-${{ needs.resolve_name.outputs.model_name }}
+ path: artifact_name
+
+ - name: Read artifact name
+ id: artifact
+ run: |
+ ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
+ echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
+
+ - name: Download model artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ steps.artifact.outputs.artifact_name }}
+ path: output
+
+ - name: Upload to HF and update default_models.json
+ env:
+ HF_OIDC_RESOURCE: datasets/${{ env.HF_REPO }}
+ ARTIFACT_NAME: ${{ steps.artifact.outputs.artifact_name }}
+ run: |
+ rm -f output/artifact_name.txt
+ export PYTHONPATH=$(pwd)
+ python3 release/ci/upload_default_model.py \
+ --hf-repo "${{ env.HF_REPO }}" \
+ --hf-defaults-path "${{ env.HF_DEFAULTS_PATH }}" \
+ --artifact-name "$ARTIFACT_NAME" \
+ --model-dir output \
+ --onnx-path "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" \
+ --onnx-ref "${{ needs.resolve_name.outputs.onnx_ref }}" \
+ --model-name "${{ needs.resolve_name.outputs.model_name }}" \
+ --tinygrad-ref "$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)" \
+ --run-number "${{ github.run_number }}"
diff --git a/.github/workflows/build-single-tinygrad-model.yaml b/.github/workflows/build-single-tinygrad-model.yaml
index cf2d870802..1ad06a54e4 100644
--- a/.github/workflows/build-single-tinygrad-model.yaml
+++ b/.github/workflows/build-single-tinygrad-model.yaml
@@ -29,11 +29,24 @@ on:
required: false
type: boolean
default: true
- bypass_push:
- description: 'Bypass pushing to GitLab for build-all'
+ target_hardware:
+ description: 'Hardware target to compile for (qcom or usbgpu)'
required: false
- default: true
- type: boolean
+ type: string
+ default: 'qcom'
+ hf_repo:
+ description: 'Hugging Face dataset repository (e.g. sunnypilot/sunnypilot_models_v1)'
+ required: false
+ type: string
+ default: 'sunnypilot/sunnypilot_models_v1'
+ set_min_version:
+ description: 'Minimum selector version'
+ required: false
+ type: string
+ tinygrad_ref:
+ description: 'Tinygrad reference'
+ required: false
+ type: string
workflow_dispatch:
inputs:
upstream_branch:
@@ -65,8 +78,8 @@ on:
- None
- Master Models
- Release Models
- - 2025 World Models
- 2026 World Models
+ - 2026 Deep RL Models
- Custom Merge Models
- Other
custom_model_folder:
@@ -81,9 +94,22 @@ on:
description: 'Minimum selector version'
required: false
type: string
+ target_hardware:
+ description: 'Hardware target to compile for'
+ required: false
+ type: choice
+ default: 'qcom'
+ options:
+ - qcom
+ - usbgpu
+ hf_repo:
+ description: 'Hugging Face dataset repository'
+ required: false
+ type: string
+ default: 'sunnypilot/sunnypilot_models_v1'
env:
RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
- JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
+ JSON_FILE: docs/docs/driving_models_${{ inputs.target_hardware == 'usbgpu' && 'usbgpu_v' || 'v' }}${{ inputs.json_version }}.json
jobs:
build_model:
@@ -93,38 +119,20 @@ jobs:
custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
is_20hz: ${{ inputs.is_20hz }}
artifact_suffix: ${{ inputs.artifact_suffix }}
+ target_hardware: ${{ inputs.target_hardware }}
secrets: inherit
publish_model:
- if: ${{ !inputs.bypass_push && !cancelled() }}
+ if: ${{ !cancelled() && needs.build_model.result == 'success' }}
concurrency:
- group: gitlab-push-${{ inputs.recompiled_dir }}
+ group: hf-push-${{ inputs.recompiled_dir }}
cancel-in-progress: false
needs: build_model
runs-on: ubuntu-latest
+ permissions:
+ id-token: write
+ contents: write
steps:
- - name: Set up SSH
- uses: webfactory/ssh-agent@v0.9.0
- with:
- ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
- - name: Add GitLab.com SSH key to known_hosts
- run: |
- mkdir -p ~/.ssh
- ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
- - name: Clone GitLab docs repo
- env:
- GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
- run: |
- echo "Cloning GitLab"
- git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
- cd gitlab_docs
- echo "checkout models/${RECOMPILED_DIR}"
- git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
- git checkout main
- cd ..
-
- name: Checkout docs repo
uses: actions/checkout@v4
with:
@@ -133,16 +141,28 @@ jobs:
path: docs
ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
- - name: Validate recompiled dir and JSON version
+ - name: Install huggingface_hub
+ run: pip install --upgrade "huggingface_hub>=0.22.0"
+
+ - name: Validate hf_repo and JSON version
+ env:
+ HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
run: |
- if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
- echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
- exit 1
- fi
if [ ! -f "$JSON_FILE" ]; then
echo "JSON file $JSON_FILE does not exist!"
exit 1
fi
+ python3 -c "
+ import sys
+ from huggingface_hub import HfApi
+ try:
+ api = HfApi()
+ api.repo_info(repo_id=sys.argv[1], repo_type='dataset')
+ print(f'Success: Repo {sys.argv[1]} exists.')
+ except Exception as e:
+ print('HF validation failed:', e)
+ sys.exit(1)
+ " "${{ inputs.hf_repo }}"
- name: Download artifact name file
uses: actions/download-artifact@v4
@@ -162,49 +182,26 @@ jobs:
name: ${{ steps.read-artifact-name.outputs.artifact_name }}
path: output
- - name: Remove unwanted files
- run: |
- find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
- find output -type f -name 'dmonitoring_model.onnx' -delete
-
- - name: Copy model artifact(s) to GitLab recompiled dir
+ - name: Create models folder
env:
ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
run: |
- ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
- mkdir -p "$ARTIFACT_DIR"
- for path in output/*; do
- if [ "$(basename "$path")" = "artifact_name.txt" ]; then
- continue
- fi
- name="$(basename "$path")"
- if [ -d "$path" ]; then
- mkdir -p "$ARTIFACT_DIR/$name"
- cp -r "$path"/* "$ARTIFACT_DIR/$name/"
- echo "Copied dir $name -> $ARTIFACT_DIR/$name"
- else
- cp "$path" "$ARTIFACT_DIR/"
- echo "Copied file $name -> $ARTIFACT_DIR/"
- fi
- done
+ mkdir -p "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
+ cp -r output/* "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/"
+ rm -f "local_models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/artifact_name.txt"
- - name: Push recompiled dir to GitLab
+ - name: Upload to Hugging Face
env:
- GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
+ HF_OIDC_RESOURCE: datasets/${{ inputs.hf_repo }}
+ ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
run: |
- cd gitlab_docs
- git checkout main
- git pull origin main
- for d in models/"$RECOMPILED_DIR"/*/; do
- git sparse-checkout add "$d"
- done
- git add models/"$RECOMPILED_DIR"
- git config --global user.name "GitHub Action"
- git config --global user.email "action@github.com"
- git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
- git push origin main
+ hf upload ${{ inputs.hf_repo }} \
+ output/ \
+ "models/${RECOMPILED_DIR}/${ARTIFACT_NAME}/" \
+ --repo-type=dataset
- - run: |
+ - name: Pull gh-pages
+ run: |
cd docs
git pull origin gh-pages
@@ -220,9 +217,11 @@ jobs:
fi
[ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
[ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
+ [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
+ [ -n "${{ inputs.tinygrad_ref }}" ] && ARGS="$ARGS --tinygrad-ref \"${{ inputs.tinygrad_ref }}\""
eval python3 docs/json_parser.py \
--json-path "$JSON_FILE" \
- --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
+ --recompiled-dir "local_models/$RECOMPILED_DIR" \
--sort-by-date \
$ARGS
diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml
index 2c435e58a0..bc132ae1bf 100644
--- a/.github/workflows/sunnypilot-build-model.yaml
+++ b/.github/workflows/sunnypilot-build-model.yaml
@@ -80,6 +80,7 @@ jobs:
with:
repository: commaai/openpilot
ref: ${{ inputs.upstream_branch }}
+ fetch-depth: 1
submodules: recursive
path: openpilot
@@ -89,26 +90,38 @@ jobs:
with:
repository: sunnypilot/sunnypilot
ref: ${{ inputs.upstream_branch }}
+ fetch-depth: 1
submodules: recursive
path: openpilot
- name: Get commit date
id: commit-date
run: |
- cd ${{ github.workspace }}/openpilot
+ cd ${{ github.workspace }}/openpilot/openpilot
commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
- run: |
- cd ${{ github.workspace }}/openpilot
- git lfs pull
+ cd ${{ github.workspace }}/openpilot/openpilot
+ if [ "${{ inputs.target_hardware }}" != "usbgpu" ]; then
+ git lfs pull -X "**/selfdrive/modeld/models/big_*.onnx,**/selfdrive/modeld/models/dmonitoring_*.onnx"
+ rm -f selfdrive/modeld/models/big_*.onnx selfdrive/modeld/models/dmonitoring_*.onnx
+ else
+ git lfs pull -I "**/selfdrive/modeld/models/big_*.onnx" -X ""
+ find selfdrive/modeld/models -name "*.onnx" ! -name "big_*.onnx" -delete
+ fi
+ if grep -lIF "version https://git-lfs.github.com/spec/v1" selfdrive/modeld/models/*.onnx; then
+ echo "::error::the ONNX files above are still LFS pointers, not real models"
+ exit 1
+ fi
- name: 'Upload Artifact'
uses: actions/upload-artifact@v4
with:
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
+ if-no-files-found: error
build_model:
- runs-on: [self-hosted, tici]
+ runs-on: [self-hosted, usbgpu]
needs: get_model
env:
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
@@ -116,24 +129,9 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
+ fetch-depth: 1
submodules: recursive
- - run: git lfs pull
- - name: Cache SCons
- uses: actions/cache@v4
- with:
- path: ${{env.SCONS_CACHE_DIR}}
- key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
- # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
- # for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
- restore-keys: |
- scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
- scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
- scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
- scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
- scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
- scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
- scons-${{ runner.os }}-${{ runner.arch }}
- name: Set environment variables
id: set-env
@@ -144,7 +142,7 @@ jobs:
export UV_PYTHON_PREFERENCE=managed
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
- uv sync
+ uv sync --frozen
printenv >> $GITHUB_ENV
if [[ "${{ runner.debug }}" == "1" ]]; then
cat $GITHUB_OUTPUT
@@ -166,15 +164,13 @@ jobs:
fi
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
- rm -rf ${{ env.MODELS_DIR }}/*.onnx
+ rm -rf ${{ env.MODELS_DIR }}/*.onnx*
- name: Download model artifacts
uses: actions/download-artifact@v4
with:
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
path: ${{ env.MODELS_DIR }}
- - run: |
- rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big_driving_supercombo}.onnx
- name: Build Model
run: |
@@ -188,34 +184,48 @@ jobs:
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
+ TG_FLAGS_QCOM="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
echo "USBGPU build"
export USBGPU=1
- TG_FLAGS="DEV=AMD USBGPU=1 IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+ TG_FLAGS="DEBUG=2 DEV=USB+AMD:LLVM WARP_DEV=QCOM FLOAT16=1 JIT_BATCH_SIZE=0 GMMU=0 TC_OPT=2"
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
else
echo "QCOM build"
- TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+ TG_FLAGS="$TG_FLAGS_QCOM"
OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
fi
# Generate metadata for all ONNX files
find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
echo "Generating metadata: $onnx_file"
- env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
+ env ${TG_FLAGS_QCOM} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
done
# Detect model type and build compile args
- VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
- POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
- OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
- ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
+ VISION_ONNX=""
+ for f in "${{ env.MODELS_DIR }}/driving_vision.onnx" "${{ env.MODELS_DIR }}/big_driving_vision.onnx"; do
+ [ -f "$f" ] && VISION_ONNX="$f" && break
+ done
+
+ POLICY_ONNX=""
+ for f in "${{ env.MODELS_DIR }}/driving_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_policy.onnx"; do
+ [ -f "$f" ] && POLICY_ONNX="$f" && break
+ done
+
+ OFF_POLICY_ONNX=""
+ for f in "${{ env.MODELS_DIR }}/driving_off_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_off_policy.onnx"; do
+ [ -f "$f" ] && OFF_POLICY_ONNX="$f" && break
+ done
+
+ ON_POLICY_ONNX=""
+ for f in "${{ env.MODELS_DIR }}/driving_on_policy.onnx" "${{ env.MODELS_DIR }}/big_driving_on_policy.onnx"; do
+ [ -f "$f" ] && ON_POLICY_ONNX="$f" && break
+ done
+
SUPERCOMBO_ONNX=""
- for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
- if [ -f "$f" ]; then
- SUPERCOMBO_ONNX="$f"
- break
- fi
+ for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx" "${{ env.MODELS_DIR }}/big_supercombo.onnx" "${{ env.MODELS_DIR }}/big_driving_supercombo.onnx"; do
+ [ -f "$f" ] && SUPERCOMBO_ONNX="$f" && break
done
MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
@@ -254,10 +264,8 @@ jobs:
# Copy the model files
rsync -avm \
--include='*.dlc' \
- --include='*.pkl' \
--include='*.chunk*' \
--include='*.chunkmanifest' \
- --include='*.onnx' \
--exclude='*' \
--delete-excluded \
--chown=comma:comma \
diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml
index 12a3a7bce0..e155a68d4a 100644
--- a/.github/workflows/sunnypilot-build-prebuilt.yaml
+++ b/.github/workflows/sunnypilot-build-prebuilt.yaml
@@ -4,12 +4,8 @@ env:
BUILD_DIR: "/data/openpilot"
OUTPUT_DIR: ${{ github.workspace }}/output
CI_DIR: ${{ github.workspace }}/release/ci
- SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
- # Branch configurations
- STAGING_SOURCE_BRANCH: 'master'
-
# Runtime configuration
SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
@@ -40,6 +36,7 @@ jobs:
publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
build: ${{ steps.strategy.outputs.build }}
+ include_big_model: ${{ steps.strategy.outputs.include_big_model }}
steps:
- uses: actions/checkout@v4
- name: Extract deploy strategy
@@ -82,6 +79,9 @@ jobs:
stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
+
+ include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')";
+ echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT
fi
echo "build=$BUILD" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
@@ -109,11 +109,6 @@ jobs:
group: build-${{ github.head_ref || github.ref_name }}
cancel-in-progress: false
runs-on: [self-hosted, tici]
- outputs:
- new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
- version: ${{ needs.prepare_strategy.outputs.version }}
- extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
- commit_sha: ${{ github.sha }}
if: ${{
(always() && !cancelled() && !failure()) &&
needs.prepare_strategy.result == 'success' &&
@@ -129,26 +124,8 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
- run: git lfs pull
- - name: Cache SCons
- uses: actions/cache@v4
- with:
- path: ${{env.SCONS_CACHE_DIR}}
- key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
- # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
- # for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
- restore-keys: |
- scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
- scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
- scons-${{ runner.os }}-${{ runner.arch }}
-
- name: Set environment variables
- id: set-env
run: |
- echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
- echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
- echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
- echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
-
# Set up common environment
source /etc/profile;
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
@@ -157,9 +134,6 @@ jobs:
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
uv sync
printenv >> $GITHUB_ENV
- if [[ "${{ runner.debug }}" == "1" ]]; then
- cat $GITHUB_OUTPUT
- fi
- name: Setup build environment
run: |
@@ -168,7 +142,7 @@ jobs:
echo "Starting build stage..."
echo "BUILD_DIR: ${BUILD_DIR}"
echo "CI_DIR: ${CI_DIR}"
- echo "VERSION: ${{ steps.set-env.outputs.version }}"
+ echo "VERSION: ${{ needs.prepare_strategy.outputs.version }}"
echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
echo "-------"
@@ -180,61 +154,44 @@ jobs:
- name: Build Main Project
run: |
- export PYTHONPATH="$BUILD_DIR"
- ./tools/release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
+ export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
+ ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" --
+ # outside the checkout, which is wiped each run. /data/scons_cache is the device's, not ours.
+ SCONS_CACHE="$RUNNER_WORKSPACE/scons_cache"
+ mkdir -p "$SCONS_CACHE"
cd $BUILD_DIR
- ln -sfn msgq_repo/msgq msgq
- ln -sfn opendbc_repo/opendbc opendbc
- ln -sfn rednose_repo/rednose rednose
- ln -sfn teleoprtc_repo/teleoprtc teleoprtc
- ln -sfn tinygrad_repo/tinygrad tinygrad
- sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
- echo "Building sunnypilot's modeld_v2..."
- scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/modeld_v2
- echo "Building sunnypilot's locationd..."
- scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/selfdrive/locationd
- echo "Building openpilot's locationd..."
- scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/selfdrive/locationd
+ echo "Building locationd..."
+ # -j1: parallel rednose generators OOM the device
+ scons -j1 cache_dir="$SCONS_CACHE" --minimal \
+ openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
echo "Building rest of sunnypilot"
- scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
+ /usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
touch ${BUILD_DIR}/prebuilt
if [[ "${{ runner.debug }}" == "1" ]]; then
ls -la ${BUILD_DIR}
fi
- - name: Prepare Output
+ - name: Strip release tree
run: |
- sudo rm -rf ${OUTPUT_DIR}
- mkdir -p ${OUTPUT_DIR}
- rsync -am${RUNNER_DEBUG:+v} \
- --exclude='.sconsign.dblite' \
- --exclude='*.a' \
- --exclude='*.o' \
- --exclude='*.os' \
- --exclude='*.pyc' \
- --exclude='moc_*' \
- --exclude='__pycache__' \
- --exclude='Jenkinsfile' \
- --exclude='**/release/' \
- --exclude='**/.github/' \
- --exclude='**/openpilot/selfdrive/ui/replay/' \
- --exclude='**/__pycache__/' \
- --exclude='${{env.SCONS_CACHE_DIR}}' \
- --exclude='**/.git/' \
- --exclude='**/SConstruct' \
- --exclude='**/SConscript' \
- --exclude='**/.venv/' \
- --exclude='openpilot/selfdrive/modeld/models/*.onnx*' \
- --exclude='openpilot/sunnypilot/modeld*/models/*.onnx*' \
- --exclude='openpilot/third_party/*x86*' \
- --exclude='openpilot/third_party/*Darwin*' \
- --delete-excluded \
- --chown=comma:comma \
- ${BUILD_DIR}/ ${OUTPUT_DIR}/
+ cd $BUILD_DIR
+ find . -name '*.a' -delete
+ find . -name '*.o' -delete
+ find . -name '*.os' -delete
+ find . -name '*.pyc' -delete
+ find . -name 'moc_*' -delete
+ find . -name '__pycache__' -type d -exec rm -rf {} +
+ find . -name 'SConstruct' -delete
+ find . -name 'SConscript' -delete
+ rm -rf .sconsign.dblite Jenkinsfile tools/release/ release/
+ rm -f openpilot/selfdrive/modeld/models/*.onnx*
+ rm -f openpilot/sunnypilot/modeld*/models/*.onnx*
+ find openpilot/third_party/ -name '*x86*' -exec rm -r {} +
+ find openpilot/third_party/ -name '*Darwin*' -exec rm -r {} +
+ cd -
- name: 'Tar.gz files'
run: |
- tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} .
+ tar czf prebuilt.tar.gz -C ${{ env.BUILD_DIR }} .
ls -la prebuilt.tar.gz
- name: 'Upload Artifact'
@@ -242,6 +199,7 @@ jobs:
with:
name: prebuilt
path: prebuilt.tar.gz
+ compression-level: 0
- name: Re-enable powersave
if: always()
@@ -249,6 +207,101 @@ jobs:
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
+ prepare_chestnut:
+ needs: [ prepare_strategy ]
+ runs-on: ubuntu-24.04
+ if: ${{ needs.prepare_strategy.outputs.include_big_model == 'true' }}
+ env:
+ HF_REPO: sunnypilot/sunnypilot_models_v1
+ HF_DEFAULTS_PATH: models/defaults/big
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ github.head_ref || github.ref_name }}
+
+ - run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
+
+ - name: Check HF defaults and build if needed
+ id: resolve
+ run: |
+ ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
+ echo "Repo ONNX hash: $ACTUAL_ONNX_HASH"
+
+ JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+
+ check_hash() {
+ DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
+ BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
+ [ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ]
+ }
+
+ if check_hash; then
+ echo "HF defaults match repo ONNX"
+ else
+ echo "No matching model on HF — triggering build"
+ gh workflow run build-default-big-model.yaml --ref "${{ github.head_ref || github.ref_name }}"
+
+ echo "Waiting for build to start..."
+ sleep 120
+
+ RUN_ID=$(gh run list --workflow=build-default-big-model.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
+ if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
+ echo "::error::Failed to find build-default-big-model run"
+ exit 1
+ fi
+
+ echo "Waiting for run $RUN_ID..."
+ gh run watch "$RUN_ID"
+
+ CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
+ if [ "$CONCLUSION" != "success" ]; then
+ echo "::error::build-default-big-model failed: $CONCLUSION"
+ exit 1
+ fi
+
+ if ! check_hash; then
+ echo "::error::HF defaults still don't match after build"
+ exit 1
+ fi
+ fi
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Download big model chunks
+ run: |
+ ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
+ JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
+ DEFAULTS=$(curl -fsSL "$JSON_URL")
+ BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
+
+ mkdir -p big_model_chunks
+ ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
+ BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
+ NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
+
+ CANONICAL="big_driving_tinygrad.pkl"
+ echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
+ CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
+ CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
+ ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
+ echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
+ curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
+ done
+
+ echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
+
+ - name: Upload big model chunks
+ uses: actions/upload-artifact@v4
+ with:
+ name: big-model-chunks
+ path: big_model_chunks/
+ compression-level: 0
+
+ - name: Cancel run on failure
+ if: failure()
+ run: gh run cancel ${{ github.run_id }}
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
publish:
concurrency:
@@ -257,14 +310,20 @@ jobs:
# Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
- if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
- needs: [ build, prepare_strategy ]
+ if: ${{
+ always() && !cancelled() &&
+ needs.build.result == 'success' &&
+ needs.prepare_strategy.result == 'success' &&
+ (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
+ (needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
+ }}
+ needs: [ build, prepare_strategy, prepare_chestnut ]
runs-on: ubuntu-24.04
environment: ${{ needs.prepare_strategy.outputs.environment }}
steps:
- uses: actions/checkout@v4
- - name: Download build artifacts
+ - name: Download prebuilt artifact
uses: actions/download-artifact@v4
with:
name: prebuilt
@@ -274,6 +333,24 @@ jobs:
mkdir -p ${{ env.OUTPUT_DIR }}
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
+ - name: Prepare chestnut output
+ if: ${{ needs.prepare_chestnut.result == 'success' }}
+ run: |
+ mkdir -p "${{ github.workspace }}/chestnut_output"
+ tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
+
+ - name: Download big model chunks
+ if: ${{ needs.prepare_chestnut.result == 'success' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: big-model-chunks
+ path: big_model_chunks
+
+ - name: Inject big model into chestnut
+ if: ${{ needs.prepare_chestnut.result == 'success' }}
+ run: |
+ cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/"
+
- name: Configure Git
run: |
git config --global user.email "github-actions[bot]@users.noreply.github.com"
@@ -283,29 +360,38 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
- echo '${{ toJSON(needs.build.outputs) }}'
+ echo '${{ toJSON(needs.prepare_strategy.outputs) }}'
ls -la ${{ env.OUTPUT_DIR }}
${{ env.CI_DIR }}/publish.sh \
"${{ github.workspace }}" \
"${{ env.OUTPUT_DIR }}" \
- "${{ needs.build.outputs.new_branch }}" \
- "${{ needs.build.outputs.version }}" \
+ "${{ needs.prepare_strategy.outputs.new_branch }}" \
+ "${{ needs.prepare_strategy.outputs.version }}" \
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
- "${{ needs.build.outputs.extra_version_identifier }}"
+ "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
- echo ""
- echo "---- ℹ️ To update the list of branches that auto deploy prebuilts -----"
- echo ""
- echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
- echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
- echo "3. Update as needed (JSON array with no spaces)"
+ - name: Publish chestnut branch
+ if: ${{ needs.prepare_chestnut.result == 'success' }}
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
+ CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
+
+ ${{ env.CI_DIR }}/publish.sh \
+ "${{ github.workspace }}" \
+ "$CHESTNUT_DIR" \
+ "$CHESTNUT_BRANCH" \
+ "${{ needs.prepare_strategy.outputs.version }}" \
+ "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
+ "${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
- name: Tag ${{ needs.prepare_strategy.outputs.environment }}
if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
run: |
TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
- git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
+ git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
git push -f origin ${TAG}
notify:
@@ -313,6 +399,7 @@ jobs:
- prepare_strategy
- build
- publish
+ - prepare_chestnut
runs-on: ubuntu-24.04
if: ${{ (always() && !cancelled() && !failure())
&& needs.publish.result == 'success'
@@ -324,7 +411,6 @@ jobs:
- name: Prepare notification message
id: message
run: |
- TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
export VERSION="${{ needs.prepare_strategy.outputs.version }}"
export branch_name="${{ env.SOURCE_BRANCH }}"
export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
@@ -333,6 +419,7 @@ jobs:
export commit_short_sha="${commit_short_sha:0:7}"
export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
+ export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}"
MESSAGE=$(cat << 'EOF' | envsubst
${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
@@ -373,7 +460,7 @@ jobs:
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
- name: process.env.LABELf
+ name: process.env.LABEL
});
console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`);
diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml
index 8b7b63f344..c6bc63c8ac 100644
--- a/.github/workflows/ui_preview.yaml
+++ b/.github/workflows/ui_preview.yaml
@@ -25,7 +25,8 @@ env:
jobs:
preview:
- if: github.repository == 'sunnypilot/sunnypilot'
+ if: false # tmp disable due to GH API rate limiting flakiness
+ #if: github.repository == 'sunnypilot/sunnypilot'
name: preview
runs-on: ubuntu-latest
timeout-minutes: 20
diff --git a/Jenkinsfile b/Jenkinsfile
index db18d13ddd..af33edb821 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -22,6 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
export CI=1
export PYTHONWARNINGS=error
+export COMMA_CACHE=/data/tmp/comma_download_cache
#export LOGPRINT=debug # this has gotten too spammy...
export TEST_DIR=${env.TEST_DIR}
export SOURCE_DIR=${env.SOURCE_DIR}
diff --git a/docs/CARS.md b/docs/CARS.md
index 22c35b9a12..0866436c0d 100644
--- a/docs/CARS.md
+++ b/docs/CARS.md
@@ -4,7 +4,7 @@
A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
-# 342 Supported Cars
+# 345 Supported Cars
|Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|![]()
Hardware Needed
|Video|Setup Video|
|---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
@@ -34,6 +34,7 @@ A supported vehicle is one that just works when you install a comma device. All
|Chrysler|Pacifica Hybrid 2019-25|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[](##)|[](##)|Parts
- 1 FCA connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy Here
||
|CUPRA[12](#footnotes)|Ateca 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[1,16](#footnotes)|0 mph|0 mph|[](##)|[](##)|Parts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 FCA connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Ford Q3 connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Ford Q3 connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch B connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Nidec connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch A connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch B connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch B connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch A connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch A connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Honda Bosch B connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Hyundai N connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Hyundai E connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 FCA connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 FCA connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 FCA connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Hyundai A connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Hyundai K connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 Hyundai E connector
- 1 OBD-C cable (2 ft)
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 Rivian B connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru B connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru B connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 Subaru A connector
- 1 comma four
- 1 comma power v3
- 1 harness box
- 1 mount
Buy HereTools
- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)Parts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy HereParts
- 1 OBD-C cable (2 ft)
- 1 VW J533 connector
- 1 comma four
- 1 harness box
- 1 long OBD-C cable (9.5 ft)
- 1 mount
Buy Here