Files
openpilot-evo/.github/workflows/sunnypilot-build-prebuilt.yaml
2026-08-23 02:48:10 -04:00

460 lines
20 KiB
YAML

name: sunnypilot prebuilt action
env:
BUILD_DIR: "/data/openpilot"
OUTPUT_DIR: ${{ github.workspace }}/output
CI_DIR: ${{ github.workspace }}/release/ci
PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
# Runtime configuration
SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
on:
push:
branches: [ master, master-dev ]
tags: [ 'release/*' ]
pull_request_target:
types: [ labeled ]
workflow_dispatch:
inputs:
wait_for_tests:
description: 'Wait for tests to finish'
required: false
type: boolean
default: false
jobs:
prepare_strategy:
runs-on: ubuntu-24.04
if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
outputs:
environment: ${{ steps.strategy.outputs.environment }}
new_branch: ${{ steps.strategy.outputs.new_branch }}
extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
version: ${{ steps.strategy.outputs.version }}
cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
build: ${{ steps.strategy.outputs.build }}
include_big_model: ${{ steps.strategy.outputs.include_big_model }}
steps:
- uses: actions/checkout@v4
- name: Extract deploy strategy
id: strategy
run: |
echo '::group::Strategy Extraction'
BRANCH="${{ github.head_ref || github.ref_name }}"
echo "Current branch: $BRANCH"
STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
.configs[] | select(.branch == $branch)
')
BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
echo "No exact strategy match found. Falling back to feature/fork logic."
IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
echo "version=$BUILD" >> $GITHUB_OUTPUT
echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
echo "environment=feature-branch" >> $GITHUB_OUTPUT
echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
else
echo "Matched config: $CONFIG"
environment=$(echo "$CONFIG" | jq -r '.environment')
echo "environment=$environment" >> $GITHUB_OUTPUT
echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
include_big_model="$(echo "$CONFIG" | jq -r '.include_big_model // false')";
echo "include_big_model=$include_big_model" >> $GITHUB_OUTPUT
fi
echo "build=$BUILD" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
validate_tests:
runs-on: ubuntu-24.04
needs: [ prepare_strategy ]
if: ${{
((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
(github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
}}
steps:
- uses: actions/checkout@v4
- name: Wait for Tests
uses: ./.github/workflows/wait-for-action # Path to where you place the action
with:
workflow: tests.yaml # The workflow file to monitor
github-token: ${{ secrets.GITHUB_TOKEN }}
should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
build:
needs: [ validate_tests, prepare_strategy ]
concurrency:
group: build-${{ github.head_ref || github.ref_name }}
cancel-in-progress: false
runs-on: [self-hosted, tici]
if: ${{
(always() && !cancelled() && !failure()) &&
needs.prepare_strategy.result == 'success' &&
(needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
(!contains(github.event_name, 'pull_request') ||
(github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
}}
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
ref: ${{ env.SOURCE_BRANCH }}
repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
- run: git lfs pull
- name: Set environment variables
run: |
# Set up common environment
source /etc/profile;
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export UV_PYTHON_PREFERENCE=managed
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
uv sync
printenv >> $GITHUB_ENV
- name: Setup build environment
run: |
mkdir -p "${BUILD_DIR}/"
sudo find $BUILD_DIR/ -mindepth 1 -delete
echo "Starting build stage..."
echo "BUILD_DIR: ${BUILD_DIR}"
echo "CI_DIR: ${CI_DIR}"
echo "VERSION: ${{ needs.prepare_strategy.outputs.version }}"
echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
echo "-------"
if [[ "${{ runner.debug }}" == "1" ]]; then
printenv
fi
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
- name: Build Main Project
run: |
export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/msgq_repo:$BUILD_DIR/opendbc_repo:$BUILD_DIR/rednose_repo:$BUILD_DIR/teleoprtc_repo:$BUILD_DIR/tinygrad_repo"
./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" --
# outside the checkout, which is wiped each run. /data/scons_cache is the device's, not ours.
SCONS_CACHE="$RUNNER_WORKSPACE/scons_cache"
mkdir -p "$SCONS_CACHE"
cd $BUILD_DIR
echo "Building locationd..."
# -j1: parallel rednose generators OOM the device
scons -j1 cache_dir="$SCONS_CACHE" --minimal \
openpilot/selfdrive/locationd openpilot/sunnypilot/selfdrive/locationd
echo "Building rest of sunnypilot"
/usr/bin/time -v scons -j$(nproc) cache_dir="$SCONS_CACHE" --minimal
touch ${BUILD_DIR}/prebuilt
if [[ "${{ runner.debug }}" == "1" ]]; then
ls -la ${BUILD_DIR}
fi
- name: Strip release tree
run: |
cd $BUILD_DIR
find . -name '*.a' -delete
find . -name '*.o' -delete
find . -name '*.os' -delete
find . -name '*.pyc' -delete
find . -name 'moc_*' -delete
find . -name '__pycache__' -type d -exec rm -rf {} +
find . -name 'SConstruct' -delete
find . -name 'SConscript' -delete
rm -rf .sconsign.dblite Jenkinsfile tools/release/ release/
rm -f openpilot/selfdrive/modeld/models/*.onnx*
rm -f openpilot/sunnypilot/modeld*/models/*.onnx*
find openpilot/third_party/ -name '*x86*' -exec rm -r {} +
find openpilot/third_party/ -name '*Darwin*' -exec rm -r {} +
cd -
- name: 'Tar.gz files'
run: |
tar czf prebuilt.tar.gz -C ${{ env.BUILD_DIR }} .
ls -la prebuilt.tar.gz
- name: 'Upload Artifact'
uses: actions/upload-artifact@v4
with:
name: prebuilt
path: prebuilt.tar.gz
compression-level: 0
- name: Re-enable powersave
if: always()
run: |
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
prepare_chestnut:
needs: [ prepare_strategy ]
runs-on: ubuntu-24.04
if: ${{ needs.prepare_strategy.outputs.include_big_model == 'true' }}
outputs:
onnx_sha256: ${{ steps.resolve.outputs.onnx_sha256 }}
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
HF_DEFAULTS_PATH: models/defaults/big
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref || github.ref_name }}
- run: git lfs pull -I "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx"
- name: Check HF defaults and build if needed
id: resolve
run: |
ACTUAL_ONNX_HASH=$(sha256sum "openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx" | cut -d' ' -f1)
echo "Repo ONNX hash: $ACTUAL_ONNX_HASH"
echo "onnx_sha256=$ACTUAL_ONNX_HASH" >> $GITHUB_OUTPUT
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
check_hash() {
DEFAULTS=$(curl -fsSL "$JSON_URL" 2>/dev/null) || return 1
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ACTUAL_ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)' 2>/dev/null)
[ -n "$BUNDLE" ] && [ "$BUNDLE" != "null" ]
}
if check_hash; then
echo "HF defaults match repo ONNX"
else
echo "No matching model on HF — triggering build"
gh workflow run build-default-big-model.yaml --ref "${{ github.head_ref || github.ref_name }}"
echo "Waiting for build to start..."
sleep 120
RUN_ID=$(gh run list --workflow=build-default-big-model.yaml --branch="${{ github.head_ref || github.ref_name }}" --limit=1 --json databaseId --jq '.[0].databaseId')
if [ -z "$RUN_ID" ] || [ "$RUN_ID" = "null" ]; then
echo "::error::Failed to find build-default-big-model run"
exit 1
fi
echo "Waiting for run $RUN_ID..."
gh run watch "$RUN_ID"
CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
if [ "$CONCLUSION" != "success" ]; then
echo "::error::build-default-big-model failed: $CONCLUSION"
exit 1
fi
if ! check_hash; then
echo "::error::HF defaults still don't match after build"
exit 1
fi
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Cancel run on failure
if: failure()
run: gh run cancel ${{ github.run_id }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
publish:
concurrency:
# We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
# This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
# Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
if: ${{
always() && !cancelled() &&
needs.build.result == 'success' &&
needs.prepare_strategy.result == 'success' &&
(!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) &&
(needs.prepare_strategy.outputs.include_big_model != 'true' || needs.prepare_chestnut.result == 'success')
}}
needs: [ build, prepare_strategy, prepare_chestnut ]
runs-on: ubuntu-24.04
environment: ${{ needs.prepare_strategy.outputs.environment }}
steps:
- uses: actions/checkout@v4
- name: Download prebuilt artifact
uses: actions/download-artifact@v4
with:
name: prebuilt
- name: Untar prebuilt
run: |
mkdir -p ${{ env.OUTPUT_DIR }}
tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
- name: Prepare chestnut output
if: ${{ needs.prepare_chestnut.result == 'success' }}
run: |
mkdir -p "${{ github.workspace }}/chestnut_output"
tar xzf prebuilt.tar.gz -C "${{ github.workspace }}/chestnut_output"
- name: Download big model chunks from HF
if: ${{ needs.prepare_chestnut.result == 'success' }}
env:
HF_REPO: sunnypilot/sunnypilot_models_v1
HF_DEFAULTS_PATH: models/defaults/big
run: |
ONNX_HASH="${{ needs.prepare_chestnut.outputs.onnx_sha256 }}"
JSON_URL="https://huggingface.co/datasets/${HF_REPO}/resolve/main/${HF_DEFAULTS_PATH}/default_models.json"
DEFAULTS=$(curl -fsSL "$JSON_URL")
BUNDLE=$(echo "$DEFAULTS" | jq --arg hash "$ONNX_HASH" '.bundles[] | select(.onnx_sha256 == $hash)')
mkdir -p big_model_chunks
ARTIFACT=$(echo "$BUNDLE" | jq -r '.models[0].artifact')
BASE_URL=$(echo "$ARTIFACT" | jq -r '.download_uri.url' | sed 's|/[^/]*$||')
NUM_CHUNKS=$(echo "$ARTIFACT" | jq -r '.chunks | length')
CANONICAL="big_driving_tinygrad.pkl"
echo "$ARTIFACT" | jq -r '.chunks[].file_name' | while read CHUNK_NAME; do
CHUNK_IDX=$(echo "$CHUNK_NAME" | grep -oP 'chunk\K[0-9]+of[0-9]+')
CANONICAL_CHUNK="${CANONICAL}.chunk${CHUNK_IDX}"
ENCODED_URL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${BASE_URL}/${CHUNK_NAME}', safe=':/'))")
echo "Downloading $CHUNK_NAME -> $CANONICAL_CHUNK"
curl -fsSL -o "big_model_chunks/${CANONICAL_CHUNK}" "$ENCODED_URL"
done
echo "$NUM_CHUNKS" > "big_model_chunks/${CANONICAL}.chunkmanifest"
- name: Inject big model into chestnut
if: ${{ needs.prepare_chestnut.result == 'success' }}
run: |
cp big_model_chunks/* "${{ github.workspace }}/chestnut_output/openpilot/selfdrive/modeld/models/"
- name: Configure Git
run: |
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Publish to Public Repository
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo '${{ toJSON(needs.prepare_strategy.outputs) }}'
ls -la ${{ env.OUTPUT_DIR }}
${{ env.CI_DIR }}/publish.sh \
"${{ github.workspace }}" \
"${{ env.OUTPUT_DIR }}" \
"${{ needs.prepare_strategy.outputs.new_branch }}" \
"${{ needs.prepare_strategy.outputs.version }}" \
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
- name: Publish chestnut branch
if: ${{ needs.prepare_chestnut.result == 'success' }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
CHESTNUT_BRANCH="${{ needs.prepare_strategy.outputs.new_branch }}-chestnut"
CHESTNUT_DIR="${{ github.workspace }}/chestnut_output"
${{ env.CI_DIR }}/publish.sh \
"${{ github.workspace }}" \
"$CHESTNUT_DIR" \
"$CHESTNUT_BRANCH" \
"${{ needs.prepare_strategy.outputs.version }}" \
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
"${{ needs.prepare_strategy.outputs.extra_version_identifier }}"
- name: Tag ${{ needs.prepare_strategy.outputs.environment }}
if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
run: |
TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.prepare_strategy.outputs.build }}."
git push -f origin ${TAG}
notify:
needs:
- prepare_strategy
- build
- publish
- prepare_chestnut
runs-on: ubuntu-24.04
if: ${{ (always() && !cancelled() && !failure())
&& needs.publish.result == 'success'
&& (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
&& (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
steps:
- uses: actions/checkout@v4
- name: Prepare notification message
id: message
run: |
export VERSION="${{ needs.prepare_strategy.outputs.version }}"
export branch_name="${{ env.SOURCE_BRANCH }}"
export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
export commit_sha="${{ github.sha }}"
export commit_short_sha="${{ github.sha }}"
export commit_short_sha="${commit_short_sha:0:7}"
export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
export chestnut_branch="${{ needs.prepare_chestnut.result == 'success' && format('{0}-chestnut', needs.prepare_strategy.outputs.new_branch) || '' }}"
MESSAGE=$(cat << 'EOF' | envsubst
${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
EOF
)
{
echo 'content<<EOFMARKER'
echo "$MESSAGE"
echo 'EOFMARKER'
} >> $GITHUB_OUTPUT
shell: bash
- name: Post to Discourse
uses: ./.github/workflows/post-to-discourse
with:
discourse-url: ${{ vars.DISCOURSE_URL }}
api-key: ${{ secrets.DISCOURSE_API_KEY }}
api-username: "system"
topic-id: ${{ fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name].topic_id }}
message: ${{ steps.message.outputs.content }}
manage-pr-labels:
name: Remove prebuilt label
runs-on: ubuntu-latest
if: (always() && contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
env:
LABEL: prebuilt
steps:
- name: Remove trust-fork-pr label if present
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const prNumber = context.payload.pull_request.number;
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
name: process.env.LABEL
});
console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`);