mirror of
https://github.com/infiniteCable2/openpilot.git
synced 2026-08-06 00:36:29 +08:00
royjr
273 lines
11 KiB
YAML
273 lines
11 KiB
YAML
name: Build Model from Upstream
|
|
|
|
env:
|
|
BUILD_DIR: "/data/openpilot"
|
|
OUTPUT_DIR: ${{ github.workspace }}/output
|
|
SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
|
|
UPSTREAM_REPO: "commaai/openpilot"
|
|
TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
|
|
MODELS_DIR: ${{ github.workspace }}/openpilot/selfdrive/modeld/models
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
upstream_branch:
|
|
description: 'Upstream branch to build from'
|
|
required: true
|
|
default: 'master'
|
|
type: string
|
|
custom_name:
|
|
description: 'Custom name for the model (no date, only name)'
|
|
required: false
|
|
type: string
|
|
is_20hz:
|
|
description: 'Is this a 20Hz model'
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
artifact_suffix:
|
|
description: 'Suffix for artifact name'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
workflow_dispatch:
|
|
inputs:
|
|
upstream_branch:
|
|
description: 'Upstream branch to build from'
|
|
required: true
|
|
default: 'master'
|
|
type: string
|
|
custom_name:
|
|
description: 'Custom name for the model (no date, only name)'
|
|
required: false
|
|
type: string
|
|
is_20hz:
|
|
description: 'Is this a 20Hz model'
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
|
|
|
|
run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
|
|
|
|
jobs:
|
|
get_model:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
REF: ${{ inputs.upstream_branch }}
|
|
outputs:
|
|
model_date: ${{ steps.commit-date.outputs.model_date }}
|
|
steps:
|
|
# Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
|
|
# and fallback to sunnypilot if the ref checkout fails.
|
|
- name: Checkout commaai/openpilot
|
|
id: checkout_upstream
|
|
continue-on-error: true
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: commaai/openpilot
|
|
ref: ${{ inputs.upstream_branch }}
|
|
submodules: recursive
|
|
path: openpilot
|
|
|
|
- name: Fallback to sunnypilot/sunnypilot
|
|
if: steps.checkout_upstream.outcome == 'failure'
|
|
uses: actions/checkout@v4
|
|
with:
|
|
repository: sunnypilot/sunnypilot
|
|
ref: ${{ inputs.upstream_branch }}
|
|
submodules: recursive
|
|
path: openpilot
|
|
- name: Get commit date
|
|
id: commit-date
|
|
run: |
|
|
cd ${{ github.workspace }}/openpilot
|
|
commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
|
|
echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
|
|
cat $GITHUB_OUTPUT
|
|
- run: |
|
|
cd ${{ github.workspace }}/openpilot
|
|
git lfs pull
|
|
- name: 'Upload Artifact'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
|
|
path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
|
|
|
|
build_model:
|
|
runs-on: [self-hosted, tici]
|
|
needs: get_model
|
|
env:
|
|
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
|
|
REF: ${{ inputs.upstream_branch }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
submodules: recursive
|
|
|
|
- run: git lfs pull
|
|
- name: Cache SCons
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ${{env.SCONS_CACHE_DIR}}
|
|
key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
|
|
# Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
|
|
# for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
|
|
restore-keys: |
|
|
scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
|
|
scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
|
|
scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
|
|
scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
|
|
scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
|
|
scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
|
|
scons-${{ runner.os }}-${{ runner.arch }}
|
|
|
|
- name: Set environment variables
|
|
id: set-env
|
|
run: |
|
|
# Set up common environment
|
|
source /etc/profile;
|
|
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
|
export UV_PYTHON_PREFERENCE=managed
|
|
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
|
|
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
|
uv sync
|
|
printenv >> $GITHUB_ENV
|
|
if [[ "${{ runner.debug }}" == "1" ]]; then
|
|
cat $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Setup build environment
|
|
run: |
|
|
mkdir -p "${BUILD_DIR}/"
|
|
sudo find $BUILD_DIR/ -mindepth 1 -delete
|
|
echo "Starting build stage..."
|
|
echo "BUILD_DIR: ${BUILD_DIR}"
|
|
echo "CI_DIR: ${CI_DIR}"
|
|
echo "VERSION: ${{ steps.set-env.outputs.version }}"
|
|
echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
|
|
echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
|
|
echo "-------"
|
|
if [[ "${{ runner.debug }}" == "1" ]]; then
|
|
printenv
|
|
fi
|
|
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
|
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
|
|
rm -rf ${{ env.MODELS_DIR }}/*.onnx
|
|
|
|
- name: Download model artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
|
|
path: ${{ env.MODELS_DIR }}
|
|
- run: |
|
|
rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
|
|
|
|
- name: Build Model
|
|
run: |
|
|
source /etc/profile
|
|
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
|
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
|
export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
|
|
|
|
COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
|
|
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
|
|
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
|
|
TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
|
|
|
|
# Generate metadata for all ONNX files
|
|
find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
|
|
echo "Generating metadata: $onnx_file"
|
|
env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
|
|
done
|
|
|
|
# Detect model type and build compile args
|
|
VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
|
|
POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
|
|
OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
|
|
ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
|
|
SUPERCOMBO_ONNX="${{ env.MODELS_DIR }}/supercombo.onnx"
|
|
|
|
MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
|
|
if [ -f "$VISION_ONNX" ]; then
|
|
ONNX_ARGS="--vision-onnx $VISION_ONNX"
|
|
if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
|
|
MODEL_TYPE=vision_multi_policy
|
|
ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
|
|
elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
|
|
MODEL_TYPE=vision_multi_policy
|
|
ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
|
|
elif [ -f "$POLICY_ONNX" ]; then
|
|
MODEL_TYPE=vision_policy
|
|
ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
|
|
fi
|
|
elif [ -f "$SUPERCOMBO_ONNX" ]; then
|
|
MODEL_TYPE=supercombo
|
|
ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
|
|
fi
|
|
|
|
if [ -n "$MODEL_TYPE" ]; then
|
|
echo "Detected: $MODEL_TYPE -> driving_tinygrad.pkl"
|
|
env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
|
|
--model-type $MODEL_TYPE \
|
|
--model-size $MODEL_SIZE \
|
|
--camera-resolutions $CAMERA_RES \
|
|
$ONNX_ARGS \
|
|
--output "${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
|
|
fi
|
|
|
|
- name: Validate Model Outputs
|
|
run: |
|
|
source /etc/profile
|
|
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
|
|
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
|
|
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
|
|
--validate-only \
|
|
--model-dir "${{ env.MODELS_DIR }}"
|
|
|
|
- name: Prepare Output
|
|
run: |
|
|
sudo rm -rf ${{ env.OUTPUT_DIR }}
|
|
mkdir -p ${{ env.OUTPUT_DIR }}
|
|
|
|
# Copy the model files
|
|
rsync -avm \
|
|
--include='*.dlc' \
|
|
--include='*.pkl' \
|
|
--include='*.chunk*' \
|
|
--include='*.chunkmanifest' \
|
|
--include='*.onnx' \
|
|
--exclude='*' \
|
|
--delete-excluded \
|
|
--chown=comma:comma \
|
|
${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/
|
|
|
|
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
|
|
--model-dir "${{ env.MODELS_DIR }}" \
|
|
--output-dir "${{ env.OUTPUT_DIR }}" \
|
|
--custom-name "${{ env.MODEL_NAME }}" \
|
|
--upstream-branch "${{ inputs.upstream_branch }}" \
|
|
${{ inputs.is_20hz && '--is-20hz' || '' }}
|
|
|
|
- name: Write artifact name to file
|
|
run: echo "model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}" > ${{ env.OUTPUT_DIR }}/artifact_name.txt
|
|
|
|
- name: Upload Build Artifacts
|
|
id: upload-artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
|
|
path: ${{ env.OUTPUT_DIR }}
|
|
|
|
- name: Upload artifact name file
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
|
|
path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
|
|
|
|
- name: Re-enable powersave
|
|
if: always()
|
|
run: |
|
|
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
|
|
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
|